diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2487372b..42f8607e2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,7 +326,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.RustXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.RustXlangTest cpp: name: C++ CI @@ -412,7 +412,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CPPXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.CPPXlangTest cpp_examples: name: C++ Examples @@ -533,8 +533,10 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.PythonXlangTest - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CrossLanguageTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PythonXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PyCrossLanguageTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.CrossLanguageTest go: name: Golang CI @@ -599,7 +601,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.GoXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest lint: name: Code Style Check diff --git a/AGENTS.md b/AGENTS.md index 2322cb945a..9202efd6a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ Run C++ xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.CPPXlangTest +FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.CPPXlangTest ``` ### Python Development @@ -125,9 +125,9 @@ cd java mvn -T16 install -DskipTests cd fory-core # disable fory cython for faster debugging -FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest +FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest # enable fory cython -FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest +FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest ``` ### Golang Development @@ -159,7 +159,7 @@ Run Go xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.GoXlangTest +FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.GoXlangTest ``` ### Rust Development @@ -215,7 +215,7 @@ Run Rust xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.RustXlangTest +FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.RustXlangTest ``` ### JavaScript/TypeScript Development diff --git a/cpp/fory/meta/field.h b/cpp/fory/meta/field.h index a01bb8c646..7ec8835208 100644 --- a/cpp/fory/meta/field.h +++ b/cpp/fory/meta/field.h @@ -50,6 +50,12 @@ struct not_null {}; /// tracking). struct ref {}; +/// Tag to mark a polymorphic shared_ptr/unique_ptr field as monomorphic. +/// Use this when the field type has virtual methods but you know the actual +/// runtime type will always be exactly T (not a derived type). +/// This avoids dynamic type dispatch overhead during serialization. +struct monomorphic {}; + namespace detail { // ============================================================================ @@ -96,10 +102,12 @@ inline constexpr bool has_option_v = (std::is_same_v || ...); // ============================================================================ /// Compile-time field tag metadata entry -template struct FieldTagEntry { +template +struct FieldTagEntry { static constexpr int16_t id = Id; static constexpr bool is_nullable = Nullable; static constexpr bool track_ref = Ref; + static constexpr bool is_monomorphic = Monomorphic; }; /// Default: no field tags defined for type T @@ -163,6 +171,11 @@ template class field { "fory::ref is only valid for shared_ptr " "(reference tracking requires shared ownership)."); + // Validate: monomorphic only for smart pointers + static_assert(!detail::has_option_v || + detail::is_smart_ptr_v, + "fory::monomorphic is only valid for shared_ptr/unique_ptr."); + // Validate: no options for optional (inherently nullable) static_assert(!detail::is_optional_v || sizeof...(Options) == 0, "std::optional is inherently nullable. No options allowed."); @@ -189,6 +202,13 @@ template class field { static constexpr bool track_ref = detail::is_shared_ptr_v && detail::has_option_v; + /// Monomorphic serialization is enabled if: + /// - It's std::shared_ptr or std::unique_ptr with fory::monomorphic option + /// - When true, the serializer will not use dynamic type dispatch + static constexpr bool is_monomorphic = + detail::is_smart_ptr_v && + detail::has_option_v; + T value{}; // Default constructor @@ -309,6 +329,19 @@ struct field_track_ref> { template inline constexpr bool field_track_ref_v = field_track_ref::value; +/// Get is_monomorphic from field type +template struct field_is_monomorphic { + static constexpr bool value = false; +}; + +template +struct field_is_monomorphic> { + static constexpr bool value = field::is_monomorphic; +}; + +template +inline constexpr bool field_is_monomorphic_v = field_is_monomorphic::value; + // ============================================================================ // FORY_FIELD_TAGS Macro Support // ============================================================================ @@ -317,7 +350,7 @@ namespace detail { // Helper to parse field tag entry from macro arguments // Supports: (field, id), (field, id, nullable), (field, id, ref), -// (field, id, nullable, ref) +// (field, id, nullable, ref), (field, id, monomorphic), etc. template struct ParseFieldTagEntry { static constexpr bool is_nullable = @@ -327,6 +360,9 @@ struct ParseFieldTagEntry { static constexpr bool track_ref = is_shared_ptr_v && has_option_v; + static constexpr bool is_monomorphic = + is_smart_ptr_v && has_option_v; + // Compile-time validation static_assert(!has_option_v || is_smart_ptr_v, @@ -335,7 +371,11 @@ struct ParseFieldTagEntry { static_assert(!has_option_v || is_shared_ptr_v, "fory::ref is only valid for shared_ptr"); - using type = FieldTagEntry; + static_assert(!has_option_v || + is_smart_ptr_v, + "fory::monomorphic is only valid for shared_ptr/unique_ptr"); + + using type = FieldTagEntry; }; /// Get field tag entry by index from ForyFieldTagsImpl @@ -343,6 +383,7 @@ template struct GetFieldTagEntry { static constexpr int16_t id = -1; static constexpr bool is_nullable = false; static constexpr bool track_ref = false; + static constexpr bool is_monomorphic = false; }; template @@ -355,6 +396,7 @@ struct GetFieldTagEntry< static constexpr int16_t id = Entry::id; static constexpr bool is_nullable = Entry::is_nullable; static constexpr bool track_ref = Entry::track_ref; + static constexpr bool is_monomorphic = Entry::is_monomorphic; }; } // namespace detail @@ -377,12 +419,14 @@ struct GetFieldTagEntry< #define FORY_FT_GET_OPT1_IMPL(f, i, o1, ...) o1 #define FORY_FT_GET_OPT2(tuple) FORY_FT_GET_OPT2_IMPL tuple #define FORY_FT_GET_OPT2_IMPL(f, i, o1, o2, ...) o2 +#define FORY_FT_GET_OPT3(tuple) FORY_FT_GET_OPT3_IMPL tuple +#define FORY_FT_GET_OPT3_IMPL(f, i, o1, o2, o3, ...) o3 -// Detect number of elements in tuple: 2, 3, or 4 +// Detect number of elements in tuple: 2, 3, 4, or 5 #define FORY_FT_TUPLE_SIZE(tuple) FORY_FT_TUPLE_SIZE_IMPL tuple #define FORY_FT_TUPLE_SIZE_IMPL(...) \ - FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 4, 3, 2, 1, 0) -#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, N, ...) N + FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 5, 4, 3, 2, 1, 0) +#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, _5, N, ...) N // Create FieldTagEntry based on tuple size using indirect call pattern // This pattern ensures the concatenated macro name is properly rescanned @@ -408,6 +452,12 @@ struct GetFieldTagEntry< decltype(std::declval().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple), \ ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple)>::type +#define FORY_FT_MAKE_ENTRY_5(Type, tuple) \ + typename ::fory::detail::ParseFieldTagEntry< \ + decltype(std::declval().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple), \ + ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple), \ + ::fory::FORY_FT_GET_OPT3(tuple)>::type + // Main macro: FORY_FIELD_TAGS(Type, (field1, id1), (field2, id2, nullable),...) // Note: Uses fory::detail:: instead of ::fory::detail:: for GCC compatibility #define FORY_FIELD_TAGS(Type, ...) \ diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index eec5f7a0c1..1000fdb31d 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -269,8 +269,12 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, if (is_same_type) { bitmap |= COLL_IS_SAME_TYPE; } + // Only set TRACKING_REF if element is shared ref AND global ref tracking is + // enabled if constexpr (elem_is_shared_ref) { - bitmap |= COLL_TRACKING_REF; + if (ctx.track_ref()) { + bitmap |= COLL_TRACKING_REF; + } } // Write header @@ -287,52 +291,54 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, } } + // Determine if we're actually tracking refs for this collection + const bool tracking_refs = (bitmap & COLL_TRACKING_REF) != 0; + // Write elements if (is_same_type) { // All elements have same type - type info written once above - if (!has_null) { - if constexpr (elem_is_shared_ref) { - // Write with ref flag, without type - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::NullOnly, false, - has_generics); - } - } else { - // Write data directly - for (const auto &elem : coll) { - if constexpr (is_nullable_v) { - using Inner = nullable_element_t; - Serializer::write_data(deref_nullable(elem), ctx); + if (tracking_refs) { + // Track refs - write ref flag per element per xlang spec + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::Tracking, false, has_generics); + } + } else if (!has_null) { + // No nulls, no ref tracking - write data directly without null flag + for (const auto &elem : coll) { + if constexpr (is_nullable_v) { + using Inner = nullable_element_t; + Serializer::write_data(deref_nullable(elem), ctx); + } else if constexpr (elem_is_shared_ref) { + // For shared_ptr, use write_data which handles polymorphic types + Serializer::write_data(elem, ctx); + } else { + if constexpr (is_generic_type_v) { + Serializer::write_data_generic(elem, ctx, has_generics); } else { - if constexpr (is_generic_type_v) { - Serializer::write_data_generic(elem, ctx, has_generics); - } else { - Serializer::write_data(elem, ctx); - } + Serializer::write_data(elem, ctx); } } } } else { - // Has null elements - write with ref flag for null tracking + // Has null elements - write with null flag for (const auto &elem : coll) { Serializer::write(elem, ctx, RefMode::NullOnly, false, has_generics); } } } else { // Heterogeneous types - write type info per element - if (!has_null) { - if constexpr (elem_is_shared_ref) { - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::NullOnly, true, - has_generics); - } - } else { - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::None, true, has_generics); - } + if (tracking_refs) { + // Track refs - write ref flag + type info per element + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::Tracking, true, has_generics); + } + } else if (!has_null) { + // No nulls - write without null flag (RefMode::None) + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::None, true, has_generics); } } else { - // Has null elements + // Has null elements - write with null flag (RefMode::NullOnly) for (const auto &elem : coll) { Serializer::write(elem, ctx, RefMode::NullOnly, true, has_generics); } @@ -416,7 +422,8 @@ inline Container read_collection_data_slow(ReadContext &ctx, uint32_t length) { return result; } if constexpr (elem_is_polymorphic) { - auto elem = Serializer::read_with_type_info(ctx, RefMode::NullOnly, + // Use RefMode::Tracking to read ref flag per element + auto elem = Serializer::read_with_type_info(ctx, RefMode::Tracking, *elem_type_info); collection_insert(result, std::move(elem)); } else { @@ -1609,8 +1616,12 @@ struct Serializer> { if (is_same_type) { bitmap |= COLL_IS_SAME_TYPE; } + // Only set TRACKING_REF if element is shared ref AND global ref tracking + // is enabled if constexpr (elem_is_shared_ref) { - bitmap |= COLL_TRACKING_REF; + if (ctx.track_ref()) { + bitmap |= COLL_TRACKING_REF; + } } // Write header diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 7a0f93e802..cfd288ea31 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -623,8 +623,11 @@ class Fory : public BaseFory { buffer.WriteInt32(-1); // Placeholder for meta offset (fixed 4 bytes) } - // Top-level serialization: YES ref flags, yes type info - Serializer::write(obj, *write_ctx_, RefMode::NullOnly, true); + // Top-level serialization: use Tracking if ref tracking is enabled, + // otherwise NullOnly for nullable handling + const RefMode top_level_ref_mode = + write_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; + Serializer::write(obj, *write_ctx_, top_level_ref_mode, true); // Check for errors at serialization boundary if (FORY_PREDICT_FALSE(write_ctx_->has_error())) { return Unexpected(write_ctx_->take_error()); @@ -654,8 +657,11 @@ class Fory : public BaseFory { } } - // Top-level deserialization: YES ref flags, yes type info - T result = Serializer::read(*read_ctx_, RefMode::NullOnly, true); + // Top-level deserialization: use Tracking if ref tracking is enabled, + // otherwise NullOnly for nullable handling + const RefMode top_level_ref_mode = + read_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; + T result = Serializer::read(*read_ctx_, top_level_ref_mode, true); // Check for errors at deserialization boundary if (FORY_PREDICT_FALSE(read_ctx_->has_error())) { return Unexpected(read_ctx_->take_error()); diff --git a/cpp/fory/serialization/map_serializer.h b/cpp/fory/serialization/map_serializer.h index 767726fa65..8947b2773f 100644 --- a/cpp/fory/serialization/map_serializer.h +++ b/cpp/fory/serialization/map_serializer.h @@ -206,8 +206,6 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, constexpr bool val_is_polymorphic = is_polymorphic_v; constexpr bool key_is_shared_ref = is_shared_ref_v; constexpr bool val_is_shared_ref = is_shared_ref_v; - constexpr bool key_needs_ref = requires_ref_metadata_v; - constexpr bool val_needs_ref = requires_ref_metadata_v; const bool is_key_declared = has_generics && !need_to_write_type_for_field(); @@ -251,7 +249,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, // Non-null key, null value // Java writes: chunk_header, then ref_flag, then type_info, then data uint8_t chunk_header = VALUE_NULL; - bool write_ref = key_is_shared_ref || key_needs_ref; + // Only track refs for shared_ptr types when global ref tracking enabled + bool write_ref = key_is_shared_ref && ctx.track_ref(); if (write_ref) { chunk_header |= TRACKING_KEY_REF; } @@ -297,7 +296,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, // key_is_none // Java writes: chunk_header, then ref_flag, then type_info, then data uint8_t chunk_header = KEY_NULL; - bool write_ref = val_is_shared_ref || val_needs_ref; + // Only track refs for shared_ptr types when global ref tracking enabled + bool write_ref = val_is_shared_ref && ctx.track_ref(); if (write_ref) { chunk_header |= TRACKING_VALUE_REF; } @@ -385,16 +385,18 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, ctx.write_uint16(0); // Placeholder for header and chunk size uint8_t chunk_header = 0; - // Set key flags - if (key_is_shared_ref || key_needs_ref) { + // Set key flags - only track refs for shared_ptr when global ref tracking + // enabled + if (key_is_shared_ref && ctx.track_ref()) { chunk_header |= TRACKING_KEY_REF; } if (is_key_declared && !key_is_polymorphic) { chunk_header |= DECL_KEY_TYPE; } - // Set value flags - if (val_is_shared_ref || val_needs_ref) { + // Set value flags - only track refs for shared_ptr when global ref + // tracking enabled + if (val_is_shared_ref && ctx.track_ref()) { chunk_header |= TRACKING_VALUE_REF; } if (is_val_declared && !val_is_polymorphic) { @@ -443,12 +445,15 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } // Write key-value pair - // For polymorphic types, we've already written type info above, - // so we write ref flag + data directly using the serializer + // For shared_ptr with ref tracking: write ref flag + data + // For other types: null cases already handled via KEY_NULL/VALUE_NULL, + // so just write data directly if constexpr (key_is_shared_ref) { - Serializer::write(key, ctx, RefMode::NullOnly, false, has_generics); - } else if constexpr (key_needs_ref) { - Serializer::write(key, ctx, RefMode::NullOnly, false); + if (ctx.track_ref()) { + Serializer::write(key, ctx, RefMode::Tracking, false, has_generics); + } else { + Serializer::write_data(key, ctx); + } } else { if (has_generics && is_generic_type_v) { Serializer::write_data_generic(key, ctx, has_generics); @@ -458,9 +463,12 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } if constexpr (val_is_shared_ref) { - Serializer::write(value, ctx, RefMode::NullOnly, false, has_generics); - } else if constexpr (val_needs_ref) { - Serializer::write(value, ctx, RefMode::NullOnly, false); + if (ctx.track_ref()) { + Serializer::write(value, ctx, RefMode::Tracking, false, + has_generics); + } else { + Serializer::write_data(value, ctx); + } } else { if (has_generics && is_generic_type_v) { Serializer::write_data_generic(value, ctx, has_generics); @@ -822,15 +830,16 @@ inline MapType read_map_data_slow(ReadContext &ctx, uint32_t length) { // Read key - use type info if available (polymorphic case) K key; if constexpr (key_is_polymorphic) { + // TRACKING_KEY_REF means full ref tracking for shared_ptr key = Serializer::read_with_type_info( - ctx, key_read_ref ? RefMode::NullOnly : RefMode::None, + ctx, key_read_ref ? RefMode::Tracking : RefMode::None, *key_type_info); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } } else if (key_read_ref) { - key = - Serializer::read(ctx, make_ref_mode(false, key_read_ref), false); + // TRACKING_KEY_REF means full ref tracking for shared_ptr + key = Serializer::read(ctx, RefMode::Tracking, false); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } @@ -845,15 +854,16 @@ inline MapType read_map_data_slow(ReadContext &ctx, uint32_t length) { // Read value - use type info if available (polymorphic case) V value; if constexpr (val_is_polymorphic) { + // TRACKING_VALUE_REF means full ref tracking for shared_ptr value = Serializer::read_with_type_info( - ctx, val_read_ref ? RefMode::NullOnly : RefMode::None, + ctx, val_read_ref ? RefMode::Tracking : RefMode::None, *value_type_info); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } } else if (val_read_ref) { - value = - Serializer::read(ctx, make_ref_mode(false, val_read_ref), false); + // TRACKING_VALUE_REF means full ref tracking for shared_ptr + value = Serializer::read(ctx, RefMode::Tracking, false); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } diff --git a/cpp/fory/serialization/ref_resolver.h b/cpp/fory/serialization/ref_resolver.h index cbea3b6e4b..cde89a1165 100644 --- a/cpp/fory/serialization/ref_resolver.h +++ b/cpp/fory/serialization/ref_resolver.h @@ -78,6 +78,11 @@ class RefWriter { return false; } + /// Reserve a ref_id slot without storing a pointer. + /// Used for types (like structs) that Java tracks but C++ doesn't reference. + /// This keeps ref ID numbering in sync across languages. + uint32_t reserve_ref_id() { return next_id_++; } + /// Reset resolver for reuse in new serialization. /// Clears all tracked references. void reset() { diff --git a/cpp/fory/serialization/smart_ptr_serializer_test.cc b/cpp/fory/serialization/smart_ptr_serializer_test.cc index 7cea160da8..63f20ff335 100644 --- a/cpp/fory/serialization/smart_ptr_serializer_test.cc +++ b/cpp/fory/serialization/smart_ptr_serializer_test.cc @@ -441,5 +441,77 @@ TEST(SmartPtrSerializerTest, MaxDynDepthDefault) { } } // namespace + +// ============================================================================ +// Monomorphic field tests (fory::field<> style) +// ============================================================================ +namespace { + +// A polymorphic base class (has virtual methods) +struct PolymorphicBaseForMono { + virtual ~PolymorphicBaseForMono() = default; + virtual std::string name() const { return "PolymorphicBaseForMono"; } + int32_t value = 0; + std::string data; +}; +FORY_STRUCT(PolymorphicBaseForMono, value, data); + +// Holder with monomorphic field using fory::field<> +struct MonomorphicFieldHolder { + // Field marked as monomorphic - no dynamic type dispatch, always + // PolymorphicBaseForMono + fory::field, 0, fory::nullable, + fory::monomorphic> + ptr; +}; +FORY_STRUCT(MonomorphicFieldHolder, ptr); + +TEST(SmartPtrSerializerTest, MonomorphicFieldWithForyField) { + MonomorphicFieldHolder original; + original.ptr.value = std::make_shared(); + original.ptr.value->value = 42; + original.ptr.value->data = "test data"; + + auto fory = Fory::builder().track_ref(false).build(); + fory.register_struct(400); + fory.register_struct(401); + + auto bytes_result = fory.serialize(original); + ASSERT_TRUE(bytes_result.ok()) << bytes_result.error().to_string(); + + auto deserialize_result = fory.deserialize( + bytes_result->data(), bytes_result->size()); + ASSERT_TRUE(deserialize_result.ok()) + << deserialize_result.error().to_string(); + + auto deserialized = std::move(deserialize_result).value(); + ASSERT_TRUE(deserialized.ptr.value); + EXPECT_EQ(deserialized.ptr.value->value, 42); + EXPECT_EQ(deserialized.ptr.value->data, "test data"); + EXPECT_EQ(deserialized.ptr.value->name(), "PolymorphicBaseForMono"); +} + +TEST(SmartPtrSerializerTest, MonomorphicFieldNullValue) { + MonomorphicFieldHolder original; + original.ptr.value = nullptr; + + auto fory = Fory::builder().track_ref(false).build(); + fory.register_struct(404); + fory.register_struct(405); + + auto bytes_result = fory.serialize(original); + ASSERT_TRUE(bytes_result.ok()) << bytes_result.error().to_string(); + + auto deserialize_result = fory.deserialize( + bytes_result->data(), bytes_result->size()); + ASSERT_TRUE(deserialize_result.ok()) + << deserialize_result.error().to_string(); + + auto deserialized = std::move(deserialize_result).value(); + EXPECT_FALSE(deserialized.ptr.value); +} + +} // namespace + } // namespace serialization } // namespace fory \ No newline at end of file diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index c0ae5ab5e7..0ca9cfc26b 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -253,7 +253,17 @@ template struct SharedPtrTypeIdHelper { /// Supports reference tracking for shared and circular references. /// When reference tracking is enabled, identical shared_ptr instances /// will serialize only once and use reference IDs for subsequent occurrences. +/// +/// Note: The element type T must be a value type (struct/class or primitive). +/// Raw pointers and nullable wrappers are not allowed as they would require +/// nested ref metadata handling, which complicates the protocol. template struct Serializer> { + static_assert(!std::is_pointer_v, + "shared_ptr of raw pointer types is not supported"); + static_assert(!requires_ref_metadata_v, + "shared_ptr of nullable types (optional/shared_ptr/unique_ptr) " + "is not supported. Use the wrapper type directly instead."); + static constexpr TypeId type_id = SharedPtrTypeIdHelper>::value; @@ -279,7 +289,6 @@ template struct Serializer> { static inline void write(const std::shared_ptr &ptr, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -310,9 +319,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } return; } @@ -359,10 +367,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - // Non-polymorphic path - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } } @@ -416,30 +422,39 @@ template struct Serializer> { static inline std::shared_ptr read(ReadContext &ctx, RefMode ref_mode, bool read_type) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) if (ref_mode == RefMode::None) { if constexpr (is_polymorphic) { - // For polymorphic types, we must read type info when read_type=true - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::shared_ptr " - "without type info (read_type=false)")); - return nullptr; - } - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; + if (read_type) { + // Polymorphic path: read type info from stream to get concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } + // Now use read_with_type_info with the concrete type info + return read_with_type_info(ctx, ref_mode, *type_info); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_shared(std::move(value)); + } } - // Now use read_with_type_info with the concrete type info - return read_with_type_info(ctx, ref_mode, *type_info); } else { - constexpr RefMode inner_ref_mode = - inner_requires_ref ? RefMode::NullOnly : RefMode::None; - T value = Serializer::read(ctx, inner_ref_mode, read_type); + // T is guaranteed to be a value type (not pointer or nullable wrapper) + // by static_assert, so no inner ref metadata needed. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -453,7 +468,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::shared_ptr(nullptr); + return nullptr; } const bool tracking_refs = ctx.track_ref(); if (flag == REF_FLAG) { @@ -481,7 +496,8 @@ template struct Serializer> { } uint32_t reserved_ref_id = 0; - if (flag == REF_VALUE_FLAG) { + const bool is_first_occurrence = flag == REF_VALUE_FLAG; + if (is_first_occurrence) { if (!tracking_refs) { ctx.set_error(Error::invalid_ref( "REF_VALUE flag encountered when reference tracking disabled")); @@ -492,48 +508,62 @@ template struct Serializer> { // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::shared_ptr " - "without type info (read_type=false)")); - return nullptr; - } - - // Check and increase dynamic depth for polymorphic deserialization - auto depth_res = ctx.increase_dyn_depth(); - if (!depth_res.ok()) { - ctx.set_error(std::move(depth_res).error()); - return nullptr; - } - DynDepthGuard dyn_depth_guard(ctx); + if (read_type) { + // Polymorphic path: read type info and use harness for deserialization + // Check and increase dynamic depth for polymorphic deserialization + auto depth_res = ctx.increase_dyn_depth(); + if (!depth_res.ok()) { + ctx.set_error(std::move(depth_res).error()); + return nullptr; + } + DynDepthGuard dyn_depth_guard(ctx); - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; - } + // Read type info from stream to get the concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } - // Use the harness to deserialize the concrete type - void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return nullptr; - } - T *obj_ptr = static_cast(raw_ptr); - auto result = std::shared_ptr(obj_ptr); - if (flag == REF_VALUE_FLAG) { - ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + // Use the harness to deserialize the concrete type + void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return nullptr; + } + T *obj_ptr = static_cast(raw_ptr); + auto result = std::shared_ptr(obj_ptr); + if (is_first_occurrence) { + ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + } + return result; + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + auto result = std::make_shared(std::move(value)); + if (is_first_occurrence) { + ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + } + return result; + } } - return result; } else { - // Non-polymorphic path - T value = Serializer::read( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - read_type); + // Non-polymorphic path: T is guaranteed to be a value type (not pointer + // or nullable wrapper) by static_assert, so no inner ref metadata needed. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } auto result = std::make_shared(std::move(value)); - if (flag == REF_VALUE_FLAG) { + if (is_first_occurrence) { ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); } return result; @@ -543,7 +573,6 @@ template struct Serializer> { static inline std::shared_ptr read_with_type_info(ReadContext &ctx, RefMode ref_mode, const TypeInfo &type_info) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -557,10 +586,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::shared_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -575,7 +603,7 @@ template struct Serializer> { } if (flag == NULL_FLAG) { - return std::shared_ptr(nullptr); + return nullptr; } const bool tracking_refs = ctx.track_ref(); if (flag == REF_FLAG) { @@ -634,10 +662,9 @@ template struct Serializer> { } return result; } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -645,7 +672,6 @@ template struct Serializer> { if (flag == REF_VALUE_FLAG) { ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); } - return result; } } @@ -678,14 +704,23 @@ template struct UniquePtrTypeIdHelper { /// Note: unique_ptr does not support reference tracking since /// it represents exclusive ownership. Each unique_ptr is serialized /// independently. +/// +/// Note: The element type T must be a value type (struct/class or primitive). +/// Raw pointers and nullable wrappers are not allowed as they would require +/// nested ref metadata handling, which complicates the protocol. template struct Serializer> { + static_assert(!std::is_pointer_v, + "unique_ptr of raw pointer types is not supported"); + static_assert(!requires_ref_metadata_v, + "unique_ptr of nullable types (optional/shared_ptr/unique_ptr) " + "is not supported. Use the wrapper type directly instead."); + static constexpr TypeId type_id = UniquePtrTypeIdHelper>::value; static inline void write(const std::unique_ptr &ptr, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -716,9 +751,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } return; } @@ -751,9 +785,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } } @@ -805,30 +838,38 @@ template struct Serializer> { static inline std::unique_ptr read(ReadContext &ctx, RefMode ref_mode, bool read_type) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) if (ref_mode == RefMode::None) { if constexpr (is_polymorphic) { - // For polymorphic types, we must read type info when read_type=true - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::unique_ptr " - "without type info (read_type=false)")); - return nullptr; - } - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; + if (read_type) { + // Polymorphic path: read type info from stream to get concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } + // Now use read_with_type_info with the concrete type info + return read_with_type_info(ctx, ref_mode, *type_info); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_unique(std::move(value)); + } } - // Now use read_with_type_info with the concrete type info - return read_with_type_info(ctx, ref_mode, *type_info); } else { - constexpr RefMode inner_ref_mode = - inner_requires_ref ? RefMode::NullOnly : RefMode::None; - T value = Serializer::read(ctx, inner_ref_mode, read_type); + // T is guaranteed to be a value type by static_assert. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -842,7 +883,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::unique_ptr(nullptr); + return nullptr; } if (flag != NOT_NULL_VALUE_FLAG) { ctx.set_error( @@ -853,39 +894,48 @@ template struct Serializer> { // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::unique_ptr " - "without type info (read_type=false)")); - return nullptr; - } - - // Check and increase dynamic depth for polymorphic deserialization - auto depth_res = ctx.increase_dyn_depth(); - if (!depth_res.ok()) { - ctx.set_error(std::move(depth_res).error()); - return nullptr; - } - DynDepthGuard dyn_depth_guard(ctx); + if (read_type) { + // Polymorphic path: read type info and use harness for deserialization + // Check and increase dynamic depth for polymorphic deserialization + auto depth_res = ctx.increase_dyn_depth(); + if (!depth_res.ok()) { + ctx.set_error(std::move(depth_res).error()); + return nullptr; + } + DynDepthGuard dyn_depth_guard(ctx); - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; - } + // Read type info from stream to get the concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } - // Use the harness to deserialize the concrete type - void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return nullptr; + // Use the harness to deserialize the concrete type + void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return nullptr; + } + T *obj_ptr = static_cast(raw_ptr); + return std::unique_ptr(obj_ptr); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_unique(std::move(value)); + } } - T *obj_ptr = static_cast(raw_ptr); - return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - read_type); + // T is guaranteed to be a value type by static_assert. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -896,7 +946,6 @@ template struct Serializer> { static inline std::unique_ptr read_with_type_info(ReadContext &ctx, RefMode ref_mode, const TypeInfo &type_info) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -910,10 +959,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -927,7 +975,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::unique_ptr(nullptr); + return nullptr; } if (flag != NOT_NULL_VALUE_FLAG) { ctx.set_error( @@ -954,10 +1002,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 3b6ccdcf57..cd27a5cffb 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -358,6 +358,31 @@ template struct CompileTimeFieldHelpers { } } + /// Returns true if the field at Index is marked as monomorphic. + /// Use this for shared_ptr/unique_ptr fields with polymorphic inner types + /// when you know the actual runtime type will always be exactly T. + template static constexpr bool field_monomorphic() { + if constexpr (FieldCount == 0) { + return false; + } else { + using PtrT = std::tuple_element_t; + using RawFieldType = meta::RemoveMemberPointerCVRefT; + + // If it's a fory::field<> wrapper, use its is_monomorphic metadata + if constexpr (is_fory_field_v) { + return RawFieldType::is_monomorphic; + } + // Else if FORY_FIELD_TAGS is defined, use that metadata + else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::is_monomorphic; + } + // Default: not monomorphic (polymorphic types use dynamic dispatch) + else { + return false; + } + } + } + /// Get the underlying field type (unwraps fory::field<> if present) template struct UnwrappedFieldTypeHelper { using PtrT = std::tuple_element_t; @@ -1292,18 +1317,17 @@ void write_single_field(const T &obj, WriteContext &ctx, // Enums: false (per Rust util.rs:58-59) // Structs/EXT: true ONLY in compatible mode (per C++ read logic) // Others: false - constexpr bool is_struct = field_type_id == TypeId::STRUCT || - field_type_id == TypeId::COMPATIBLE_STRUCT || - field_type_id == TypeId::NAMED_STRUCT || - field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; - constexpr bool is_ext = - field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_struct = is_struct_type(field_type_id); + constexpr bool is_ext = is_ext_type(field_type_id); constexpr bool is_polymorphic = field_type_id == TypeId::UNKNOWN; + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + // Per C++ read logic: struct fields need type info only in compatible mode - // Polymorphic types always need type info - bool write_type = - is_polymorphic || ((is_struct || is_ext) && ctx.is_compatible()); + // Polymorphic types always need type info, UNLESS marked as monomorphic + bool write_type = (is_polymorphic && !is_monomorphic) || + ((is_struct || is_ext) && ctx.is_compatible()); Serializer::write(field_value, ctx, field_ref_mode, write_type); } @@ -1456,10 +1480,17 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { // `Serializer::read` can dispatch to `read_compatible` with the correct // remote schema. constexpr bool field_requires_ref = requires_ref_metadata_v; - constexpr bool is_struct_field = is_fory_serializable_v; - constexpr bool is_polymorphic_field = - Serializer::type_id == TypeId::UNKNOWN; - bool read_type = is_polymorphic_field; + constexpr TypeId field_type_id = Serializer::type_id; + // Check if field is a struct type - use type_id to handle shared_ptr + constexpr bool is_struct_field = is_struct_type(field_type_id); + constexpr bool is_ext_field = is_ext_type(field_type_id); + constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; + + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + + // Polymorphic types need type info, UNLESS marked as monomorphic + bool read_type = is_polymorphic_field && !is_monomorphic; // Get field metadata from fory::field<> or FORY_FIELD_TAGS or defaults constexpr bool is_nullable = Helpers::template field_nullable(); @@ -1470,14 +1501,14 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { // `Serializer::read` can dispatch to `read_compatible` with the correct // remote TypeMeta instead of treating the bytes as part of the first field // value. - if (!is_polymorphic_field && is_struct_field && ctx.is_compatible()) { + if (!is_polymorphic_field && (is_struct_field || is_ext_field) && + ctx.is_compatible()) { read_type = true; } // Per xlang spec, all non-primitive fields have ref flags. // Primitive types: bool, int8-64, var_int32/64, sli_int64, float16/32/64 // Non-primitives include: string, list, set, map, struct, enum, etc. - constexpr TypeId field_type_id = Serializer::type_id; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); // Compute RefMode based on field metadata @@ -1526,6 +1557,7 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { template void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, RefMode remote_ref_mode) { + using Helpers = CompileTimeFieldHelpers; const auto field_info = ForyFieldInfo(obj); const auto field_ptrs = decltype(field_info)::Ptrs; const auto field_ptr = std::get(field_ptrs); @@ -1534,20 +1566,26 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, // Unwrap fory::field<> to get the actual type for deserialization using FieldType = unwrap_field_t; - constexpr bool is_struct_field = is_fory_serializable_v; - constexpr bool is_polymorphic_field = - Serializer::type_id == TypeId::UNKNOWN; constexpr TypeId field_type_id = Serializer::type_id; + // Check if field is a struct type - use type_id to handle shared_ptr + constexpr bool is_struct_field = is_struct_type(field_type_id); + constexpr bool is_ext_field = is_ext_type(field_type_id); + constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); - bool read_type = is_polymorphic_field; + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + + // Polymorphic types need type info, UNLESS marked as monomorphic + bool read_type = is_polymorphic_field && !is_monomorphic; // In compatible mode, nested struct fields always carry type metadata // (xtypeId + meta index). We must read this metadata so that // `Serializer::read` can dispatch to `read_compatible` with the correct // remote TypeMeta instead of treating the bytes as part of the first field // value. - if (!is_polymorphic_field && is_struct_field && ctx.is_compatible()) { + if (!is_polymorphic_field && (is_struct_field || is_ext_field) && + ctx.is_compatible()) { read_type = true; } @@ -1947,7 +1985,15 @@ struct Serializer>> { static void write(const T &obj, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - write_not_null_ref_flag(ctx, ref_mode); + // Handle ref flag based on mode + if (ref_mode == RefMode::Tracking && ctx.track_ref()) { + // In Tracking mode, write REF_VALUE_FLAG (0) and reserve a ref_id slot + // to keep ref IDs in sync with Java (which tracks all objects) + ctx.write_int8(REF_VALUE_FLAG); + ctx.ref_writer().reserve_ref_id(); + } else if (ref_mode != RefMode::None) { + ctx.write_int8(NOT_NULL_VALUE_FLAG); + } if (write_type) { // Direct lookup using compile-time type_index() - O(1) hash lookup @@ -2048,6 +2094,13 @@ struct Serializer>> { constexpr int8_t null_flag = static_cast(RefFlag::Null); if (ref_flag == not_null_value_flag || ref_flag == ref_value_flag) { + // When ref_flag is RefValue (0), Java assigned a ref_id to this object. + // We must reserve a matching ref_id slot so that nested refs line up. + // Structs can't actually be referenced (only shared_ptrs can), but we + // need the ref_id numbering to stay in sync with Java. + if (ctx.track_ref() && ref_flag == ref_value_flag) { + ctx.ref_reader().reserve_ref_id(); + } // In compatible mode: use meta sharing (matches Rust behavior) if (ctx.is_compatible()) { // In compatible mode: always use remote TypeMeta for schema evolution diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 3f85389c4f..3380c553d5 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -133,8 +133,11 @@ Result, Error> FieldInfo::to_bytes() const { uint8_t header = (std::min(FIELD_NAME_SIZE_THRESHOLD, name_size - 1) << 2) & 0x3C; + if (field_type.ref_tracking) { + header |= 1; // bit 0 for ref tracking + } if (field_type.nullable) { - header |= 2; + header |= 2; // bit 1 for nullable } header |= (encoding_idx << 6); @@ -974,16 +977,19 @@ std::string TypeMeta::compute_struct_fingerprint( // Java's ObjectSerializer.getTypeId returns Types.UNKNOWN (0) for: // - abstract classes, interfaces, and enum types + // - user-defined struct types (in xlang mode) // This aligns the hash computation with Java's behavior. uint32_t effective_type_id = fi.field_type.type_id; if (effective_type_id == static_cast(TypeId::ENUM) || - effective_type_id == static_cast(TypeId::NAMED_ENUM)) { + effective_type_id == static_cast(TypeId::NAMED_ENUM) || + effective_type_id == static_cast(TypeId::STRUCT) || + effective_type_id == static_cast(TypeId::NAMED_STRUCT)) { effective_type_id = static_cast(TypeId::UNKNOWN); } fingerprint.append(std::to_string(effective_type_id)); fingerprint.push_back(','); - // ref flag: currently always 0 in C++ (no ref tracking support yet) - fingerprint.push_back('0'); + // Use field-level ref tracking flag from FORY_FIELD_TAGS or fory::field<> + fingerprint.push_back(fi.field_type.ref_tracking ? '1' : '0'); fingerprint.push_back(','); fingerprint.append(fi.field_type.nullable ? "1;" : "0;"); } @@ -1002,9 +1008,10 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { uint64_t low = static_cast(hash_out[0]); uint32_t version = static_cast(low & 0xFFFF'FFFFu); #ifdef FORY_DEBUG + // DEBUG: Print fingerprint for debugging version mismatch std::cerr << "[xlang][debug] struct_version type_name=" << meta.type_name - << ", fingerprint=\"" << fingerprint << "\" version=" << version - << std::endl; + << ", fingerprint=\"" << fingerprint + << "\" version=" << static_cast(version) << std::endl; #endif return static_cast(version); } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 7fb6cc8a26..8baa42042b 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -482,6 +482,33 @@ struct FieldTypeBuilder< } }; +// Helper template functions to compute is_nullable and track_ref at compile +// time. These replace constexpr lambdas which have issues on MSVC. +template +constexpr bool compute_is_nullable() { + if constexpr (is_fory_field_v) { + return ActualFieldType::is_nullable; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::is_nullable; + } else { + // Default: nullable if std::optional or std::shared_ptr + return is_optional_v || + is_shared_ptr_v; + } +} + +template +constexpr bool compute_track_ref() { + if constexpr (is_fory_field_v) { + return ActualFieldType::track_ref; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::track_ref; + } else { + return false; + } +} + template struct FieldInfoBuilder { static FieldInfo build() { const auto meta = ForyFieldInfo(T{}); @@ -497,7 +524,22 @@ template struct FieldInfoBuilder { // Unwrap fory::field<> to get the underlying type for FieldTypeBuilder using UnwrappedFieldType = fory::unwrap_field_t; + // Get nullable and track_ref from field tags (FORY_FIELD_TAGS or + // fory::field<>) + constexpr bool is_nullable = + compute_is_nullable(); + constexpr bool track_ref = compute_track_ref(); + FieldType field_type = FieldTypeBuilder::build(false); + // Override nullable and ref_tracking from field-level metadata + field_type.nullable = is_nullable; + field_type.ref_tracking = track_ref; + field_type.ref_mode = make_ref_mode(is_nullable, track_ref); + // DEBUG: Print field info for debugging fingerprint mismatch + std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() + << " Index=" << Index << " field=" << field_name << " has_tags=" + << ::fory::detail::has_field_tags_v << " is_nullable=" + << is_nullable << " track_ref=" << track_ref << std::endl; return FieldInfo(std::move(field_name), std::move(field_type)); } }; diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index f63fdeaed9..9cc1bb9d8d 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -464,6 +464,94 @@ FORY_STRUCT(NullableComprehensiveCompatible, byte_field, short_field, int_field, nullable_float1, nullable_double1, nullable_bool1, nullable_string2, nullable_list2, nullable_set2, nullable_map2); +// ============================================================================ +// Reference Tracking Test Types - Cross-language shared reference tests +// ============================================================================ + +// Inner struct for reference tracking test (SCHEMA_CONSISTENT mode) +// Matches Java RefInnerSchemaConsistent with type ID 501 +struct RefInnerSchemaConsistent { + int32_t id; + std::string name; + bool operator==(const RefInnerSchemaConsistent &other) const { + return id == other.id && name == other.name; + } + bool operator!=(const RefInnerSchemaConsistent &other) const { + return !(*this == other); + } +}; +FORY_STRUCT(RefInnerSchemaConsistent, id, name); + +// Outer struct for reference tracking test (SCHEMA_CONSISTENT mode) +// Contains two fields that both point to the same inner object. +// Matches Java RefOuterSchemaConsistent with type ID 502 +// Uses std::shared_ptr for reference tracking to share the same object +struct RefOuterSchemaConsistent { + std::shared_ptr inner1; + std::shared_ptr inner2; + bool operator==(const RefOuterSchemaConsistent &other) const { + bool inner1_eq = (inner1 == nullptr && other.inner1 == nullptr) || + (inner1 != nullptr && other.inner1 != nullptr && + *inner1 == *other.inner1); + bool inner2_eq = (inner2 == nullptr && other.inner2 == nullptr) || + (inner2 != nullptr && other.inner2 != nullptr && + *inner2 == *other.inner2); + return inner1_eq && inner2_eq; + } +}; +FORY_STRUCT(RefOuterSchemaConsistent, inner1, inner2); +FORY_FIELD_TAGS(RefOuterSchemaConsistent, (inner1, 0, nullable, ref), + (inner2, 1, nullable, ref)); +// Verify field tags are correctly parsed +static_assert(fory::detail::has_field_tags_v, + "RefOuterSchemaConsistent should have field tags"); +static_assert(fory::detail::GetFieldTagEntry::id == + 0, + "inner1 should have id=0"); +static_assert( + fory::detail::GetFieldTagEntry::is_nullable == + true, + "inner1 should be nullable"); +static_assert( + fory::detail::GetFieldTagEntry::track_ref == + true, + "inner1 should have track_ref=true"); + +// Inner struct for reference tracking test (COMPATIBLE mode) +// Matches Java RefInnerCompatible with type ID 503 +struct RefInnerCompatible { + int32_t id; + std::string name; + bool operator==(const RefInnerCompatible &other) const { + return id == other.id && name == other.name; + } + bool operator!=(const RefInnerCompatible &other) const { + return !(*this == other); + } +}; +FORY_STRUCT(RefInnerCompatible, id, name); + +// Outer struct for reference tracking test (COMPATIBLE mode) +// Contains two fields that both point to the same inner object. +// Matches Java RefOuterCompatible with type ID 504 +// Uses std::shared_ptr for reference tracking to share the same object +struct RefOuterCompatible { + std::shared_ptr inner1; + std::shared_ptr inner2; + bool operator==(const RefOuterCompatible &other) const { + bool inner1_eq = (inner1 == nullptr && other.inner1 == nullptr) || + (inner1 != nullptr && other.inner1 != nullptr && + *inner1 == *other.inner1); + bool inner2_eq = (inner2 == nullptr && other.inner2 == nullptr) || + (inner2 != nullptr && other.inner2 != nullptr && + *inner2 == *other.inner2); + return inner1_eq && inner2_eq; + } +}; +FORY_STRUCT(RefOuterCompatible, inner1, inner2); +FORY_FIELD_TAGS(RefOuterCompatible, (inner1, 0, nullable, ref), + (inner2, 1, nullable, ref)); + namespace fory { namespace serialization { @@ -589,11 +677,12 @@ void AppendSerialized(Fory &fory, const T &value, std::vector &out) { } Fory BuildFory(bool compatible = true, bool xlang = true, - bool check_struct_version = false) { + bool check_struct_version = false, bool track_ref = false) { return Fory::builder() .compatible(compatible) .xlang(xlang) .check_struct_version(check_struct_version) + .track_ref(track_ref) .build(); } @@ -639,6 +728,8 @@ void RunTestNullableFieldSchemaConsistentNotNull(const std::string &data_file); void RunTestNullableFieldSchemaConsistentNull(const std::string &data_file); void RunTestNullableFieldCompatibleNotNull(const std::string &data_file); void RunTestNullableFieldCompatibleNull(const std::string &data_file); +void RunTestRefSchemaConsistent(const std::string &data_file); +void RunTestRefCompatible(const std::string &data_file); } // namespace int main(int argc, char **argv) { @@ -730,6 +821,10 @@ int main(int argc, char **argv) { RunTestNullableFieldCompatibleNotNull(data_file); } else if (case_name == "test_nullable_field_compatible_null") { RunTestNullableFieldCompatibleNull(data_file); + } else if (case_name == "test_ref_schema_consistent") { + RunTestRefSchemaConsistent(data_file); + } else if (case_name == "test_ref_compatible") { + RunTestRefCompatible(data_file); } else { Fail("Unknown test case: " + case_name); } @@ -2152,4 +2247,95 @@ void RunTestNullableFieldCompatibleNull(const std::string &data_file) { WriteFile(data_file, out); } +// ============================================================================ +// Reference Tracking Tests - Cross-language shared reference tests +// ============================================================================ + +void RunTestRefSchemaConsistent(const std::string &data_file) { + auto bytes = ReadFile(data_file); + // SCHEMA_CONSISTENT mode: compatible=false, xlang=true, + // check_struct_version=true, track_ref=true + auto fory = BuildFory(false, true, true, true); + EnsureOk(fory.register_struct(501), + "register RefInnerSchemaConsistent"); + EnsureOk(fory.register_struct(502), + "register RefOuterSchemaConsistent"); + + Buffer buffer = MakeBuffer(bytes); + auto outer = ReadNext(fory, buffer); + + // Both inner1 and inner2 should have values + if (outer.inner1 == nullptr) { + Fail("RefOuterSchemaConsistent: inner1 should not be null"); + } + if (outer.inner2 == nullptr) { + Fail("RefOuterSchemaConsistent: inner2 should not be null"); + } + + // Both should have the same values (they reference the same object in Java) + if (outer.inner1->id != 42) { + Fail("RefOuterSchemaConsistent: inner1.id should be 42, got " + + std::to_string(outer.inner1->id)); + } + if (outer.inner1->name != "shared_inner") { + Fail( + "RefOuterSchemaConsistent: inner1.name should be 'shared_inner', got " + + outer.inner1->name); + } + if (*outer.inner1 != *outer.inner2) { + Fail("RefOuterSchemaConsistent: inner1 and inner2 should be equal (same " + "reference)"); + } + + // In C++, shared_ptr may or may not point to the same object after + // deserialization, depending on reference tracking implementation. + // The key test is that both have equal values. + + // Re-serialize and write back + std::vector out; + AppendSerialized(fory, outer, out); + WriteFile(data_file, out); +} + +void RunTestRefCompatible(const std::string &data_file) { + auto bytes = ReadFile(data_file); + // COMPATIBLE mode: compatible=true, xlang=true, check_struct_version=false, + // track_ref=true + auto fory = BuildFory(true, true, false, true); + EnsureOk(fory.register_struct(503), + "register RefInnerCompatible"); + EnsureOk(fory.register_struct(504), + "register RefOuterCompatible"); + + Buffer buffer = MakeBuffer(bytes); + auto outer = ReadNext(fory, buffer); + + // Both inner1 and inner2 should have values + if (outer.inner1 == nullptr) { + Fail("RefOuterCompatible: inner1 should not be null"); + } + if (outer.inner2 == nullptr) { + Fail("RefOuterCompatible: inner2 should not be null"); + } + + // Both should have the same values (they reference the same object in Java) + if (outer.inner1->id != 99) { + Fail("RefOuterCompatible: inner1.id should be 99, got " + + std::to_string(outer.inner1->id)); + } + if (outer.inner1->name != "compatible_shared") { + Fail("RefOuterCompatible: inner1.name should be 'compatible_shared', got " + + outer.inner1->name); + } + if (*outer.inner1 != *outer.inner2) { + Fail("RefOuterCompatible: inner1 and inner2 should be equal (same " + "reference)"); + } + + // Re-serialize and write back + std::vector out; + AppendSerialized(fory, outer, out); + WriteFile(data_file, out); +} + } // namespace diff --git a/cpp/fory/type/type.h b/cpp/fory/type/type.h index 75c622f842..540a6ba366 100644 --- a/cpp/fory/type/type.h +++ b/cpp/fory/type/type.h @@ -175,6 +175,21 @@ inline bool IsTypeShareMeta(int32_t type_id) { } } +/// Check if type_id represents a struct type. +/// Struct types include STRUCT, COMPATIBLE_STRUCT, NAMED_STRUCT, and +/// NAMED_COMPATIBLE_STRUCT. +inline constexpr bool is_struct_type(TypeId type_id) { + return type_id == TypeId::STRUCT || type_id == TypeId::COMPATIBLE_STRUCT || + type_id == TypeId::NAMED_STRUCT || + type_id == TypeId::NAMED_COMPATIBLE_STRUCT; +} + +/// Check if type_id represents an ext type. +/// Ext types include EXT and NAMED_EXT. +inline constexpr bool is_ext_type(TypeId type_id) { + return type_id == TypeId::EXT || type_id == TypeId::NAMED_EXT; +} + /// Check if type_id represents an internal (built-in) type. /// Internal types are all types except user-defined types (ENUM, STRUCT, EXT). /// UNKNOWN is excluded because it's a marker for polymorphic types, not a diff --git a/go/fory/codegen/decoder.go b/go/fory/codegen/decoder.go index 64d3bd7765..9542f499ae 100644 --- a/go/fory/codegen/decoder.go +++ b/go/fory/codegen/decoder.go @@ -99,7 +99,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { case "time.Time", "github.com/apache/fory/go/fory.Date": // These types are "other internal types" in the new spec // They use: | null flag | value data | format - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } } @@ -107,7 +107,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { // Handle pointer types if _, ok := field.Type.(*types.Pointer); ok { // For pointer types, use ReadValue - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -156,13 +156,13 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := elemType.(*types.Interface); ok && iface.Empty() { // For []interface{}, we need to manually implement the deserialization // to match our custom encoding. - // Slices are nullable in Go, so read null flag first. + // In xlang mode, slices are NOT nullable by default. + // In native Go mode, slices can be nil and need null flags. fmt.Fprintf(buf, "\t// Dynamic slice []interface{} handling - manual deserialization\n") fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make([]interface{}, 0)\n", fieldAccess) @@ -173,7 +173,27 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t%s = make([]interface{}, sliceLen)\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t// ReadData each element using ReadValue\n") fmt.Fprintf(buf, "\t\t\t\tfor i := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem(), fory.RefModeTracking, true)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make([]interface{}, 0)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\t\t// ReadData collection flags (ignore for now)\n") + fmt.Fprintf(buf, "\t\t\t\t\t_ = buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\t\t\t// Create slice with proper capacity\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make([]interface{}, sliceLen)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t// ReadData each element using ReadValue\n") + fmt.Fprintf(buf, "\t\t\t\t\tfor i := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem(), fory.RefModeTracking, true)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t}\n") @@ -200,14 +220,14 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := field.Type.(*types.Interface); ok { if iface.Empty() { // For interface{}, use ReadValue for dynamic type handling - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } } // Handle struct types if _, ok := field.Type.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -323,14 +343,14 @@ func generateSliceElementRead(buf *bytes.Buffer, elemType types.Type, elemAccess } // Check if it's a struct if _, ok := named.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } } // Handle struct types if _, ok := elemType.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } @@ -351,68 +371,90 @@ func generateSliceReadInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAcc // Check if element type is referencable (needs ref tracking) elemIsReferencable := isReferencableType(elemType) - // Slices are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // ReadData slice with null flag - use block scope to avoid variable name conflicts + // ReadData slice with conditional null flag - use block scope to avoid variable name conflicts fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, 0)\n", fieldAccess, sliceType.String()) fmt.Fprintf(buf, "\t\t\t} else {\n") + // ReadData collection header in xlang mode + if err := writeSliceReadElements(buf, sliceType, elemType, fieldAccess, elemIsReferencable, "\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t}\n") // end else (sliceLen > 0) + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make(%s, 0)\n", fieldAccess, sliceType.String()) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + // ReadData collection header in native mode + if err := writeSliceReadElements(buf, sliceType, elemType, fieldAccess, elemIsReferencable, "\t\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t\t}\n") // end else (sliceLen > 0) + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writeSliceReadElements generates the element reading code for a slice with specified indentation +func writeSliceReadElements(buf *bytes.Buffer, sliceType *types.Slice, elemType types.Type, fieldAccess string, elemIsReferencable bool, indent string) error { // ReadData collection header - fmt.Fprintf(buf, "\t\t\t\tcollectFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\t\t\t// Check if CollectionIsDeclElementType is set (bit 2, value 4)\n") - fmt.Fprintf(buf, "\t\t\t\thasDeclType := (collectFlag & 4) != 0\n") + fmt.Fprintf(buf, "%scollectFlag := buf.ReadInt8(err)\n", indent) + fmt.Fprintf(buf, "%s// Check if CollectionIsDeclElementType is set (bit 2, value 4)\n", indent) + fmt.Fprintf(buf, "%shasDeclType := (collectFlag & 4) != 0\n", indent) if elemIsReferencable { - fmt.Fprintf(buf, "\t\t\t\t// Check if CollectionTrackingRef is set (bit 0, value 1)\n") - fmt.Fprintf(buf, "\t\t\t\ttrackRefs := (collectFlag & 1) != 0\n") + fmt.Fprintf(buf, "%s// Check if CollectionTrackingRef is set (bit 0, value 1)\n", indent) + fmt.Fprintf(buf, "%strackRefs := (collectFlag & 1) != 0\n", indent) } // Create slice - fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, sliceLen)\n", fieldAccess, sliceType.String()) + fmt.Fprintf(buf, "%s%s = make(%s, sliceLen)\n", indent, fieldAccess, sliceType.String()) // ReadData elements based on whether CollectionIsDeclElementType is set - fmt.Fprintf(buf, "\t\t\t\tif hasDeclType {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// Elements are written directly without type IDs\n") - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < sliceLen; i++ {\n") + fmt.Fprintf(buf, "%sif hasDeclType {\n", indent) + fmt.Fprintf(buf, "%s\t// Elements are written directly without type IDs\n", indent) + fmt.Fprintf(buf, "%s\tfor i := 0; i < sliceLen; i++ {\n", indent) if elemIsReferencable { - // For referencable elements (like strings), need to read ref flag when tracking - fmt.Fprintf(buf, "\t\t\t\t\t\tif trackRefs {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tif trackRefs {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) } - if err := generateSliceElementReadDirect(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess)); err != nil { + if err := generateSliceElementReadDirectIndented(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess), indent+"\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// Need to read type ID once if CollectionIsSameType is set\n") - fmt.Fprintf(buf, "\t\t\t\t\tif (collectFlag & 8) != 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t// ReadData element type ID once for all elements\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t_ = buf.ReadVaruint32(err)\n") - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < sliceLen; i++ {\n") + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s} else {\n", indent) + fmt.Fprintf(buf, "%s\t// Need to read type ID once if CollectionIsSameType is set\n", indent) + fmt.Fprintf(buf, "%s\tif (collectFlag & 8) != 0 {\n", indent) + fmt.Fprintf(buf, "%s\t\t// ReadData element type ID once for all elements\n", indent) + fmt.Fprintf(buf, "%s\t\t_ = buf.ReadVaruint32(err)\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s\tfor i := 0; i < sliceLen; i++ {\n", indent) if elemIsReferencable { - // For referencable elements (like strings), need to read ref flag when tracking - fmt.Fprintf(buf, "\t\t\t\t\t\tif trackRefs {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tif trackRefs {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) } - // For same type without declared type, read elements directly - if err := generateSliceElementReadDirect(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess)); err != nil { + if err := generateSliceElementReadDirectIndented(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess), indent+"\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t}\n") // end else (sliceLen > 0) - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) return nil } @@ -422,47 +464,67 @@ func generatePrimitiveSliceReadInline(buf *bytes.Buffer, sliceType *types.Slice, elemType := sliceType.Elem() basic := elemType.Underlying().(*types.Basic) - // Slices are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // Read null flag first fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") + + // Read primitive slice in xlang mode + if err := writePrimitiveSliceReadCall(buf, basic, fieldAccess, "\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") - // Call the exported helper function for each primitive type + // Read primitive slice in native mode + if err := writePrimitiveSliceReadCall(buf, basic, fieldAccess, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writePrimitiveSliceReadCall writes the helper function call for reading a primitive slice +func writePrimitiveSliceReadCall(buf *bytes.Buffer, basic *types.Basic, fieldAccess string, indent string) error { switch basic.Kind() { case types.Bool: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadBoolSlice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadBoolSlice(buf, err)\n", indent, fieldAccess) case types.Int8: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt8Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt8Slice(buf, err)\n", indent, fieldAccess) case types.Uint8: - fmt.Fprintf(buf, "\t\t\tsizeBytes := buf.ReadLength(err)\n") - fmt.Fprintf(buf, "\t\t\t%s = make([]uint8, sizeBytes)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tif sizeBytes > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\traw := buf.ReadBinary(sizeBytes, err)\n") - fmt.Fprintf(buf, "\t\t\t\tif raw != nil {\n") - fmt.Fprintf(buf, "\t\t\t\t\tcopy(%s, raw)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "%ssizeBytes := buf.ReadLength(err)\n", indent) + fmt.Fprintf(buf, "%s%s = make([]uint8, sizeBytes)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%sif sizeBytes > 0 {\n", indent) + fmt.Fprintf(buf, "%s\traw := buf.ReadBinary(sizeBytes, err)\n", indent) + fmt.Fprintf(buf, "%s\tif raw != nil {\n", indent) + fmt.Fprintf(buf, "%s\t\tcopy(%s, raw)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) case types.Int16: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt16Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt16Slice(buf, err)\n", indent, fieldAccess) case types.Int32: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt32Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt32Slice(buf, err)\n", indent, fieldAccess) case types.Int64: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt64Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt64Slice(buf, err)\n", indent, fieldAccess) case types.Float32: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadFloat32Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadFloat32Slice(buf, err)\n", indent, fieldAccess) case types.Float64: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadFloat64Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadFloat64Slice(buf, err)\n", indent, fieldAccess) default: return fmt.Errorf("unsupported primitive type for ARRAY protocol read: %s", basic.String()) } - - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope return nil } @@ -549,7 +611,7 @@ func generateSliceElementReadInline(buf *bytes.Buffer, elemType types.Type, elem if iface, ok := elemType.(*types.Interface); ok { if iface.Empty() { // For interface{} elements, use ReadValue for dynamic type handling - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } } @@ -599,6 +661,43 @@ func generateSliceElementReadDirect(buf *bytes.Buffer, elemType types.Type, elem return fmt.Errorf("unsupported element type for direct read: %s", elemType.String()) } +// generateSliceElementReadDirectIndented generates code to read slice elements directly with custom indentation +func generateSliceElementReadDirectIndented(buf *bytes.Buffer, elemType types.Type, elemAccess string, indent string) error { + if basic, ok := elemType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Bool: + fmt.Fprintf(buf, "%s%s = buf.ReadBool(err)\n", indent, elemAccess) + case types.Int8: + fmt.Fprintf(buf, "%s%s = buf.ReadInt8(err)\n", indent, elemAccess) + case types.Int16: + fmt.Fprintf(buf, "%s%s = buf.ReadInt16(err)\n", indent, elemAccess) + case types.Int32: + fmt.Fprintf(buf, "%s%s = buf.ReadVarint32(err)\n", indent, elemAccess) + case types.Int, types.Int64: + fmt.Fprintf(buf, "%s%s = buf.ReadVarint64(err)\n", indent, elemAccess) + case types.Uint8: + fmt.Fprintf(buf, "%s%s = buf.ReadByte(err)\n", indent, elemAccess) + case types.Uint16: + fmt.Fprintf(buf, "%s%s = uint16(buf.ReadInt16(err))\n", indent, elemAccess) + case types.Uint32: + fmt.Fprintf(buf, "%s%s = uint32(buf.ReadInt32(err))\n", indent, elemAccess) + case types.Uint, types.Uint64: + fmt.Fprintf(buf, "%s%s = uint64(buf.ReadInt64(err))\n", indent, elemAccess) + case types.Float32: + fmt.Fprintf(buf, "%s%s = buf.ReadFloat32(err)\n", indent, elemAccess) + case types.Float64: + fmt.Fprintf(buf, "%s%s = buf.ReadFloat64(err)\n", indent, elemAccess) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, elemAccess) + default: + return fmt.Errorf("unsupported basic type for direct element read: %s", basic.String()) + } + return nil + } + + return fmt.Errorf("unsupported element type for direct read: %s", elemType.String()) +} + // generateMapReadInline generates inline map deserialization code following the chunk-based format // Uses error-aware methods for deferred error checking func generateMapReadInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess string) error { @@ -615,77 +714,100 @@ func generateMapReadInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess st valueIsInterface = true } - // Maps are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where maps have nullable=true. + // In xlang mode, maps are NOT nullable by default (only pointer types are nullable). + // In native Go mode, maps can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // ReadData map with null flag + // ReadData map with conditional null flag fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: maps are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tmapLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif mapLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make(%s)\n", fieldAccess, mapType.String()) fmt.Fprintf(buf, "\t\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, mapLen)\n", fieldAccess, mapType.String()) - fmt.Fprintf(buf, "\t\t\t\tmapSize := mapLen\n") + // Read map chunks in xlang mode + if err := writeMapReadChunks(buf, mapType, fieldAccess, keyType, valueType, keyIsInterface, valueIsInterface, "\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t}\n") // end else (mapLen > 0) + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: maps are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tmapLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif mapLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make(%s)\n", fieldAccess, mapType.String()) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + // Read map chunks in native mode + if err := writeMapReadChunks(buf, mapType, fieldAccess, keyType, valueType, keyIsInterface, valueIsInterface, "\t\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t\t}\n") // end else (mapLen > 0) + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + + return nil +} + +// writeMapReadChunks generates the map chunk reading code with specified indentation +func writeMapReadChunks(buf *bytes.Buffer, mapType *types.Map, fieldAccess string, keyType, valueType types.Type, keyIsInterface, valueIsInterface bool, indent string) error { + fmt.Fprintf(buf, "%s%s = make(%s, mapLen)\n", indent, fieldAccess, mapType.String()) + fmt.Fprintf(buf, "%smapSize := mapLen\n", indent) // ReadData chunks - fmt.Fprintf(buf, "\t\t\t\tfor mapSize > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// ReadData KV header\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader := buf.ReadByte(err)\n") - fmt.Fprintf(buf, "\t\t\t\t\tchunkSize := int(buf.ReadByte(err))\n") + fmt.Fprintf(buf, "%sfor mapSize > 0 {\n", indent) + fmt.Fprintf(buf, "%s\t// ReadData KV header\n", indent) + fmt.Fprintf(buf, "%s\tkvHeader := buf.ReadByte(err)\n", indent) + fmt.Fprintf(buf, "%s\tchunkSize := int(buf.ReadByte(err))\n", indent) // Parse header flags - fmt.Fprintf(buf, "\t\t\t\t\ttrackKeyRef := (kvHeader & 0x1) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\tkeyNotDeclared := (kvHeader & 0x4) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\ttrackValueRef := (kvHeader & 0x8) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\tvalueNotDeclared := (kvHeader & 0x20) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = trackKeyRef\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = keyNotDeclared\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = trackValueRef\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = valueNotDeclared\n") + fmt.Fprintf(buf, "%s\ttrackKeyRef := (kvHeader & 0x1) != 0\n", indent) + fmt.Fprintf(buf, "%s\tkeyNotDeclared := (kvHeader & 0x4) != 0\n", indent) + fmt.Fprintf(buf, "%s\ttrackValueRef := (kvHeader & 0x8) != 0\n", indent) + fmt.Fprintf(buf, "%s\tvalueNotDeclared := (kvHeader & 0x20) != 0\n", indent) + fmt.Fprintf(buf, "%s\t_ = trackKeyRef\n", indent) + fmt.Fprintf(buf, "%s\t_ = keyNotDeclared\n", indent) + fmt.Fprintf(buf, "%s\t_ = trackValueRef\n", indent) + fmt.Fprintf(buf, "%s\t_ = valueNotDeclared\n", indent) // ReadData key-value pairs in this chunk - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < chunkSize; i++ {\n") + fmt.Fprintf(buf, "%s\tfor i := 0; i < chunkSize; i++ {\n", indent) // ReadData key if keyIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapKey interface{}\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem())\n") + fmt.Fprintf(buf, "%s\t\tvar mapKey interface{}\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem(), fory.RefModeTracking, true)\n", indent) } else { - // Declare key variable with appropriate type keyVarType := getGoTypeString(keyType) - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapKey %s\n", keyVarType) - if err := generateMapKeyRead(buf, keyType, "mapKey"); err != nil { + fmt.Fprintf(buf, "%s\t\tvar mapKey %s\n", indent, keyVarType) + if err := generateMapKeyReadIndented(buf, keyType, "mapKey", indent+"\t\t"); err != nil { return err } } // ReadData value if valueIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapValue interface{}\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem())\n") + fmt.Fprintf(buf, "%s\t\tvar mapValue interface{}\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem(), fory.RefModeTracking, true)\n", indent) } else { - // Declare value variable with appropriate type valueVarType := getGoTypeString(valueType) - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapValue %s\n", valueVarType) - if err := generateMapValueRead(buf, valueType, "mapValue"); err != nil { + fmt.Fprintf(buf, "%s\t\tvar mapValue %s\n", indent, valueVarType) + if err := generateMapValueReadIndented(buf, valueType, "mapValue", indent+"\t\t"); err != nil { return err } } // Set key-value pair in map - fmt.Fprintf(buf, "\t\t\t\t\t\t%s[mapKey] = mapValue\n", fieldAccess) - - fmt.Fprintf(buf, "\t\t\t\t\t}\n") // end chunk loop - fmt.Fprintf(buf, "\t\t\t\t\tmapSize -= chunkSize\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") // end mapSize > 0 loop + fmt.Fprintf(buf, "%s\t\t%s[mapKey] = mapValue\n", indent, fieldAccess) - fmt.Fprintf(buf, "\t\t\t}\n") // end else (mapLen > 0) - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s\t}\n", indent) // end chunk loop + fmt.Fprintf(buf, "%s\tmapSize -= chunkSize\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) // end mapSize > 0 loop return nil } @@ -725,7 +847,7 @@ func generateMapKeyRead(buf *bytes.Buffer, keyType types.Type, varName string) e } // For other types, use ReadValue - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", varName) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", varName) return nil } @@ -748,6 +870,40 @@ func generateMapValueRead(buf *bytes.Buffer, valueType types.Type, varName strin } // For other types, use ReadValue - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", varName) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", varName) + return nil +} + +// generateMapKeyReadIndented generates code to read a map key with custom indentation +func generateMapKeyReadIndented(buf *bytes.Buffer, keyType types.Type, varName string, indent string) error { + if basic, ok := keyType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + fmt.Fprintf(buf, "%s%s = int(buf.ReadInt64(err))\n", indent, varName) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, varName) + default: + return fmt.Errorf("unsupported map key type: %v", keyType) + } + return nil + } + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", indent, varName) + return nil +} + +// generateMapValueReadIndented generates code to read a map value with custom indentation +func generateMapValueReadIndented(buf *bytes.Buffer, valueType types.Type, varName string, indent string) error { + if basic, ok := valueType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + fmt.Fprintf(buf, "%s%s = int(buf.ReadInt64(err))\n", indent, varName) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, varName) + default: + return fmt.Errorf("unsupported map value type: %v", valueType) + } + return nil + } + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", indent, varName) return nil } diff --git a/go/fory/codegen/encoder.go b/go/fory/codegen/encoder.go index 28c866c7be..b4a0d141a3 100644 --- a/go/fory/codegen/encoder.go +++ b/go/fory/codegen/encoder.go @@ -85,7 +85,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { case "time.Time", "github.com/apache/fory/go/fory.Date": // These types are "other internal types" in the new spec // They use: | null flag | value data | format - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } } @@ -93,7 +93,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { // Handle pointer types if _, ok := field.Type.(*types.Pointer); ok { // For all pointer types, use WriteValue - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -143,14 +143,17 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := elemType.(*types.Interface); ok && iface.Empty() { // For []interface{}, we need to manually implement the serialization // because WriteValue produces incorrect length encoding. - // Slices are nullable in Go, so write null flag first. + // In xlang mode, slices are NOT nullable by default. + // In native Go mode, slices can be nil and need null flags. fmt.Fprintf(buf, "\t// Dynamic slice []interface{} handling - manual serialization\n") fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tsliceLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tsliceLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen > 0 {\n") fmt.Fprintf(buf, "\t\t\t\t// WriteData collection flags for dynamic slice []interface{}\n") @@ -158,7 +161,25 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(1) // CollectionTrackingRef only\n") fmt.Fprintf(buf, "\t\t\t\t// WriteData each element using WriteValue\n") fmt.Fprintf(buf, "\t\t\t\tfor _, elem := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem))\n") + fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true)\n") + fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen > 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t// WriteData collection flags for dynamic slice []interface{}\n") + fmt.Fprintf(buf, "\t\t\t\t\t// Only CollectionTrackingRef is set (no declared type, may have different types)\n") + fmt.Fprintf(buf, "\t\t\t\t\tbuf.WriteInt8(1) // CollectionTrackingRef only\n") + fmt.Fprintf(buf, "\t\t\t\t\t// WriteData each element using WriteValue\n") + fmt.Fprintf(buf, "\t\t\t\t\tfor _, elem := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true)\n") + fmt.Fprintf(buf, "\t\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t}\n") @@ -185,14 +206,14 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := field.Type.(*types.Interface); ok { if iface.Empty() { // For interface{}, use WriteValue for dynamic type handling - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } } // Handle struct types if _, ok := field.Type.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -274,28 +295,23 @@ func generateSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAc // Check if element type is referencable (needs ref tracking) elemIsReferencable := isReferencableType(elemType) - // Slices are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. - // RefMode is either RefModeTracking (if trackRef && nullable) or RefModeNullOnly (if nullable only). - // Since codegen always writes null flag for slices, we match the RefModeNullOnly behavior. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // WriteData slice with null flag - use block scope to avoid variable name conflicts + // WriteData slice with conditional null flag - use block scope to avoid variable name conflicts fmt.Fprintf(buf, "\t{\n") - // Write null flag for slices (nullable=true) - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tsliceLen := len(%s)\n", fieldAccess) + // Check if xlang mode - in xlang mode, slices are not nullable by default + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tsliceLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tsliceLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") - - // WriteData collection header and elements for non-empty slice + // Write elements in xlang mode fmt.Fprintf(buf, "\t\t\tif sliceLen > 0 {\n") - - // For codegen, follow reflection's behavior for struct fields: - // Set both CollectionIsSameType and CollectionIsDeclElementType - // Add CollectionTrackingRef when ref tracking is enabled AND element is referencable - // This matches sliceConcreteValueSerializer.WriteData which adds CollectionTrackingRef for referencable elements fmt.Fprintf(buf, "\t\t\t\tcollectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType\n") if elemIsReferencable { fmt.Fprintf(buf, "\t\t\t\tif ctx.TrackRef() {\n") @@ -303,26 +319,57 @@ func generateSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAc fmt.Fprintf(buf, "\t\t\t\t}\n") } fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(int8(collectFlag))\n") - - // Element type ID is NOT written when CollectionIsDeclElementType is set - // The reader knows the element type from the field type - - // WriteData elements - with ref flags if element is referencable and tracking is enabled fmt.Fprintf(buf, "\t\t\t\tfor _, elem := range %s {\n", fieldAccess) if elemIsReferencable { - // For referencable elements (like strings), need to write ref flag when tracking fmt.Fprintf(buf, "\t\t\t\t\tif ctx.TrackRef() {\n") fmt.Fprintf(buf, "\t\t\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag for element\n") fmt.Fprintf(buf, "\t\t\t\t\t}\n") } - if err := generateSliceElementWriteInline(buf, elemType, "elem"); err != nil { + if err := generateSliceElementWriteInlineIndented(buf, elemType, "elem", "\t\t\t\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t}\n") // end for loop fmt.Fprintf(buf, "\t\t\t}\n") // end if sliceLen > 0 - fmt.Fprintf(buf, "\t\t}\n") // end else (not nil) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + // Write null flag for slices in native mode + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") + + // WriteData collection header and elements for non-empty slice in native mode + fmt.Fprintf(buf, "\t\t\t\tif sliceLen > 0 {\n") + + // For codegen, follow reflection's behavior for struct fields: + // Set both CollectionIsSameType and CollectionIsDeclElementType + // Add CollectionTrackingRef when ref tracking is enabled AND element is referencable + fmt.Fprintf(buf, "\t\t\t\t\tcollectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType\n") + if elemIsReferencable { + fmt.Fprintf(buf, "\t\t\t\t\tif ctx.TrackRef() {\n") + fmt.Fprintf(buf, "\t\t\t\t\t\tcollectFlag |= 1 // CollectionTrackingRef for referencable element type\n") + fmt.Fprintf(buf, "\t\t\t\t\t}\n") + } + fmt.Fprintf(buf, "\t\t\t\t\tbuf.WriteInt8(int8(collectFlag))\n") + + // WriteData elements - with ref flags if element is referencable and tracking is enabled + fmt.Fprintf(buf, "\t\t\t\t\tfor _, elem := range %s {\n", fieldAccess) + if elemIsReferencable { + fmt.Fprintf(buf, "\t\t\t\t\t\tif ctx.TrackRef() {\n") + fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag for element\n") + fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + } + if err := generateSliceElementWriteInlineIndented(buf, elemType, "elem", "\t\t\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t\t\t}\n") // end for loop + fmt.Fprintf(buf, "\t\t\t\t}\n") // end if sliceLen > 0 + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) in native mode + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope return nil } @@ -344,41 +391,63 @@ func generatePrimitiveSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice elemType := sliceType.Elem() basic := elemType.Underlying().(*types.Basic) - // Slices are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. - // Write null flag first, then call the helper function for the actual data. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - fmt.Fprintf(buf, "\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t} else {\n") - fmt.Fprintf(buf, "\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t{\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") - // Call the exported helper function for each primitive type + // Write primitive slice directly in xlang mode + if err := writePrimitiveSliceCall(buf, basic, fieldAccess, "\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + + // Write primitive slice in native mode + if err := writePrimitiveSliceCall(buf, basic, fieldAccess, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writePrimitiveSliceCall writes the helper function call for a primitive slice type +func writePrimitiveSliceCall(buf *bytes.Buffer, basic *types.Basic, fieldAccess string, indent string) error { switch basic.Kind() { case types.Bool: - fmt.Fprintf(buf, "\t\tfory.WriteBoolSlice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteBoolSlice(buf, %s)\n", indent, fieldAccess) case types.Int8: - fmt.Fprintf(buf, "\t\tfory.WriteInt8Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt8Slice(buf, %s)\n", indent, fieldAccess) case types.Uint8: - fmt.Fprintf(buf, "\t\tbuf.WriteLength(len(%s))\n", fieldAccess) - fmt.Fprintf(buf, "\t\tif len(%s) > 0 {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteBinary(%s)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t}\n") + fmt.Fprintf(buf, "%sbuf.WriteLength(len(%s))\n", indent, fieldAccess) + fmt.Fprintf(buf, "%sif len(%s) > 0 {\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\tbuf.WriteBinary(%s)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s}\n", indent) case types.Int16: - fmt.Fprintf(buf, "\t\tfory.WriteInt16Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt16Slice(buf, %s)\n", indent, fieldAccess) case types.Int32: - fmt.Fprintf(buf, "\t\tfory.WriteInt32Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt32Slice(buf, %s)\n", indent, fieldAccess) case types.Int64: - fmt.Fprintf(buf, "\t\tfory.WriteInt64Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt64Slice(buf, %s)\n", indent, fieldAccess) case types.Float32: - fmt.Fprintf(buf, "\t\tfory.WriteFloat32Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteFloat32Slice(buf, %s)\n", indent, fieldAccess) case types.Float64: - fmt.Fprintf(buf, "\t\tfory.WriteFloat64Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteFloat64Slice(buf, %s)\n", indent, fieldAccess) default: return fmt.Errorf("unsupported primitive type for ARRAY protocol: %s", basic.String()) } - - fmt.Fprintf(buf, "\t}\n") return nil } @@ -397,104 +466,133 @@ func generateMapWriteInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess s valueIsInterface = true } - // Maps are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where maps have nullable=true. + // In xlang mode, maps are NOT nullable by default (only pointer types are nullable). + // In native Go mode, maps can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // WriteData map with null flag + // WriteData map with conditional null flag fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tmapLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: maps are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tmapLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tmapLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(mapLen))\n") + // Write map chunks in xlang mode + if err := writeMapChunksCode(buf, keyType, valueType, fieldAccess, keyIsInterface, valueIsInterface, "\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: maps are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tmapLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(mapLen))\n") + + // Write map chunks in native mode + if err := writeMapChunksCode(buf, keyType, valueType, fieldAccess, keyIsInterface, valueIsInterface, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + + return nil +} + +// writeMapChunksCode generates the map chunk writing code with specified indentation +func writeMapChunksCode(buf *bytes.Buffer, keyType, valueType types.Type, fieldAccess string, keyIsInterface, valueIsInterface bool, indent string) error { // WriteData chunks for non-empty map - fmt.Fprintf(buf, "\t\t\tif mapLen > 0 {\n") + fmt.Fprintf(buf, "%sif mapLen > 0 {\n", indent) // Calculate KV header based on types - fmt.Fprintf(buf, "\t\t\t\t// Calculate KV header flags\n") - fmt.Fprintf(buf, "\t\t\t\tkvHeader := uint8(0)\n") + fmt.Fprintf(buf, "%s\t// Calculate KV header flags\n", indent) + fmt.Fprintf(buf, "%s\tkvHeader := uint8(0)\n", indent) // Check if ref tracking is enabled - fmt.Fprintf(buf, "\t\t\t\tisRefTracking := ctx.TrackRef()\n") - fmt.Fprintf(buf, "\t\t\t\t_ = isRefTracking // Mark as used to avoid warning\n") + fmt.Fprintf(buf, "%s\tisRefTracking := ctx.TrackRef()\n", indent) + fmt.Fprintf(buf, "%s\t_ = isRefTracking // Mark as used to avoid warning\n", indent) // Set header flags based on type properties if !keyIsInterface { // For concrete key types, check if they're referencable if isReferencableType(keyType) { - fmt.Fprintf(buf, "\t\t\t\tif isRefTracking {\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader |= 0x1 // track key ref\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif isRefTracking {\n", indent) + fmt.Fprintf(buf, "%s\t\tkvHeader |= 0x1 // track key ref\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) } } else { // For interface{} keys, always set not declared type flag - fmt.Fprintf(buf, "\t\t\t\tkvHeader |= 0x4 // key type not declared\n") + fmt.Fprintf(buf, "%s\tkvHeader |= 0x4 // key type not declared\n", indent) } if !valueIsInterface { // For concrete value types, check if they're referencable if isReferencableType(valueType) { - fmt.Fprintf(buf, "\t\t\t\tif isRefTracking {\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader |= 0x8 // track value ref\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif isRefTracking {\n", indent) + fmt.Fprintf(buf, "%s\t\tkvHeader |= 0x8 // track value ref\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) } } else { // For interface{} values, always set not declared type flag - fmt.Fprintf(buf, "\t\t\t\tkvHeader |= 0x20 // value type not declared\n") + fmt.Fprintf(buf, "%s\tkvHeader |= 0x20 // value type not declared\n", indent) } // WriteData map elements in chunks - fmt.Fprintf(buf, "\t\t\t\tchunkSize := 0\n") - fmt.Fprintf(buf, "\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n") - fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n") - fmt.Fprintf(buf, "\t\t\t\tchunkSizeOffset := buf.WriterIndex()\n") - fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n") + fmt.Fprintf(buf, "%s\tchunkSize := 0\n", indent) + fmt.Fprintf(buf, "%s\t_ = buf.WriterIndex() // chunkHeaderOffset\n", indent) + fmt.Fprintf(buf, "%s\tbuf.WriteInt8(int8(kvHeader)) // KV header\n", indent) + fmt.Fprintf(buf, "%s\tchunkSizeOffset := buf.WriterIndex()\n", indent) + fmt.Fprintf(buf, "%s\tbuf.WriteInt8(0) // placeholder for chunk size\n", indent) - fmt.Fprintf(buf, "\t\t\t\tfor mapKey, mapValue := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "%s\tfor mapKey, mapValue := range %s {\n", indent, fieldAccess) // WriteData key if keyIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(mapKey))\n") + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapKey), fory.RefModeTracking, true)\n", indent) } else { - if err := generateMapKeyWrite(buf, keyType, "mapKey"); err != nil { + if err := generateMapKeyWriteIndented(buf, keyType, "mapKey", indent+"\t\t"); err != nil { return err } } // WriteData value if valueIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(mapValue))\n") + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapValue), fory.RefModeTracking, true)\n", indent) } else { - if err := generateMapValueWrite(buf, valueType, "mapValue"); err != nil { + if err := generateMapValueWriteIndented(buf, valueType, "mapValue", indent+"\t\t"); err != nil { return err } } - fmt.Fprintf(buf, "\t\t\t\t\tchunkSize++\n") - fmt.Fprintf(buf, "\t\t\t\t\tif chunkSize >= 255 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t// WriteData chunk size and start new chunk\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tif len(%s) > chunkSize {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\t\t\tchunkSize = 0\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tchunkSizeOffset = buf.WriterIndex()\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tchunkSize++\n", indent) + fmt.Fprintf(buf, "%s\t\tif chunkSize >= 255 {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t// WriteData chunk size and start new chunk\n", indent) + fmt.Fprintf(buf, "%s\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n", indent) + fmt.Fprintf(buf, "%s\t\t\tif len(%s) > chunkSize {\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\t\t\t\tchunkSize = 0\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tchunkSizeOffset = buf.WriterIndex()\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n", indent) + fmt.Fprintf(buf, "%s\t\t\t}\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) - fmt.Fprintf(buf, "\t\t\t\t}\n") // end for loop + fmt.Fprintf(buf, "%s\t}\n", indent) // end for loop // WriteData final chunk size - fmt.Fprintf(buf, "\t\t\t\tif chunkSize > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif chunkSize > 0 {\n", indent) + fmt.Fprintf(buf, "%s\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) - fmt.Fprintf(buf, "\t\t\t}\n") // end if mapLen > 0 - fmt.Fprintf(buf, "\t\t}\n") // end else (not nil) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s}\n", indent) // end if mapLen > 0 return nil } @@ -543,7 +641,7 @@ func generateMapKeyWrite(buf *bytes.Buffer, keyType types.Type, varName string) } // For other types, use WriteValue - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", varName) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", varName) return nil } @@ -565,7 +663,51 @@ func generateMapValueWrite(buf *bytes.Buffer, valueType types.Type, varName stri } // For other types, use WriteValue - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", varName) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", varName) + return nil +} + +// generateMapKeyWriteIndented generates code to write a map key with custom indentation +func generateMapKeyWriteIndented(buf *bytes.Buffer, keyType types.Type, varName string, indent string) error { + // For basic types, match reflection's serializer behavior + if basic, ok := keyType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + // intSerializer uses WriteInt64, not WriteVarint64 + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, varName) + case types.String: + // stringSerializer.NeedWriteRef() = false, write directly + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, varName) + default: + return fmt.Errorf("unsupported map key type: %v", keyType) + } + return nil + } + + // For other types, use WriteValue + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, varName) + return nil +} + +// generateMapValueWriteIndented generates code to write a map value with custom indentation +func generateMapValueWriteIndented(buf *bytes.Buffer, valueType types.Type, varName string, indent string) error { + // For basic types, match reflection's serializer behavior + if basic, ok := valueType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + // intSerializer uses WriteInt64, not WriteVarint64 + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, varName) + case types.String: + // stringSerializer.NeedWriteRef() = false, write directly + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, varName) + default: + return fmt.Errorf("unsupported map value type: %v", valueType) + } + return nil + } + + // For other types, use WriteValue + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, varName) return nil } @@ -645,7 +787,54 @@ func generateSliceElementWriteInline(buf *bytes.Buffer, elemType types.Type, ele if iface, ok := elemType.(*types.Interface); ok { if iface.Empty() { // For interface{} elements, use WriteValue for dynamic type handling - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", elemAccess) + return nil + } + } + + return fmt.Errorf("unsupported element type for write: %s", elemType.String()) +} + +// generateSliceElementWriteInlineIndented generates code to write a single slice element value with custom indentation +func generateSliceElementWriteInlineIndented(buf *bytes.Buffer, elemType types.Type, elemAccess string, indent string) error { + // Handle basic types - write the actual value without type info (type already written above) + if basic, ok := elemType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Bool: + fmt.Fprintf(buf, "%sbuf.WriteBool(%s)\n", indent, elemAccess) + case types.Int8: + fmt.Fprintf(buf, "%sbuf.WriteInt8(%s)\n", indent, elemAccess) + case types.Int16: + fmt.Fprintf(buf, "%sbuf.WriteInt16(%s)\n", indent, elemAccess) + case types.Int32: + fmt.Fprintf(buf, "%sbuf.WriteVarint32(%s)\n", indent, elemAccess) + case types.Int, types.Int64: + fmt.Fprintf(buf, "%sbuf.WriteVarint64(%s)\n", indent, elemAccess) + case types.Uint8: + fmt.Fprintf(buf, "%sbuf.WriteByte_(%s)\n", indent, elemAccess) + case types.Uint16: + fmt.Fprintf(buf, "%sbuf.WriteInt16(int16(%s))\n", indent, elemAccess) + case types.Uint32: + fmt.Fprintf(buf, "%sbuf.WriteInt32(int32(%s))\n", indent, elemAccess) + case types.Uint, types.Uint64: + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, elemAccess) + case types.Float32: + fmt.Fprintf(buf, "%sbuf.WriteFloat32(%s)\n", indent, elemAccess) + case types.Float64: + fmt.Fprintf(buf, "%sbuf.WriteFloat64(%s)\n", indent, elemAccess) + case types.String: + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, elemAccess) + default: + return fmt.Errorf("unsupported basic type for element write: %s", basic.String()) + } + return nil + } + + // Handle interface types + if iface, ok := elemType.(*types.Interface); ok { + if iface.Empty() { + // For interface{} elements, use WriteValue for dynamic type handling + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, elemAccess) return nil } } diff --git a/go/fory/fory.go b/go/fory/fory.go index 0dbaa2b3a8..a6948c84df 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -169,11 +169,13 @@ func New(opts ...Option) *Fory { f.writeCtx.typeResolver = f.typeResolver f.writeCtx.refResolver = f.refResolver f.writeCtx.compatible = f.config.Compatible + f.writeCtx.xlang = f.config.IsXlang f.readCtx = NewReadContext(f.config.TrackRef) f.readCtx.typeResolver = f.typeResolver f.readCtx.refResolver = f.refResolver f.readCtx.compatible = f.config.Compatible + f.readCtx.xlang = f.config.IsXlang return f } @@ -418,7 +420,7 @@ func (f *Fory) Serialize(value any) ([]byte, error) { } // SerializeWithCallback the value - f.writeCtx.WriteValue(reflect.ValueOf(value)) + f.writeCtx.WriteValue(reflect.ValueOf(value), RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } @@ -477,7 +479,7 @@ func (f *Fory) Deserialize(data []byte, v interface{}) error { // Read directly into target value target := reflect.ValueOf(v).Elem() - f.readCtx.ReadValue(target) + f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } @@ -559,7 +561,7 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value interface{}) error { } // Standard path - f.writeCtx.WriteValue(rv) + f.writeCtx.WriteValue(rv, RefModeTracking, true) if f.writeCtx.HasError() { f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() @@ -633,7 +635,7 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v interface{}) error { // Read directly into target value target := reflect.ValueOf(v).Elem() - f.readCtx.ReadValue(target) + f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { f.readCtx.buffer = origBuffer return f.readCtx.TakeError() @@ -710,7 +712,7 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v interface{}, callback } // SerializeWithCallback the value - f.writeCtx.WriteValue(reflect.ValueOf(v)) + f.writeCtx.WriteValue(reflect.ValueOf(v), RefModeTracking, true) if f.writeCtx.HasError() { return f.writeCtx.TakeError() } @@ -798,7 +800,7 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v interface{}, return fmt.Errorf("v must be a non-nil pointer") } // DeserializeWithCallbackBuffers directly into v - f.readCtx.ReadValue(rv.Elem()) + f.readCtx.ReadValue(rv.Elem(), RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } @@ -826,7 +828,7 @@ func (f *Fory) serializeReflectValue(value reflect.Value) ([]byte, error) { } // SerializeWithCallback the value - f.writeCtx.WriteValue(value) + f.writeCtx.WriteValue(value, RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } diff --git a/go/fory/fory_test.go b/go/fory/fory_test.go index e60735916d..c74c6be6cf 100644 --- a/go/fory/fory_test.go +++ b/go/fory/fory_test.go @@ -222,7 +222,8 @@ func TestSerializeArray(t *testing.T) { func TestSerializeStructSimple(t *testing.T) { for _, referenceTracking := range []bool{false, true} { - fory := NewFory(WithRefTracking(referenceTracking)) + // Use WithXlang(false) for native Go mode where nil slices/maps are preserved + fory := NewFory(WithXlang(false), WithRefTracking(referenceTracking)) type A struct { F1 []string } diff --git a/go/fory/map.go b/go/fory/map.go index f910146e6e..6aabbe3729 100644 --- a/go/fory/map.go +++ b/go/fory/map.go @@ -22,6 +22,7 @@ import ( "reflect" ) +// Map chunk header flags const ( TRACKING_KEY_REF = 1 << 0 // 0b00000001 KEY_HAS_NULL = 1 << 1 // 0b00000010 @@ -32,654 +33,597 @@ const ( MAX_CHUNK_SIZE = 255 ) +// Combined header constants for null entry cases const ( - KV_NULL = KEY_HAS_NULL | VALUE_HAS_NULL // 0b00010010 - NULL_KEY_VALUE_DECL_TYPE = KEY_HAS_NULL | VALUE_DECL_TYPE // 0b00100010 - NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF = KEY_HAS_NULL | VALUE_DECL_TYPE | TRACKING_VALUE_REF // 0b00101010 - NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE // 0b00010100 - NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF // 0b00010101 + KV_NULL = KEY_HAS_NULL | VALUE_HAS_NULL + NULL_KEY_VALUE_DECL_TYPE = KEY_HAS_NULL | VALUE_DECL_TYPE + NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF = KEY_HAS_NULL | VALUE_DECL_TYPE | TRACKING_VALUE_REF + NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE + NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF ) -// writeMapRefAndType handles reference and type writing for map serializers. -// Returns true if the value was already written (nil or ref), false if data should be written. -func writeMapRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, value reflect.Value) bool { - switch refMode { - case RefModeTracking: - if value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return true - } - refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) - if err != nil { - ctx.SetError(FromError(err)) - return false - } - if refWritten { - return true - } - case RefModeNullOnly: - if value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return true - } - ctx.buffer.WriteInt8(NotNullValueFlag) - } - if writeType { - ctx.buffer.WriteVaruint32Small7(uint32(MAP)) - } - return false -} - -// readMapRefAndType handles reference and type reading for map serializers. -// Returns true if a reference was resolved (value already set), false if data should be read. -func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value reflect.Value) bool { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return false - } - if refID < int32(NotNullValueFlag) { - obj := ctx.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) - } - return true - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return true - } - } - if readType { - buf.ReadVaruint32Small7(ctxErr) - } - return false -} - type mapSerializer struct { type_ reflect.Type keySerializer Serializer valueSerializer Serializer keyReferencable bool valueReferencable bool - mapInStruct bool // Use mapInStruct to distinguish concrete map types during deserialization + hasGenerics bool // True when map is a struct field with declared key/value types +} +// Write handles ref tracking and type writing, then delegates to WriteData +func (s mapSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { + if writeMapRefAndType(ctx, refMode, writeType, value) || ctx.HasError() { + return + } + s.WriteData(ctx, value) } +// WriteData serializes map data using chunk protocol func (s mapSerializer) WriteData(ctx *WriteContext, value reflect.Value) { buf := ctx.Buffer() - if value.Kind() == reflect.Interface { - value = value.Elem() - } + value = unwrapInterface(value) length := value.Len() buf.WriteVaruint32(uint32(length)) if length == 0 { return } - typeResolver := ctx.TypeResolver() - // Use declared serializers if available (mapInStruct case) - // Don't clear them - we need them for KEY_DECL_TYPE/VALUE_DECL_TYPE flags - keySerializer := s.keySerializer - valueSerializer := s.valueSerializer + iter := value.MapRange() if !iter.Next() { return } - entryKey, entryVal := iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } + + typeResolver := ctx.TypeResolver() + trackRef := ctx.TrackRef() + entryKey := unwrapInterface(iter.Key()) + entryVal := unwrapInterface(iter.Value()) hasNext := true - // For xlang struct fields (mapInStruct = true), use declared types (set DECL_TYPE flags) - // For internal Go serialization (mapInStruct = false), always write type info (don't set DECL_TYPE flags) - isXlang := s.mapInStruct + for hasNext { + // Phase 1: Handle null entries (single-item chunks) for { - keyValid := isValid(entryKey) - valValid := isValid(entryVal) - if keyValid { - if valValid { - break - } - // Null value case - use DECL_TYPE only for xlang struct fields - if isXlang && keySerializer != nil { - if s.keyReferencable { - buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF) - keySerializer.Write(ctx, RefModeTracking, false, false, entryKey) - if ctx.HasError() { - return - } - } else { - buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE) - keySerializer.WriteData(ctx, entryKey) - if ctx.HasError() { - return - } - } - } else { - buf.WriteInt8(VALUE_HAS_NULL | TRACKING_KEY_REF) - ctx.WriteValue(entryKey) - if ctx.HasError() { - return - } - } + keyValid := entryKey.IsValid() + valValid := entryVal.IsValid() + + if keyValid && valValid { + break // Proceed to regular chunk + } + + if !keyValid && !valValid { + buf.WriteInt8(KV_NULL) + } else if !valValid { + s.writeNullValueEntry(ctx, entryKey, typeResolver, trackRef) } else { - if valValid { - // Null key case - use DECL_TYPE only for xlang struct fields - if isXlang && valueSerializer != nil { - if s.valueReferencable { - buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF) - valueSerializer.Write(ctx, RefModeTracking, false, false, entryVal) - if ctx.HasError() { - return - } - } else { - buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE) - valueSerializer.WriteData(ctx, entryVal) - if ctx.HasError() { - return - } - } - } else { - buf.WriteInt8(KEY_HAS_NULL | TRACKING_VALUE_REF) - ctx.WriteValue(entryVal) - if ctx.HasError() { - return - } - } - } else { - buf.WriteInt8(KV_NULL) - } + s.writeNullKeyEntry(ctx, entryVal, typeResolver, trackRef) + } + + if ctx.HasError() { + return } + if iter.Next() { - entryKey, entryVal = iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } + entryKey = unwrapInterface(iter.Key()) + entryVal = unwrapInterface(iter.Value()) } else { - hasNext = false - break + return } } - if !hasNext { - break + + // Phase 2: Write regular chunk with same-type entries + hasNext = s.writeChunk(ctx, iter, &entryKey, &entryVal, typeResolver, trackRef) + if ctx.HasError() { + return } - keyCls := getActualType(entryKey) - valueCls := getActualType(entryVal) - buf.WriteInt16(-1) - chunkSizeOffset := buf.writerIndex - chunkHeader := 0 - keyWriteRef := s.keyReferencable - valueWriteRef := s.valueReferencable - // For xlang struct fields, use declared types (set DECL_TYPE flags) - // For internal Go serialization, always write type info - if isXlang && keySerializer != nil { - chunkHeader |= KEY_DECL_TYPE + } +} + +// writeNullValueEntry writes a single entry where the value is null +func (s mapSerializer) writeNullValueEntry(ctx *WriteContext, key reflect.Value, resolver *TypeResolver, trackRef bool) { + buf := ctx.Buffer() + + if s.hasGenerics && s.keySerializer != nil { + if s.keyReferencable && trackRef { + buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF) + s.keySerializer.Write(ctx, RefModeTracking, false, false, key) } else { - keyTypeInfo, _ := getActualTypeInfo(entryKey, typeResolver) - typeResolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) - keySerializer = keyTypeInfo.Serializer - keyWriteRef = keyTypeInfo.NeedWriteRef + buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE) + s.keySerializer.WriteData(ctx, key) } - if isXlang && valueSerializer != nil { - chunkHeader |= VALUE_DECL_TYPE + return + } + + // Polymorphic key + keyTypeInfo, err := getTypeInfoForValue(key, resolver) + if err != nil { + ctx.SetError(FromError(err)) + return + } + + header := int8(VALUE_HAS_NULL) + writeKeyRef := trackRef && keyTypeInfo.NeedWriteRef + if writeKeyRef { + header |= TRACKING_KEY_REF + } + buf.WriteInt8(header) + resolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) + + refMode := RefModeNone + if writeKeyRef { + refMode = RefModeTracking + } + keyTypeInfo.Serializer.Write(ctx, refMode, false, false, key) +} + +// writeNullKeyEntry writes a single entry where the key is null +func (s mapSerializer) writeNullKeyEntry(ctx *WriteContext, value reflect.Value, resolver *TypeResolver, trackRef bool) { + buf := ctx.Buffer() + + if s.hasGenerics && s.valueSerializer != nil { + if s.valueReferencable && trackRef { + buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF) + s.valueSerializer.Write(ctx, RefModeTracking, false, false, value) } else { - valueTypeInfo, _ := getActualTypeInfo(entryVal, typeResolver) - typeResolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) - valueSerializer = valueTypeInfo.Serializer - valueWriteRef = valueTypeInfo.NeedWriteRef + buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE) + s.valueSerializer.WriteData(ctx, value) } + return + } - if keyWriteRef { - chunkHeader |= TRACKING_KEY_REF - } - if valueWriteRef { - chunkHeader |= TRACKING_VALUE_REF - } - buf.PutUint8(chunkSizeOffset-2, uint8(chunkHeader)) - chunkSize := 0 - keyRefMode := RefModeNone - if keyWriteRef { - keyRefMode = RefModeTracking - } - valueRefMode := RefModeNone - if valueWriteRef { - valueRefMode = RefModeTracking - } - for chunkSize < MAX_CHUNK_SIZE { - if !isValid(entryKey) || !isValid(entryVal) || getActualType(entryKey) != keyCls || getActualType(entryVal) != valueCls { - break - } + // Polymorphic value + valueTypeInfo, err := getTypeInfoForValue(value, resolver) + if err != nil { + ctx.SetError(FromError(err)) + return + } - // WriteData key with optional ref tracking - keySerializer.Write(ctx, keyRefMode, false, false, entryKey) - if ctx.HasError() { - return - } + header := int8(KEY_HAS_NULL) + writeValueRef := trackRef && valueTypeInfo.NeedWriteRef + if writeValueRef { + header |= TRACKING_VALUE_REF + } + buf.WriteInt8(header) + resolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) - // WriteData value with optional ref tracking - valueSerializer.Write(ctx, valueRefMode, false, false, entryVal) - if ctx.HasError() { - return - } + refMode := RefModeNone + if writeValueRef { + refMode = RefModeTracking + } + valueTypeInfo.Serializer.Write(ctx, refMode, false, false, value) +} - chunkSize++ +// writeChunk writes a chunk of entries with the same key/value types +func (s mapSerializer) writeChunk(ctx *WriteContext, iter *reflect.MapIter, entryKey, entryVal *reflect.Value, resolver *TypeResolver, trackRef bool) bool { + buf := ctx.Buffer() + keyType := (*entryKey).Type() + valueType := (*entryVal).Type() - if iter.Next() { - entryKey, entryVal = iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } - } else { - hasNext = false - break - } + // Reserve space: header (1 byte) + size (1 byte) + headerOffset := buf.writerIndex + buf.WriteInt16(-1) + + header := 0 + var keySer, valSer Serializer + keyWriteRef := s.keyReferencable + valueWriteRef := s.valueReferencable + + // Determine key serializer and write type info if needed + if s.hasGenerics && s.keySerializer != nil { + header |= KEY_DECL_TYPE + keySer = s.keySerializer + } else { + keyTypeInfo, _ := getTypeInfoForValue(*entryKey, resolver) + resolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) + keySer = keyTypeInfo.Serializer + keyWriteRef = keyTypeInfo.NeedWriteRef + } + + // Determine value serializer and write type info if needed + if s.hasGenerics && s.valueSerializer != nil { + header |= VALUE_DECL_TYPE + valSer = s.valueSerializer + } else { + valueTypeInfo, _ := getTypeInfoForValue(*entryVal, resolver) + resolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) + valSer = valueTypeInfo.Serializer + valueWriteRef = valueTypeInfo.NeedWriteRef + } + + // Set ref tracking flags + keyRefMode := RefModeNone + if keyWriteRef && trackRef { + header |= TRACKING_KEY_REF + keyRefMode = RefModeTracking + } + valueRefMode := RefModeNone + if valueWriteRef && trackRef { + header |= TRACKING_VALUE_REF + valueRefMode = RefModeTracking + } + + buf.PutUint8(headerOffset, uint8(header)) + + // Write entries with same type + chunkSize := 0 + for chunkSize < MAX_CHUNK_SIZE { + k := *entryKey + v := *entryVal + + // Break if null or type changed + if !k.IsValid() || !v.IsValid() || k.Type() != keyType || v.Type() != valueType { + break + } + + keySer.Write(ctx, keyRefMode, false, false, k) + if ctx.HasError() { + return false + } + valSer.Write(ctx, valueRefMode, false, false, v) + if ctx.HasError() { + return false + } + chunkSize++ + + if iter.Next() { + *entryKey = unwrapInterface(iter.Key()) + *entryVal = unwrapInterface(iter.Value()) + } else { + buf.PutUint8(headerOffset+1, uint8(chunkSize)) + return false } - keySerializer = s.keySerializer - valueSerializer = s.valueSerializer - buf.PutUint8(chunkSizeOffset-1, uint8(chunkSize)) } + + buf.PutUint8(headerOffset+1, uint8(chunkSize)) + return true } -func (s mapSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - done := writeMapRefAndType(ctx, refMode, writeType, value) - if done || ctx.HasError() { +// Read handles ref tracking and type reading, then delegates to ReadData +func (s mapSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { + if readMapRefAndType(ctx, refMode, readType, value) || ctx.HasError() { return } - s.WriteData(ctx, value) -} - -func (s mapSerializer) writeObj(ctx *WriteContext, serializer Serializer, obj reflect.Value) { - serializer.WriteData(ctx, obj) + s.ReadData(ctx, value.Type(), value) } +// ReadData deserializes map data using chunk protocol func (s mapSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { buf := ctx.Buffer() ctxErr := ctx.Err() refResolver := ctx.RefResolver() - if s.type_ == nil { - s.type_ = type_ - } + typeResolver := ctx.TypeResolver() + // Initialize map if value.IsNil() { - isIfaceMap := func(t reflect.Type) bool { - return t.Kind() == reflect.Map && - t.Key().Kind() == reflect.Interface && - t.Elem().Kind() == reflect.Interface - } - // case 1: A map inside a struct will have a fixed key and value type. - // case 2: The user has specified the type of the map explicitly. - // Otherwise, a generic map type will be used - switch { - case s.mapInStruct: - value.Set(reflect.MakeMap(type_)) - case !isIfaceMap(type_): - value.Set(reflect.MakeMap(type_)) - default: + mapType := type_ + // For interface{} maps without declared types, use map[interface{}]interface{} + if !s.hasGenerics && type_.Key().Kind() == reflect.Interface && type_.Elem().Kind() == reflect.Interface { iface := reflect.TypeOf((*interface{})(nil)).Elem() - newMapType := reflect.MapOf(iface, iface) - value.Set(reflect.MakeMap(newMapType)) + mapType = reflect.MapOf(iface, iface) } + value.Set(reflect.MakeMap(mapType)) } - refResolver.Reference(value) + size := int(buf.ReadVaruint32(ctxErr)) - var chunkHeader uint8 - if size > 0 { - chunkHeader = buf.ReadUint8(ctxErr) + if size == 0 || ctx.HasError() { + return } + + chunkHeader := buf.ReadUint8(ctxErr) if ctx.HasError() { return } keyType := type_.Key() valueType := type_.Elem() - keySer := s.keySerializer - valSer := s.valueSerializer - typeResolver := ctx.TypeResolver() for size > 0 { + // Phase 1: Handle null entries for { keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0 valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0 - var k, v reflect.Value - if !keyHasNull { - if !valueHasNull { - break - } else { - // Null value case: read key only - keyDeclared := (chunkHeader & KEY_DECL_TYPE) != 0 - trackKeyRef := (chunkHeader & TRACKING_KEY_REF) != 0 - - // When trackKeyRef is set and type is not declared, Java writes: - // ref flag + type info + data - // So we need to read ref flag first, then type info, then data - if trackKeyRef && !keyDeclared { - // Read ref flag first - refID, err := refResolver.TryPreserveRefId(buf) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := refResolver.GetReadObject(refID) - if obj.IsValid() { - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(obj, nullVal) - } - size-- - if size == 0 { - return - } - chunkHeader = buf.ReadUint8(ctxErr) - continue - } - - // Read type info - ti := typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = ti.Serializer - keyType = ti.Type - - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() - - // Read data (ref already handled) - keySer.ReadData(ctx, keyType, k) - if ctx.HasError() { - return - } - refResolver.Reference(k) - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(k, nullVal) - } else { - // ReadData type info if not declared - var keyTypeInfo *TypeInfo - if !keyDeclared { - keyTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = keyTypeInfo.Serializer - keyType = keyTypeInfo.Type - } - - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() - - // Use ReadWithTypeInfo if type was read, otherwise Read - keyRefMode := RefModeNone - if trackKeyRef { - keyRefMode = RefModeTracking - } - if keyTypeInfo != nil { - keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) - } else { - keySer.Read(ctx, keyRefMode, false, false, k) - } - if ctx.HasError() { - return - } - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(k, nullVal) - } + + if !keyHasNull && !valueHasNull { + break // Proceed to regular chunk + } + + if keyHasNull && valueHasNull { + value.SetMapIndex(reflect.Zero(keyType), reflect.Zero(valueType)) + } else if valueHasNull { + k := s.readNullValueEntry(ctx, chunkHeader, keyType, typeResolver, refResolver) + if ctx.HasError() { + return } + value.SetMapIndex(k, reflect.Zero(valueType)) } else { - if !valueHasNull { - // Null key case: read value only - valueDeclared := (chunkHeader & VALUE_DECL_TYPE) != 0 - trackValueRef := (chunkHeader & TRACKING_VALUE_REF) != 0 - - // When trackValueRef is set and type is not declared, Java writes: - // ref flag + type info + data - // So we need to read ref flag first, then type info, then data - if trackValueRef && !valueDeclared { - // Read ref flag first - refID, err := refResolver.TryPreserveRefId(buf) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := refResolver.GetReadObject(refID) - if obj.IsValid() { - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, obj) - } - size-- - if size == 0 { - return - } - chunkHeader = buf.ReadUint8(ctxErr) - continue - } - - // Read type info - ti := typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - valSer = ti.Serializer - valueType = ti.Type - - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() - - // Read data (ref already handled) - valSer.ReadData(ctx, valueType, v) - if ctx.HasError() { - return - } - refResolver.Reference(v) - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, v) - } else { - // ReadData type info if not declared - var valueTypeInfo *TypeInfo - if !valueDeclared { - valueTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - valSer = valueTypeInfo.Serializer - valueType = valueTypeInfo.Type - } - - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() - - // Use ReadWithTypeInfo if type was read, otherwise Read - valueRefMode := RefModeNone - if trackValueRef { - valueRefMode = RefModeTracking - } - if valueTypeInfo != nil { - valSer.ReadWithTypeInfo(ctx, valueRefMode, valueTypeInfo, v) - } else { - valSer.Read(ctx, valueRefMode, false, false, v) - } - if ctx.HasError() { - return - } - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, v) - } - } else { - // Both key and value are null - nullKey := reflect.Zero(value.Type().Key()) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(nullKey, nullVal) + v := s.readNullKeyEntry(ctx, chunkHeader, valueType, typeResolver, refResolver) + if ctx.HasError() { + return } + value.SetMapIndex(reflect.Zero(keyType), v) } size-- if size == 0 { return - } else { - chunkHeader = buf.ReadUint8(ctxErr) + } + chunkHeader = buf.ReadUint8(ctxErr) + if ctx.HasError() { + return } } - trackKeyRef := (chunkHeader & TRACKING_KEY_REF) != 0 - trackValRef := (chunkHeader & TRACKING_VALUE_REF) != 0 - keyDeclType := (chunkHeader & KEY_DECL_TYPE) != 0 - valDeclType := (chunkHeader & VALUE_DECL_TYPE) != 0 - chunkSize := int(buf.ReadUint8(ctxErr)) + // Phase 2: Read regular chunk + size = s.readChunk(ctx, value, chunkHeader, size, keyType, valueType, typeResolver) + if ctx.HasError() { + return + } - // ReadData type info if not declared - var keyTypeInfo *TypeInfo - if !keyDeclType { - keyTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = keyTypeInfo.Serializer - keyType = keyTypeInfo.Type - } else if keySer == nil { - // KEY_DECL_TYPE is set but we don't have a serializer - get one from the map's key type - keySer, _ = typeResolver.getSerializerByType(keyType, false) - } - var valueTypeInfo *TypeInfo - if !valDeclType { - valueTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { + if size > 0 { + chunkHeader = buf.ReadUint8(ctxErr) + if ctx.HasError() { return } - valSer = valueTypeInfo.Serializer - valueType = valueTypeInfo.Type - } else if valSer == nil { - // VALUE_DECL_TYPE is set but we don't have a serializer - get one from the map's value type - valSer, _ = typeResolver.getSerializerByType(valueType, false) } + } +} + +// readNullValueEntry reads an entry where value is null, returns the key +func (s mapSerializer) readNullValueEntry(ctx *ReadContext, header uint8, keyType reflect.Type, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + buf := ctx.Buffer() + ctxErr := ctx.Err() + keyDeclared := (header & KEY_DECL_TYPE) != 0 + trackKeyRef := (header & TRACKING_KEY_REF) != 0 - keyRefMode := RefModeNone - if trackKeyRef { - keyRefMode = RefModeTracking + return s.readSingleValue(ctx, buf, ctxErr, keyDeclared, trackKeyRef, keyType, s.keySerializer, resolver, refResolver) +} + +// readNullKeyEntry reads an entry where key is null, returns the value +func (s mapSerializer) readNullKeyEntry(ctx *ReadContext, header uint8, valueType reflect.Type, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + buf := ctx.Buffer() + ctxErr := ctx.Err() + valueDeclared := (header & VALUE_DECL_TYPE) != 0 + trackValueRef := (header & TRACKING_VALUE_REF) != 0 + + return s.readSingleValue(ctx, buf, ctxErr, valueDeclared, trackValueRef, valueType, s.valueSerializer, resolver, refResolver) +} + +// readSingleValue reads a single key or value with proper ref/type handling +func (s mapSerializer) readSingleValue(ctx *ReadContext, buf *ByteBuffer, ctxErr *Error, isDeclared, trackRef bool, staticType reflect.Type, declaredSer Serializer, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + // When ref tracking AND not declared, ref flag comes before type info + if trackRef && !isDeclared { + refID, err := refResolver.TryPreserveRefId(buf) + if err != nil { + ctx.SetError(FromError(err)) + return reflect.Value{} } - valRefMode := RefModeNone - if trackValRef { - valRefMode = RefModeTracking + if refID < int32(NotNullValueFlag) { + return refResolver.GetReadObject(refID) } - for i := 0; i < chunkSize; i++ { - var k, v reflect.Value - // ReadData key - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() + // Read type info and data + ti := resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return reflect.Value{} + } - // Use ReadWithTypeInfo if type was read, otherwise Read - if keyTypeInfo != nil { - keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) - } else { - keySer.Read(ctx, keyRefMode, false, false, k) - } - if ctx.HasError() { - return - } + valType := ti.Type + if valType == nil { + valType = staticType + } + v := reflect.New(valType).Elem() + ti.Serializer.ReadData(ctx, valType, v) + if ctx.HasError() { + return reflect.Value{} + } + refResolver.Reference(v) + return v + } - // ReadData value - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() + // Read type info if not declared + var typeInfo *TypeInfo + var ser Serializer + valType := staticType - // Use ReadWithTypeInfo if type was read, otherwise Read - if valueTypeInfo != nil { - valSer.ReadWithTypeInfo(ctx, valRefMode, valueTypeInfo, v) - } else { - valSer.Read(ctx, valRefMode, false, false, v) - } - if ctx.HasError() { - return - } - // Unwrap interfaces if they're not the map's type - if k.Kind() == reflect.Interface { - k = k.Elem() - } - if v.Kind() == reflect.Interface { - v = v.Elem() - } - setMapValue(value, k, v) - size-- + if !isDeclared { + typeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return reflect.Value{} + } + ser = typeInfo.Serializer + valType = typeInfo.Type + } else { + ser = declaredSer + if ser == nil { + ser, _ = resolver.getSerializerByType(staticType, false) } + } + + if valType == nil { + valType = staticType + } + v := reflect.New(valType).Elem() + + refMode := RefModeNone + if trackRef { + refMode = RefModeTracking + } + + if typeInfo != nil { + ser.ReadWithTypeInfo(ctx, refMode, typeInfo, v) + } else { + ser.Read(ctx, refMode, false, false, v) + } + + return v +} +// readChunk reads a chunk of entries, returns remaining size +func (s mapSerializer) readChunk(ctx *ReadContext, mapVal reflect.Value, header uint8, size int, keyType, valueType reflect.Type, resolver *TypeResolver) int { + buf := ctx.Buffer() + ctxErr := ctx.Err() + + trackKeyRef := (header & TRACKING_KEY_REF) != 0 + trackValRef := (header & TRACKING_VALUE_REF) != 0 + keyDeclType := (header & KEY_DECL_TYPE) != 0 + valDeclType := (header & VALUE_DECL_TYPE) != 0 + + chunkSize := int(buf.ReadUint8(ctxErr)) + if ctx.HasError() { + return 0 + } + + // Read type info if not declared + var keyTypeInfo, valueTypeInfo *TypeInfo + var keySer, valSer Serializer + + if !keyDeclType { + keyTypeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return 0 + } + keySer = keyTypeInfo.Serializer + keyType = keyTypeInfo.Type + } else { keySer = s.keySerializer + if keySer == nil { + keySer, _ = resolver.getSerializerByType(keyType, false) + } + } + + if !valDeclType { + valueTypeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return 0 + } + valSer = valueTypeInfo.Serializer + valueType = valueTypeInfo.Type + } else { valSer = s.valueSerializer - if size > 0 { - chunkHeader = buf.ReadUint8(ctxErr) + if valSer == nil { + valSer, _ = resolver.getSerializerByType(valueType, false) } } -} -func (s mapSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - done := readMapRefAndType(ctx, refMode, readType, value) - if done || ctx.HasError() { - return + keyRefMode := RefModeNone + if trackKeyRef { + keyRefMode = RefModeTracking } - s.ReadData(ctx, value.Type(), value) + valRefMode := RefModeNone + if trackValRef { + valRefMode = RefModeTracking + } + + for i := 0; i < chunkSize; i++ { + k := reflect.New(keyType).Elem() + if keyTypeInfo != nil { + keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) + } else { + keySer.Read(ctx, keyRefMode, false, false, k) + } + if ctx.HasError() { + return 0 + } + + v := reflect.New(valueType).Elem() + if valueTypeInfo != nil { + valSer.ReadWithTypeInfo(ctx, valRefMode, valueTypeInfo, v) + } else { + valSer.Read(ctx, valRefMode, false, false, v) + } + if ctx.HasError() { + return 0 + } + + setMapValue(mapVal, unwrapInterface(k), unwrapInterface(v)) + size-- + } + + return size } func (s mapSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again s.Read(ctx, refMode, false, false, value) } -func (s mapSerializer) readObj( - ctx *ReadContext, - v *reflect.Value, - serializer Serializer, -) { - serializer.ReadData(ctx, v.Type(), *v) +// Helper functions + +// writeMapRefAndType handles reference and type writing for maps. +// Returns true if value was already written (nil or ref). +func writeMapRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, value reflect.Value) bool { + switch refMode { + case RefModeTracking: + if value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return true + } + refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) + if err != nil { + ctx.SetError(FromError(err)) + return false + } + if refWritten { + return true + } + case RefModeNullOnly: + if value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return true + } + ctx.buffer.WriteInt8(NotNullValueFlag) + } + if writeType { + ctx.buffer.WriteVaruint32Small7(uint32(MAP)) + } + return false } -func getActualType(v reflect.Value) reflect.Type { - if v.Kind() == reflect.Interface && !v.IsNil() { - return v.Elem().Type() +// readMapRefAndType handles reference and type reading for maps. +// Returns true if a reference was resolved. +func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value reflect.Value) bool { + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, err := ctx.RefResolver().TryPreserveRefId(buf) + if err != nil { + ctx.SetError(FromError(err)) + return false + } + if refID < int32(NotNullValueFlag) { + obj := ctx.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return true + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return true + } } - return v.Type() + if readType { + buf.ReadVaruint32Small7(ctxErr) + } + return false +} + +func unwrapInterface(v reflect.Value) reflect.Value { + // For map serialization, we need to unwrap interfaces including nil ones + // A nil interface should become an invalid (zero) Value for proper null detection + if v.Kind() == reflect.Interface { + return v.Elem() + } + return v +} + +// UnwrapReflectValue is exported for use by other packages +func UnwrapReflectValue(v reflect.Value) reflect.Value { + return unwrapInterface(v) } -func getActualTypeInfo(v reflect.Value, resolver *TypeResolver) (*TypeInfo, error) { +func getTypeInfoForValue(v reflect.Value, resolver *TypeResolver) (*TypeInfo, error) { if v.Kind() == reflect.Interface && !v.IsNil() { elem := v.Elem() if !elem.IsValid() { @@ -690,44 +634,23 @@ func getActualTypeInfo(v reflect.Value, resolver *TypeResolver) (*TypeInfo, erro return resolver.getTypeInfo(v, true) } -func UnwrapReflectValue(v reflect.Value) reflect.Value { - for v.Kind() == reflect.Interface && !v.IsNil() { - v = v.Elem() - } - return v -} - -func isValid(v reflect.Value) bool { - // Zero values are valid, so apply this change temporarily. - return v.IsValid() -} - -// setMapValue sets a key-value pair into a map, handling interface types where -// the concrete type may need to be wrapped in a pointer to implement the interface. +// setMapValue sets a key-value pair into a map, handling interface types func setMapValue(mapVal, key, value reflect.Value) { mapKeyType := mapVal.Type().Key() mapValueType := mapVal.Type().Elem() - // Handle key finalKey := key - if mapKeyType.Kind() == reflect.Interface { - if !key.Type().AssignableTo(mapKeyType) { - // Try pointer - common case where interface has pointer receivers - ptr := reflect.New(key.Type()) - ptr.Elem().Set(key) - finalKey = ptr - } + if mapKeyType.Kind() == reflect.Interface && !key.Type().AssignableTo(mapKeyType) { + ptr := reflect.New(key.Type()) + ptr.Elem().Set(key) + finalKey = ptr } - // Handle value finalValue := value - if mapValueType.Kind() == reflect.Interface { - if !value.Type().AssignableTo(mapValueType) { - // Try pointer - common case where interface has pointer receivers - ptr := reflect.New(value.Type()) - ptr.Elem().Set(value) - finalValue = ptr - } + if mapValueType.Kind() == reflect.Interface && !value.Type().AssignableTo(mapValueType) { + ptr := reflect.New(value.Type()) + ptr.Elem().Set(value) + finalValue = ptr } mapVal.SetMapIndex(finalKey, finalValue) diff --git a/go/fory/pointer.go b/go/fory/pointer.go index 003287c356..7c3ef54818 100644 --- a/go/fory/pointer.go +++ b/go/fory/pointer.go @@ -178,8 +178,8 @@ func (s *ptrToInterfaceSerializer) WriteData(ctx *WriteContext, value reflect.Va // Get the concrete element that the interface pointer points to elemValue := value.Elem() - // Use WriteValue to handle the polymorphic interface value - ctx.WriteValue(elemValue) + // Use WriteValue to handle the polymorphic interface value with ref tracking and type info + ctx.WriteValue(elemValue, RefModeTracking, true) } func (s *ptrToInterfaceSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { @@ -206,7 +206,7 @@ func (s *ptrToInterfaceSerializer) Write(ctx *WriteContext, refMode RefMode, wri } // For interface pointers, we don't write type info here - // WriteValue will handle the type info for the concrete value + // WriteData will call WriteValue which handles the type info for the concrete value s.WriteData(ctx, value) } @@ -214,8 +214,8 @@ func (s *ptrToInterfaceSerializer) ReadData(ctx *ReadContext, type_ reflect.Type // Create a new interface pointer newVal := reflect.New(type_.Elem()) - // Use ReadValue to handle the polymorphic interface value - ctx.ReadValue(newVal.Elem()) + // Use ReadValue to handle the polymorphic interface value with ref tracking and type info + ctx.ReadValue(newVal.Elem(), RefModeTracking, true) if ctx.HasError() { return } diff --git a/go/fory/reader.go b/go/fory/reader.go index 8bd05a883c..2248947510 100644 --- a/go/fory/reader.go +++ b/go/fory/reader.go @@ -32,6 +32,7 @@ type ReadContext struct { buffer *ByteBuffer refReader *RefReader trackRef bool // Cached flag to avoid indirection + xlang bool // Cross-language serialization mode compatible bool // Schema evolution compatibility mode typeResolver *TypeResolver // For complex type deserialization refResolver *RefResolver // For reference tracking (legacy) @@ -42,6 +43,11 @@ type ReadContext struct { err Error // Accumulated error state for deferred checking } +// IsXlang returns whether cross-language serialization mode is enabled +func (c *ReadContext) IsXlang() bool { + return c.xlang +} + // NewReadContext creates a new read context func NewReadContext(trackRef bool) *ReadContext { return &ReadContext{ @@ -548,8 +554,11 @@ func (c *ReadContext) decDepth() { c.depth-- } -// ReadValue reads a polymorphic value - queries serializer by type and deserializes -func (c *ReadContext) ReadValue(value reflect.Value) { +// ReadValue reads a polymorphic value with configurable reference tracking and type info reading. +// Parameters: +// - refMode: controls reference tracking behavior (RefModeNone, RefModeTracking, RefModeNullOnly) +// - readType: if true, reads type info from the buffer +func (c *ReadContext) ReadValue(value reflect.Value, refMode RefMode, readType bool) { if !value.IsValid() { c.SetError(DeserializationError("invalid reflect.Value")) return @@ -557,28 +566,41 @@ func (c *ReadContext) ReadValue(value reflect.Value) { // Handle array targets (arrays are serialized as slices) if value.Type().Kind() == reflect.Array { - c.readArrayValue(value) + c.ReadArrayValue(value, refMode, readType) return } // For interface{} types, we need to read the actual type from the buffer first if value.Type().Kind() == reflect.Interface { - // Read ref flag - refID, err := c.RefResolver().TryPreserveRefId(c.buffer) - if err != nil { - c.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found - obj := c.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) + // Handle ref tracking based on refMode + var refID int32 = int32(NotNullValueFlag) + if refMode == RefModeTracking { + var err error + refID, err = c.RefResolver().TryPreserveRefId(c.buffer) + if err != nil { + c.SetError(FromError(err)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found + obj := c.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return + } + } else if refMode == RefModeNullOnly { + flag := c.buffer.ReadInt8(c.Err()) + if flag == NullFlag { + return } - return } // Read type info to determine the actual type + if !readType { + c.SetError(DeserializationError("cannot read interface{} without type info")) + return + } ctxErr := c.Err() typeInfo := c.typeResolver.ReadTypeInfo(c.buffer, ctxErr) if ctxErr.HasError() { @@ -625,7 +647,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { // For named structs, register the pointer BEFORE reading data // This is critical for circular references to work correctly - if isNamedStruct && refID >= int32(NotNullValueFlag) { + if isNamedStruct && refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, newValue) } @@ -643,7 +665,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { } // Register reference after reading data for non-struct types - if !isNamedStruct && refID >= int32(NotNullValueFlag) { + if !isNamedStruct && refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, newValue) } @@ -652,15 +674,17 @@ func (c *ReadContext) ReadValue(value reflect.Value) { return } - // For struct types, use optimized ReadStruct path + // For struct types, use optimized ReadStruct path when using full ref tracking and type info valueType := value.Type() - if valueType.Kind() == reflect.Struct { - c.ReadStruct(value) - return - } - if valueType.Kind() == reflect.Ptr && valueType.Elem().Kind() == reflect.Struct { - c.ReadStruct(value) - return + if refMode == RefModeTracking && readType { + if valueType.Kind() == reflect.Struct { + c.ReadStruct(value) + return + } + if valueType.Kind() == reflect.Ptr && valueType.Elem().Kind() == reflect.Struct { + c.ReadStruct(value) + return + } } // Get serializer for the value's type @@ -671,7 +695,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { } // Read handles ref tracking and type info internally - serializer.Read(c, RefModeTracking, true, false, value) + serializer.Read(c, refMode, readType, false, value) } // ReadStruct reads a struct value with optimized type resolution. @@ -762,26 +786,38 @@ func (c *ReadContext) ReadInto(value reflect.Value, serializer Serializer, refMo serializer.Read(c, refMode, readTypeInfo, false, value) } -// readArrayValue handles array targets when stream contains slice data -// Arrays are serialized as slices in xlang protocol -func (c *ReadContext) readArrayValue(target reflect.Value) { - // Read ref flag - refID, err := c.RefResolver().TryPreserveRefId(c.buffer) - if err != nil { - c.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := c.RefResolver().GetReadObject(refID) - if obj.IsValid() { - reflect.Copy(target, obj) +// ReadArrayValue handles array targets with configurable ref mode and type reading. +// Arrays are serialized as slices in xlang protocol. +func (c *ReadContext) ReadArrayValue(target reflect.Value, refMode RefMode, readType bool) { + var refID int32 = int32(NotNullValueFlag) + + // Handle ref tracking based on refMode + if refMode == RefModeTracking { + var err error + refID, err = c.RefResolver().TryPreserveRefId(c.buffer) + if err != nil { + c.SetError(FromError(err)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference to existing object + obj := c.RefResolver().GetReadObject(refID) + if obj.IsValid() { + reflect.Copy(target, obj) + } + return + } + } else if refMode == RefModeNullOnly { + flag := c.buffer.ReadInt8(c.Err()) + if flag == NullFlag { + return } - return } - // Read type ID (will be slice type in stream) - c.buffer.ReadVaruint32Small7(c.Err()) + // Read type ID if requested (will be slice type in stream) + if readType { + c.buffer.ReadVaruint32Small7(c.Err()) + } // Get slice serializer to read the data sliceType := reflect.SliceOf(target.Type().Elem()) @@ -812,7 +848,7 @@ func (c *ReadContext) readArrayValue(target reflect.Value) { reflect.Copy(target, tempSlice) // Register for circular refs - if refID >= int32(NotNullValueFlag) { + if refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, target) } } diff --git a/go/fory/ref_resolver.go b/go/fory/ref_resolver.go index 19ed5705f4..8003a1a6f9 100644 --- a/go/fory/ref_resolver.go +++ b/go/fory/ref_resolver.go @@ -88,6 +88,33 @@ func isReferencable(t reflect.Type) bool { } } +// isRefType determines if a type should track references. +// For xlang mode, only pointer to struct/ext, interface, slice, and map track refs. +// Arrays do NOT track refs in xlang mode (they are value types). +// For native Go mode, uses standard Go reference semantics. +// +// Note: Struct fields can provide tags to override ref tracking behavior for specific +// field types. However, if the value is not a struct field (e.g., top-level value, +// collection element), then this function's result is always used. +func isRefType(t reflect.Type, xlang bool) bool { + kind := t.Kind() + if xlang { + switch kind { + case reflect.Ptr: + // Only pointer to struct tracks ref in xlang + elemKind := t.Elem().Kind() + return elemKind == reflect.Struct + case reflect.Interface, reflect.Slice, reflect.Map: + return true + default: + // Arrays and other types don't track refs in xlang + return false + } + } + // Native Go mode: pointers, maps, slices, and interfaces track refs + return isReferencable(t) +} + func newRefResolver(refTracking bool) *RefResolver { refResolver := &RefResolver{ refTracking: refTracking, diff --git a/go/fory/slice.go b/go/fory/slice.go index e085c6acb3..4b40374fd3 100644 --- a/go/fory/slice.go +++ b/go/fory/slice.go @@ -106,45 +106,45 @@ func isNull(v reflect.Value) bool { } } -// sliceConcreteValueSerializer serialize a slice whose elem is not an interface or pointer to interface. -// Use newSliceConcreteValueSerializer to create instances with proper type validation. +// sliceSerializer serialize a slice whose elem is not an interface or pointer to interface. +// Use newSliceSerializer to create instances with proper type validation. // This serializer uses LIST protocol for non-primitive element types. -type sliceConcreteValueSerializer struct { +type sliceSerializer struct { type_ reflect.Type elemSerializer Serializer referencable bool } -// newSliceConcreteValueSerializer creates a sliceConcreteValueSerializer for slices with concrete element types. +// newSliceSerializer creates a sliceSerializer for slices with concrete element types. // It returns an error if the element type is an interface, pointer to interface, or a primitive type. // Primitive numeric types (bool, int8, int16, int32, int64, uint8, float32, float64) must use // dedicated primitive slice serializers that use ARRAY protocol (binary size + binary). -func newSliceConcreteValueSerializer(type_ reflect.Type, elemSerializer Serializer) (*sliceConcreteValueSerializer, error) { +func newSliceSerializer(type_ reflect.Type, elemSerializer Serializer, xlang bool) (*sliceSerializer, error) { elem := type_.Elem() if elem.Kind() == reflect.Interface { - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support interface element type: %v", type_) + return nil, fmt.Errorf("sliceSerializer does not support interface element type: %v", type_) } if elem.Kind() == reflect.Ptr && elem.Elem().Kind() == reflect.Interface { - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support pointer to interface element type: %v", type_) + return nil, fmt.Errorf("sliceSerializer does not support pointer to interface element type: %v", type_) } // Primitive numeric types must use dedicated primitive slice serializers (ARRAY protocol) switch elem.Kind() { case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Float32, reflect.Float64: - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support primitive element type %v: use dedicated primitive slice serializer", type_) + return nil, fmt.Errorf("sliceSerializer does not support primitive element type %v: use dedicated primitive slice serializer", type_) } - return &sliceConcreteValueSerializer{ + return &sliceSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(elem), + referencable: isRefType(elem, xlang), }, nil } -func (s *sliceConcreteValueSerializer) WriteData(ctx *WriteContext, value reflect.Value) { +func (s *sliceSerializer) WriteData(ctx *WriteContext, value reflect.Value) { s.writeDataWithGenerics(ctx, value, false) } -func (s *sliceConcreteValueSerializer) writeDataWithGenerics(ctx *WriteContext, value reflect.Value, hasGenerics bool) { +func (s *sliceSerializer) writeDataWithGenerics(ctx *WriteContext, value reflect.Value, hasGenerics bool) { length := value.Len() buf := ctx.Buffer() @@ -233,7 +233,7 @@ func (s *sliceConcreteValueSerializer) writeDataWithGenerics(ctx *WriteContext, } } -func (s *sliceConcreteValueSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { +func (s *sliceSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { done := writeSliceRefAndType(ctx, refMode, writeType, value, LIST) if done || ctx.HasError() { return @@ -241,7 +241,7 @@ func (s *sliceConcreteValueSerializer) Write(ctx *WriteContext, refMode RefMode, s.writeDataWithGenerics(ctx, value, hasGenerics) } -func (s *sliceConcreteValueSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { +func (s *sliceSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { done := readSliceRefAndType(ctx, refMode, readType, value) if done || ctx.HasError() { return @@ -249,12 +249,12 @@ func (s *sliceConcreteValueSerializer) Read(ctx *ReadContext, refMode RefMode, r s.ReadData(ctx, value.Type(), value) } -func (s *sliceConcreteValueSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { +func (s *sliceSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { // typeInfo is already read, don't read it again s.Read(ctx, refMode, false, false, value) } -func (s *sliceConcreteValueSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { +func (s *sliceSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { buf := ctx.Buffer() ctxErr := ctx.Err() length := int(buf.ReadVaruint32(ctxErr)) diff --git a/go/fory/slice_dyn.go b/go/fory/slice_dyn.go index 64cd22ce38..69c339d857 100644 --- a/go/fory/slice_dyn.go +++ b/go/fory/slice_dyn.go @@ -26,7 +26,7 @@ import ( // element values at runtime. // This serializer is designed for slices with any interface element type // (e.g., []interface{}, []io.Reader, []fmt.Stringer, or pointers to interfaces). -// For slices with concrete element types, use sliceConcreteValueSerializer instead. +// For slices with concrete element types, use sliceSerializer instead. type sliceDynSerializer struct { elemType reflect.Type isInterfaceElem bool @@ -35,7 +35,7 @@ type sliceDynSerializer struct { // newSliceDynSerializer creates a new sliceDynSerializer. // This serializer is ONLY for slices with interface or pointer to interface element types. -// For other slice types, use sliceConcreteValueSerializer instead. +// For other slice types, use sliceSerializer instead. func newSliceDynSerializer(elemType reflect.Type) (sliceDynSerializer, error) { // Nil element type is allowed for fully dynamic slices (e.g., []interface{}) if elemType == nil { @@ -48,7 +48,7 @@ func newSliceDynSerializer(elemType reflect.Type) (sliceDynSerializer, error) { isPointerToInterface := elemType.Kind() == reflect.Ptr && elemType.Elem().Kind() == reflect.Interface if !isInterface && !isPointerToInterface { return sliceDynSerializer{}, fmt.Errorf( - "sliceDynSerializer only supports interface or pointer to interface element types, got %v; use sliceConcreteValueSerializer for other types", elemType) + "sliceDynSerializer only supports interface or pointer to interface element types, got %v; use sliceSerializer for other types", elemType) } return sliceDynSerializer{ elemType: elemType, diff --git a/go/fory/struct.go b/go/fory/struct.go index e7e655fc7e..aa0a1cd510 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -18,78 +18,78 @@ package fory import ( - "encoding/binary" - "errors" - "fmt" - "math" - "reflect" - "sort" - "strings" - "unicode" - "unicode/utf8" - "unsafe" - - "github.com/spaolacci/murmur3" + "encoding/binary" + "errors" + "fmt" + "math" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + "unsafe" + + "github.com/spaolacci/murmur3" ) // FieldInfo stores field metadata computed ENTIRELY at init time. // All flags and decisions are pre-computed to eliminate runtime checks. type FieldInfo struct { - Name string - Offset uintptr - Type reflect.Type - StaticId StaticTypeId - TypeId TypeId // Fory type ID for the serializer - Serializer Serializer - Referencable bool - FieldIndex int // -1 if field doesn't exist in current struct (for compatible mode) - FieldDef FieldDef // original FieldDef from remote TypeDef (for compatible mode skip) - - // Pre-computed sizes and offsets (for fixed primitives) - FixedSize int // 0 if not fixed-size, else 1/2/4/8 - WriteOffset int // Offset within fixed-fields buffer region (sum of preceding field sizes) - - // Pre-computed flags for serialization (computed at init time) - RefMode RefMode // ref mode for serializer.Write/Read - WriteType bool // whether to write type info (true for struct fields in compatible mode) - HasGenerics bool // whether element types are known from TypeDef (for container fields) - - // Tag-based configuration (from fory struct tags) - TagID int // -1 = use field name, >=0 = use tag ID - HasForyTag bool // Whether field has explicit fory tag - TagRefSet bool // Whether ref was explicitly set via fory tag - TagRef bool // The ref value from fory tag (only valid if TagRefSet is true) - TagNullableSet bool // Whether nullable was explicitly set via fory tag - TagNullable bool // The nullable value from fory tag (only valid if TagNullableSet is true) + Name string + Offset uintptr + Type reflect.Type + StaticId StaticTypeId + TypeId TypeId // Fory type ID for the serializer + Serializer Serializer + Referencable bool + FieldIndex int // -1 if field doesn't exist in current struct (for compatible mode) + FieldDef FieldDef // original FieldDef from remote TypeDef (for compatible mode skip) + + // Pre-computed sizes and offsets (for fixed primitives) + FixedSize int // 0 if not fixed-size, else 1/2/4/8 + WriteOffset int // Offset within fixed-fields buffer region (sum of preceding field sizes) + + // Pre-computed flags for serialization (computed at init time) + RefMode RefMode // ref mode for serializer.Write/Read + WriteType bool // whether to write type info (true for struct fields in compatible mode) + HasGenerics bool // whether element types are known from TypeDef (for container fields) + + // Tag-based configuration (from fory struct tags) + TagID int // -1 = use field name, >=0 = use tag ID + HasForyTag bool // Whether field has explicit fory tag + TagRefSet bool // Whether ref was explicitly set via fory tag + TagRef bool // The ref value from fory tag (only valid if TagRefSet is true) + TagNullableSet bool // Whether nullable was explicitly set via fory tag + TagNullable bool // The nullable value from fory tag (only valid if TagNullableSet is true) } // fieldHasNonPrimitiveSerializer returns true if the field has a serializer with a non-primitive type ID. // This is used to skip the fast path for fields like enums where StaticId is int32 but the serializer // writes a different format (e.g., unsigned varint for enum ordinals vs signed zigzag for int32). func fieldHasNonPrimitiveSerializer(field *FieldInfo) bool { - if field.Serializer == nil { - return false - } - // ENUM (numeric ID), NAMED_ENUM (namespace/typename), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT - // all require special serialization and should not use the primitive fast path - // Note: ENUM uses unsigned Varuint32Small7 for ordinals, not signed zigzag varint - // Use internal type ID (low 8 bits) since registered types have composite TypeIds like (userID << 8) | internalID - internalTypeId := TypeId(field.TypeId & 0xFF) - switch internalTypeId { - case ENUM, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: - return true - default: - return false - } + if field.Serializer == nil { + return false + } + // ENUM (numeric ID), NAMED_ENUM (namespace/typename), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT + // all require special serialization and should not use the primitive fast path + // Note: ENUM uses unsigned Varuint32Small7 for ordinals, not signed zigzag varint + // Use internal type ID (low 8 bits) since registered types have composite TypeIds like (userID << 8) | internalID + internalTypeId := TypeId(field.TypeId & 0xFF) + switch internalTypeId { + case ENUM, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: + return true + default: + return false + } } // isEnumField checks if a field is an enum type based on its TypeId func isEnumField(field *FieldInfo) bool { - if field.Serializer == nil { - return false - } - internalTypeId := field.TypeId & 0xFF - return internalTypeId == ENUM || internalTypeId == NAMED_ENUM + if field.Serializer == nil { + return false + } + internalTypeId := field.TypeId & 0xFF + return internalTypeId == ENUM || internalTypeId == NAMED_ENUM } // writeEnumField writes an enum field respecting the field's RefMode. @@ -97,37 +97,37 @@ func isEnumField(field *FieldInfo) bool { // RefMode determines whether null flag is written, regardless of whether the local type is a pointer. // This is important for compatible mode where remote TypeDef's nullable flag controls the wire format. func writeEnumField(ctx *WriteContext, field *FieldInfo, fieldValue reflect.Value) { - buf := ctx.Buffer() - isPointer := fieldValue.Kind() == reflect.Ptr - - // Write null flag based on RefMode only (not based on whether local type is pointer) - if field.RefMode != RefModeNone { - if isPointer && fieldValue.IsNil() { - buf.WriteInt8(NullFlag) - return - } - buf.WriteInt8(NotNullValueFlag) - } - - // Get the actual value to serialize - targetValue := fieldValue - if isPointer { - if fieldValue.IsNil() { - // RefModeNone but nil pointer - this is a protocol error in schema-consistent mode - // Write zero value as fallback - targetValue = reflect.Zero(field.Type.Elem()) - } else { - targetValue = fieldValue.Elem() - } - } - - // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. - // We need to call the inner enumSerializer directly with the dereferenced value. - if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - ptrSer.valueSerializer.WriteData(ctx, targetValue) - } else { - field.Serializer.WriteData(ctx, targetValue) - } + buf := ctx.Buffer() + isPointer := fieldValue.Kind() == reflect.Ptr + + // Write null flag based on RefMode only (not based on whether local type is pointer) + if field.RefMode != RefModeNone { + if isPointer && fieldValue.IsNil() { + buf.WriteInt8(NullFlag) + return + } + buf.WriteInt8(NotNullValueFlag) + } + + // Get the actual value to serialize + targetValue := fieldValue + if isPointer { + if fieldValue.IsNil() { + // RefModeNone but nil pointer - this is a protocol error in schema-consistent mode + // Write zero value as fallback + targetValue = reflect.Zero(field.Type.Elem()) + } else { + targetValue = fieldValue.Elem() + } + } + + // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. + // We need to call the inner enumSerializer directly with the dereferenced value. + if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + ptrSer.valueSerializer.WriteData(ctx, targetValue) + } else { + field.Serializer.WriteData(ctx, targetValue) + } } // readEnumField reads an enum field respecting the field's RefMode. @@ -135,1464 +135,1465 @@ func writeEnumField(ctx *WriteContext, field *FieldInfo, fieldValue reflect.Valu // This is important for compatible mode where remote TypeDef's nullable flag controls the wire format. // Uses context error state for deferred error checking. func readEnumField(ctx *ReadContext, field *FieldInfo, fieldValue reflect.Value) { - buf := ctx.Buffer() - isPointer := fieldValue.Kind() == reflect.Ptr - - // Read null flag based on RefMode only (not based on whether local type is pointer) - if field.RefMode != RefModeNone { - nullFlag := buf.ReadInt8(ctx.Err()) - if nullFlag == NullFlag { - // For pointer enum fields, leave as nil; for non-pointer, set to zero - if !isPointer { - fieldValue.SetInt(0) - } - return - } - } - - // For pointer enum fields, allocate a new value - targetValue := fieldValue - if isPointer { - newVal := reflect.New(field.Type.Elem()) - fieldValue.Set(newVal) - targetValue = newVal.Elem() - } - - // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. - // We need to call the inner enumSerializer directly with the dereferenced value. - if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - ptrSer.valueSerializer.ReadData(ctx, field.Type.Elem(), targetValue) - } else { - field.Serializer.ReadData(ctx, field.Type, targetValue) - } + buf := ctx.Buffer() + isPointer := fieldValue.Kind() == reflect.Ptr + + // Read null flag based on RefMode only (not based on whether local type is pointer) + if field.RefMode != RefModeNone { + nullFlag := buf.ReadInt8(ctx.Err()) + if nullFlag == NullFlag { + // For pointer enum fields, leave as nil; for non-pointer, set to zero + if !isPointer { + fieldValue.SetInt(0) + } + return + } + } + + // For pointer enum fields, allocate a new value + targetValue := fieldValue + if isPointer { + newVal := reflect.New(field.Type.Elem()) + fieldValue.Set(newVal) + targetValue = newVal.Elem() + } + + // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. + // We need to call the inner enumSerializer directly with the dereferenced value. + if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + ptrSer.valueSerializer.ReadData(ctx, field.Type.Elem(), targetValue) + } else { + field.Serializer.ReadData(ctx, field.Type, targetValue) + } } type structSerializer struct { - // Identity - typeTag string - type_ reflect.Type - structHash int32 - - // Pre-sorted field lists by category (computed at init) - fixedFields []*FieldInfo // fixed-size primitives (bool, int8, int16, float32, float64) - varintFields []*FieldInfo // varint primitives (int32, int64, int) - remainingFields []*FieldInfo // all other fields (string, slice, map, struct, etc.) - - // All fields in protocol order (for compatible mode) - fields []*FieldInfo // all fields in sorted order - fieldMap map[string]*FieldInfo // for compatible reading - fieldDefs []FieldDef // for type_def compatibility - - // Pre-computed buffer sizes - fixedSize int // Total bytes for fixed-size primitives - maxVarintSize int // Max bytes for varints (5 per int32, 10 per int64) - - // Mode flags (set at init) - isCompatibleMode bool // true when compatible=true - typeDefDiffers bool // true when compatible=true AND remote TypeDef != local (requires ordered read) - - // Initialization state - initialized bool + // Identity + typeTag string + type_ reflect.Type + structHash int32 + + // Pre-sorted field lists by category (computed at init) + fixedFields []*FieldInfo // fixed-size primitives (bool, int8, int16, float32, float64) + varintFields []*FieldInfo // varint primitives (int32, int64, int) + remainingFields []*FieldInfo // all other fields (string, slice, map, struct, etc.) + + // All fields in protocol order (for compatible mode) + fields []*FieldInfo // all fields in sorted order + fieldMap map[string]*FieldInfo // for compatible reading + fieldDefs []FieldDef // for type_def compatibility + + // Pre-computed buffer sizes + fixedSize int // Total bytes for fixed-size primitives + maxVarintSize int // Max bytes for varints (5 per int32, 10 per int64) + + // Mode flags (set at init) + isCompatibleMode bool // true when compatible=true + typeDefDiffers bool // true when compatible=true AND remote TypeDef != local (requires ordered read) + + // Initialization state + initialized bool } // newStructSerializer creates a new structSerializer with the given parameters. // typeTag can be empty and will be derived from type_.Name() if not provided. // fieldDefs can be nil for local structs without remote schema. func newStructSerializer(type_ reflect.Type, typeTag string, fieldDefs []FieldDef) *structSerializer { - if typeTag == "" && type_ != nil { - typeTag = type_.Name() - } - return &structSerializer{ - type_: type_, - typeTag: typeTag, - fieldDefs: fieldDefs, - } + if typeTag == "" && type_ != nil { + typeTag = type_.Name() + } + return &structSerializer{ + type_: type_, + typeTag: typeTag, + fieldDefs: fieldDefs, + } } // initialize performs eager initialization of the struct serializer. // This should be called at registration time to pre-compute all field metadata. func (s *structSerializer) initialize(typeResolver *TypeResolver) error { - if s.initialized { - return nil - } - - // Ensure type is set - if s.type_ == nil { - return errors.New("struct type not set") - } - - // Normalize pointer types - for s.type_.Kind() == reflect.Ptr { - s.type_ = s.type_.Elem() - } - - // Build fields from type or fieldDefs - if s.fieldDefs != nil { - if err := s.initFieldsFromDefsWithResolver(typeResolver); err != nil { - return err - } - } else { - if err := s.initFieldsFromTypeResolver(typeResolver); err != nil { - return err - } - } - - // Compute struct hash - s.structHash = s.computeHash() - - // Set compatible mode flag - s.isCompatibleMode = typeResolver.Compatible() - - s.initialized = true - return nil + if s.initialized { + return nil + } + + // Ensure type is set + if s.type_ == nil { + return errors.New("struct type not set") + } + + // Normalize pointer types + for s.type_.Kind() == reflect.Ptr { + s.type_ = s.type_.Elem() + } + + // Build fields from type or fieldDefs + if s.fieldDefs != nil { + if err := s.initFieldsFromDefsWithResolver(typeResolver); err != nil { + return err + } + } else { + if err := s.initFieldsFromTypeResolver(typeResolver); err != nil { + return err + } + } + + // Compute struct hash + s.structHash = s.computeHash() + + // Set compatible mode flag + s.isCompatibleMode = typeResolver.Compatible() + + s.initialized = true + return nil } func (s *structSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - switch refMode { - case RefModeTracking: - if value.Kind() == reflect.Ptr && value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return - } - refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refWritten { - return - } - case RefModeNullOnly: - if value.Kind() == reflect.Ptr && value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return - } - ctx.buffer.WriteInt8(NotNullValueFlag) - } - if writeType { - // Structs have dynamic type IDs, need to look up from TypeResolver - typeInfo, err := ctx.TypeResolver().getTypeInfo(value, true) - if err != nil { - ctx.SetError(FromError(err)) - return - } - ctx.TypeResolver().WriteTypeInfo(ctx.buffer, typeInfo, ctx.Err()) - } - s.WriteData(ctx, value) + switch refMode { + case RefModeTracking: + if value.Kind() == reflect.Ptr && value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return + } + refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) + if err != nil { + ctx.SetError(FromError(err)) + return + } + if refWritten { + return + } + case RefModeNullOnly: + if value.Kind() == reflect.Ptr && value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return + } + ctx.buffer.WriteInt8(NotNullValueFlag) + } + if writeType { + // Structs have dynamic type IDs, need to look up from TypeResolver + typeInfo, err := ctx.TypeResolver().getTypeInfo(value, true) + if err != nil { + ctx.SetError(FromError(err)) + return + } + ctx.TypeResolver().WriteTypeInfo(ctx.buffer, typeInfo, ctx.Err()) + } + s.WriteData(ctx, value) } func (s *structSerializer) WriteData(ctx *WriteContext, value reflect.Value) { - // Early error check - skip all intermediate checks for normal path performance - if ctx.HasError() { - return - } - - // Lazy initialization - if !s.initialized { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] structSerializer.WriteData: calling initialize for type=%v\n", s.type_) - } - if err := s.initialize(ctx.TypeResolver()); err != nil { - ctx.SetError(FromError(err)) - return - } - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] structSerializer.WriteData: after initialize, remainingFields=%d\n", len(s.remainingFields)) - } - } - - buf := ctx.Buffer() - - // Dereference pointer if needed - if value.Kind() == reflect.Ptr { - if value.IsNil() { - ctx.SetError(SerializationError("cannot write nil pointer")) - return - } - value = value.Elem() - } - - // In compatible mode with meta share, struct hash is not written - if !ctx.Compatible() { - buf.WriteInt32(s.structHash) - } - - // Check if value is addressable for unsafe access - canUseUnsafe := value.CanAddr() - var ptr unsafe.Pointer - if canUseUnsafe { - ptr = unsafe.Pointer(value.UnsafeAddr()) - } - - // ========================================================================== - // Phase 1: Fixed-size primitives (bool, int8, int16, float32, float64) - // - Reserve once, inline unsafe writes with endian handling, update index once - // - field.WriteOffset computed at init time - // ========================================================================== - if canUseUnsafe && s.fixedSize > 0 { - buf.Reserve(s.fixedSize) - baseOffset := buf.WriterIndex() - data := buf.GetData() - - for _, field := range s.fixedFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - bufOffset := baseOffset + field.WriteOffset - switch field.StaticId { - case ConcreteTypeBool: - if *(*bool)(fieldPtr) { - data[bufOffset] = 1 - } else { - data[bufOffset] = 0 - } - case ConcreteTypeInt8: - data[bufOffset] = *(*byte)(fieldPtr) - case ConcreteTypeInt16: - if isLittleEndian { - *(*int16)(unsafe.Pointer(&data[bufOffset])) = *(*int16)(fieldPtr) - } else { - binary.LittleEndian.PutUint16(data[bufOffset:], uint16(*(*int16)(fieldPtr))) - } - case ConcreteTypeFloat32: - if isLittleEndian { - *(*float32)(unsafe.Pointer(&data[bufOffset])) = *(*float32)(fieldPtr) - } else { - binary.LittleEndian.PutUint32(data[bufOffset:], math.Float32bits(*(*float32)(fieldPtr))) - } - case ConcreteTypeFloat64: - if isLittleEndian { - *(*float64)(unsafe.Pointer(&data[bufOffset])) = *(*float64)(fieldPtr) - } else { - binary.LittleEndian.PutUint64(data[bufOffset:], math.Float64bits(*(*float64)(fieldPtr))) - } - } - } - // Update writer index ONCE after all fixed fields - buf.SetWriterIndex(baseOffset + s.fixedSize) - } else if len(s.fixedFields) > 0 { - // Fallback to reflect-based access for unaddressable values - for _, field := range s.fixedFields { - fieldValue := value.Field(field.FieldIndex) - switch field.StaticId { - case ConcreteTypeBool: - buf.WriteBool(fieldValue.Bool()) - case ConcreteTypeInt8: - buf.WriteByte_(byte(fieldValue.Int())) - case ConcreteTypeInt16: - buf.WriteInt16(int16(fieldValue.Int())) - case ConcreteTypeFloat32: - buf.WriteFloat32(float32(fieldValue.Float())) - case ConcreteTypeFloat64: - buf.WriteFloat64(fieldValue.Float()) - } - } - } - - // ========================================================================== - // Phase 2: Varint primitives (int32, int64, int) - // - Reserve max size, track offset locally, update index once at end - // ========================================================================== - if canUseUnsafe && s.maxVarintSize > 0 { - buf.Reserve(s.maxVarintSize) - offset := buf.WriterIndex() - - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - offset += buf.UnsafePutVarInt32(offset, *(*int32)(fieldPtr)) - case ConcreteTypeInt64: - offset += buf.UnsafePutVarInt64(offset, *(*int64)(fieldPtr)) - case ConcreteTypeInt: - offset += buf.UnsafePutVarInt64(offset, int64(*(*int)(fieldPtr))) - } - } - // Update writer index ONCE after all varint fields - buf.SetWriterIndex(offset) - } else if len(s.varintFields) > 0 { - // Fallback to reflect-based access for unaddressable values - for _, field := range s.varintFields { - fieldValue := value.Field(field.FieldIndex) - switch field.StaticId { - case ConcreteTypeInt32: - buf.WriteVarint32(int32(fieldValue.Int())) - case ConcreteTypeInt64, ConcreteTypeInt: - buf.WriteVarint64(fieldValue.Int()) - } - } - } - - // ========================================================================== - // Phase 3: Remaining fields (strings, slices, maps, structs, enums) - // - These require per-field handling (ref flags, type info, serializers) - // - No intermediate error checks - trade error path performance for normal path - // ========================================================================== - for _, field := range s.remainingFields { - s.writeRemainingField(ctx, ptr, field, value) - } + // Early error check - skip all intermediate checks for normal path performance + if ctx.HasError() { + return + } + + // Lazy initialization + if !s.initialized { + if err := s.initialize(ctx.TypeResolver()); err != nil { + ctx.SetError(FromError(err)) + return + } + } + + buf := ctx.Buffer() + + // Dereference pointer if needed + if value.Kind() == reflect.Ptr { + if value.IsNil() { + ctx.SetError(SerializationError("cannot write nil pointer")) + return + } + value = value.Elem() + } + + // In compatible mode with meta share, struct hash is not written + if !ctx.Compatible() { + buf.WriteInt32(s.structHash) + } + + // Check if value is addressable for unsafe access + canUseUnsafe := value.CanAddr() + var ptr unsafe.Pointer + if canUseUnsafe { + ptr = unsafe.Pointer(value.UnsafeAddr()) + } + + // ========================================================================== + // Phase 1: Fixed-size primitives (bool, int8, int16, float32, float64) + // - Reserve once, inline unsafe writes with endian handling, update index once + // - field.WriteOffset computed at init time + // ========================================================================== + if canUseUnsafe && s.fixedSize > 0 { + buf.Reserve(s.fixedSize) + baseOffset := buf.WriterIndex() + data := buf.GetData() + + for _, field := range s.fixedFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + bufOffset := baseOffset + field.WriteOffset + switch field.StaticId { + case ConcreteTypeBool: + if *(*bool)(fieldPtr) { + data[bufOffset] = 1 + } else { + data[bufOffset] = 0 + } + case ConcreteTypeInt8: + data[bufOffset] = *(*byte)(fieldPtr) + case ConcreteTypeInt16: + if isLittleEndian { + *(*int16)(unsafe.Pointer(&data[bufOffset])) = *(*int16)(fieldPtr) + } else { + binary.LittleEndian.PutUint16(data[bufOffset:], uint16(*(*int16)(fieldPtr))) + } + case ConcreteTypeFloat32: + if isLittleEndian { + *(*float32)(unsafe.Pointer(&data[bufOffset])) = *(*float32)(fieldPtr) + } else { + binary.LittleEndian.PutUint32(data[bufOffset:], math.Float32bits(*(*float32)(fieldPtr))) + } + case ConcreteTypeFloat64: + if isLittleEndian { + *(*float64)(unsafe.Pointer(&data[bufOffset])) = *(*float64)(fieldPtr) + } else { + binary.LittleEndian.PutUint64(data[bufOffset:], math.Float64bits(*(*float64)(fieldPtr))) + } + } + } + // Update writer index ONCE after all fixed fields + buf.SetWriterIndex(baseOffset + s.fixedSize) + } else if len(s.fixedFields) > 0 { + // Fallback to reflect-based access for unaddressable values + for _, field := range s.fixedFields { + fieldValue := value.Field(field.FieldIndex) + switch field.StaticId { + case ConcreteTypeBool: + buf.WriteBool(fieldValue.Bool()) + case ConcreteTypeInt8: + buf.WriteByte_(byte(fieldValue.Int())) + case ConcreteTypeInt16: + buf.WriteInt16(int16(fieldValue.Int())) + case ConcreteTypeFloat32: + buf.WriteFloat32(float32(fieldValue.Float())) + case ConcreteTypeFloat64: + buf.WriteFloat64(fieldValue.Float()) + } + } + } + + // ========================================================================== + // Phase 2: Varint primitives (int32, int64, int) + // - Reserve max size, track offset locally, update index once at end + // ========================================================================== + if canUseUnsafe && s.maxVarintSize > 0 { + buf.Reserve(s.maxVarintSize) + offset := buf.WriterIndex() + + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + offset += buf.UnsafePutVarInt32(offset, *(*int32)(fieldPtr)) + case ConcreteTypeInt64: + offset += buf.UnsafePutVarInt64(offset, *(*int64)(fieldPtr)) + case ConcreteTypeInt: + offset += buf.UnsafePutVarInt64(offset, int64(*(*int)(fieldPtr))) + } + } + // Update writer index ONCE after all varint fields + buf.SetWriterIndex(offset) + } else if len(s.varintFields) > 0 { + // Fallback to reflect-based access for unaddressable values + for _, field := range s.varintFields { + fieldValue := value.Field(field.FieldIndex) + switch field.StaticId { + case ConcreteTypeInt32: + buf.WriteVarint32(int32(fieldValue.Int())) + case ConcreteTypeInt64, ConcreteTypeInt: + buf.WriteVarint64(fieldValue.Int()) + } + } + } + + // ========================================================================== + // Phase 3: Remaining fields (strings, slices, maps, structs, enums) + // - These require per-field handling (ref flags, type info, serializers) + // - No intermediate error checks - trade error path performance for normal path + // ========================================================================== + for _, field := range s.remainingFields { + s.writeRemainingField(ctx, ptr, field, value) + } } // writeRemainingField writes a non-primitive field (string, slice, map, struct, enum) func (s *structSerializer) writeRemainingField(ctx *WriteContext, ptr unsafe.Pointer, field *FieldInfo, value reflect.Value) { - buf := ctx.Buffer() - - // Fast path dispatch using pre-computed StaticId - // ptr must be valid (addressable value) - if ptr != nil { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeString: - if field.RefMode == RefModeTracking { - break // Fall through to slow path - } - // Only write null flag if RefMode requires it (nullable field) - if field.RefMode == RefModeNullOnly { - buf.WriteInt8(NotNullValueFlag) - } - ctx.WriteString(*(*string)(fieldPtr)) - return - case ConcreteTypeEnum: - // Enums don't track refs - always use fast path - writeEnumField(ctx, field, value.Field(field.FieldIndex)) - return - case ConcreteTypeStringSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringSlice(*(*[]string)(fieldPtr), field.RefMode, false, true) - return - case ConcreteTypeBoolSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteBoolSlice(*(*[]bool)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt8Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt8Slice(*(*[]int8)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeByteSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteByteSlice(*(*[]byte)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt16Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt16Slice(*(*[]int16)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt32Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt32Slice(*(*[]int32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt64Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt64Slice(*(*[]int64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeIntSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteIntSlice(*(*[]int)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeUintSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteUintSlice(*(*[]uint)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeFloat32Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteFloat32Slice(*(*[]float32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeFloat64Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteFloat64Slice(*(*[]float64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringStringMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringStringMap(*(*map[string]string)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringInt64Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringInt64Map(*(*map[string]int64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringInt32Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringInt32Map(*(*map[string]int32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringIntMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringIntMap(*(*map[string]int)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringFloat64Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringFloat64Map(*(*map[string]float64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringBoolMap: - // NOTE: map[string]bool is used to represent SETs in Go xlang mode. - // We CANNOT use the fast path here because it writes MAP format, - // but the data should be written in SET format. Fall through to slow path - // which uses setSerializer to correctly write the SET format. - break - case ConcreteTypeIntIntMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteIntIntMap(*(*map[int]int)(fieldPtr), field.RefMode, false) - return - } - } - - // Slow path: use full serializer - fieldValue := value.Field(field.FieldIndex) - if field.Serializer != nil { - field.Serializer.Write(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.WriteValue(fieldValue) - } + buf := ctx.Buffer() + + // Fast path dispatch using pre-computed StaticId + // ptr must be valid (addressable value) + if ptr != nil { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeString: + if field.RefMode == RefModeTracking { + break // Fall through to slow path + } + // Only write null flag if RefMode requires it (nullable field) + if field.RefMode == RefModeNullOnly { + buf.WriteInt8(NotNullValueFlag) + } + ctx.WriteString(*(*string)(fieldPtr)) + return + case ConcreteTypeEnum: + // Enums don't track refs - always use fast path + writeEnumField(ctx, field, value.Field(field.FieldIndex)) + return + case ConcreteTypeStringSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringSlice(*(*[]string)(fieldPtr), field.RefMode, false, true) + return + case ConcreteTypeBoolSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteBoolSlice(*(*[]bool)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt8Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt8Slice(*(*[]int8)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeByteSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteByteSlice(*(*[]byte)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt16Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt16Slice(*(*[]int16)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt32Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt32Slice(*(*[]int32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt64Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt64Slice(*(*[]int64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeIntSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteIntSlice(*(*[]int)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeUintSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteUintSlice(*(*[]uint)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeFloat32Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteFloat32Slice(*(*[]float32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeFloat64Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteFloat64Slice(*(*[]float64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringStringMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringStringMap(*(*map[string]string)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringInt64Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringInt64Map(*(*map[string]int64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringInt32Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringInt32Map(*(*map[string]int32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringIntMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringIntMap(*(*map[string]int)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringFloat64Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringFloat64Map(*(*map[string]float64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringBoolMap: + // NOTE: map[string]bool is used to represent SETs in Go xlang mode. + // We CANNOT use the fast path here because it writes MAP format, + // but the data should be written in SET format. Fall through to slow path + // which uses setSerializer to correctly write the SET format. + break + case ConcreteTypeIntIntMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteIntIntMap(*(*map[int]int)(fieldPtr), field.RefMode, false) + return + } + } + + // Slow path: use full serializer + fieldValue := value.Field(field.FieldIndex) + if field.Serializer != nil { + field.Serializer.Write(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.WriteValue(fieldValue, RefModeTracking, true) + } } func (s *structSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found - obj := ctx.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) - } - return - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return - } - } - if readType { - // Read type info - in compatible mode this returns the serializer with remote fieldDefs - typeID := buf.ReadVaruint32Small7(ctxErr) - internalTypeID := TypeId(typeID & 0xFF) - // Check if this is a struct type that needs type meta reading - if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { - // For struct types in compatible mode, use the serializer from TypeInfo - typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) - // Use the serializer from TypeInfo which has the remote field definitions - if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { - structSer.ReadData(ctx, value.Type(), value) - return - } - } - } - s.ReadData(ctx, value.Type(), value) + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) + if refErr != nil { + ctx.SetError(FromError(refErr)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found + obj := ctx.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return + } + } + if readType { + // Read type info - in compatible mode this returns the serializer with remote fieldDefs + typeID := buf.ReadVaruint32Small7(ctxErr) + internalTypeID := TypeId(typeID & 0xFF) + // Check if this is a struct type that needs type meta reading + if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { + // For struct types in compatible mode, use the serializer from TypeInfo + typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) + // Use the serializer from TypeInfo which has the remote field definitions + if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { + structSer.ReadData(ctx, value.Type(), value) + return + } + } + } + s.ReadData(ctx, value.Type(), value) } func (s *structSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { - // Early error check - skip all intermediate checks for normal path performance - if ctx.HasError() { - return - } - - // Lazy initialization - if !s.initialized { - if err := s.initialize(ctx.TypeResolver()); err != nil { - ctx.SetError(FromError(err)) - return - } - } - - buf := ctx.Buffer() - if value.Kind() == reflect.Ptr { - if value.IsNil() { - value.Set(reflect.New(type_.Elem())) - } - value = value.Elem() - type_ = type_.Elem() - } - - // In compatible mode with meta share, struct hash is not written - if !ctx.Compatible() { - err := ctx.Err() - structHash := buf.ReadInt32(err) - if structHash != s.structHash { - ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) - return - } - } - - // Use ordered reading only when TypeDef differs from local type (schema evolution) - // When types match (typeDefDiffers=false), use grouped reading for better performance - if s.typeDefDiffers { - s.readFieldsInOrder(ctx, value) - return - } - - // Check if value is addressable for unsafe access - if !value.CanAddr() { - s.readFieldsInOrder(ctx, value) - return - } - - // ========================================================================== - // Grouped reading for matching types (optimized path) - // - Types match, so all fields exist locally (no FieldIndex < 0 checks) - // - Use UnsafeGet at pre-computed offsets, update reader index once per phase - // ========================================================================== - ptr := unsafe.Pointer(value.UnsafeAddr()) - - // Phase 1: Fixed-size primitives (inline unsafe reads with endian handling) - if s.fixedSize > 0 { - baseOffset := buf.ReaderIndex() - data := buf.GetData() - - for _, field := range s.fixedFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - bufOffset := baseOffset + field.WriteOffset - switch field.StaticId { - case ConcreteTypeBool: - *(*bool)(fieldPtr) = data[bufOffset] != 0 - case ConcreteTypeInt8: - *(*int8)(fieldPtr) = int8(data[bufOffset]) - case ConcreteTypeInt16: - if isLittleEndian { - *(*int16)(fieldPtr) = *(*int16)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*int16)(fieldPtr) = int16(binary.LittleEndian.Uint16(data[bufOffset:])) - } - case ConcreteTypeFloat32: - if isLittleEndian { - *(*float32)(fieldPtr) = *(*float32)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*float32)(fieldPtr) = math.Float32frombits(binary.LittleEndian.Uint32(data[bufOffset:])) - } - case ConcreteTypeFloat64: - if isLittleEndian { - *(*float64)(fieldPtr) = *(*float64)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*float64)(fieldPtr) = math.Float64frombits(binary.LittleEndian.Uint64(data[bufOffset:])) - } - } - } - // Update reader index ONCE after all fixed fields - buf.SetReaderIndex(baseOffset + s.fixedSize) - } - - // Phase 2: Varint primitives (must read sequentially - variable length) - // Use unsafe reads when we have enough buffer remaining - if s.maxVarintSize > 0 && buf.remaining() >= s.maxVarintSize { - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.UnsafeReadVarint32() - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.UnsafeReadVarint64() - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.UnsafeReadVarint64()) - } - } - } else if len(s.varintFields) > 0 { - // Slow path with bounds checking - err := ctx.Err() - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.ReadVarint32(err) - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.ReadVarint64(err) - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) - } - } - } - - // Phase 3: Remaining fields (strings, slices, maps, structs, enums) - // No intermediate error checks - trade error path performance for normal path - for _, field := range s.remainingFields { - s.readRemainingField(ctx, ptr, field, value) - } + // Early error check - skip all intermediate checks for normal path performance + if ctx.HasError() { + return + } + + // Lazy initialization + if !s.initialized { + if err := s.initialize(ctx.TypeResolver()); err != nil { + ctx.SetError(FromError(err)) + return + } + } + + buf := ctx.Buffer() + if value.Kind() == reflect.Ptr { + if value.IsNil() { + value.Set(reflect.New(type_.Elem())) + } + value = value.Elem() + type_ = type_.Elem() + } + + // In compatible mode with meta share, struct hash is not written + if !ctx.Compatible() { + err := ctx.Err() + structHash := buf.ReadInt32(err) + if structHash != s.structHash { + ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) + return + } + } + + // Use ordered reading only when TypeDef differs from local type (schema evolution) + // When types match (typeDefDiffers=false), use grouped reading for better performance + if s.typeDefDiffers { + s.readFieldsInOrder(ctx, value) + return + } + + // Check if value is addressable for unsafe access + if !value.CanAddr() { + s.readFieldsInOrder(ctx, value) + return + } + + // ========================================================================== + // Grouped reading for matching types (optimized path) + // - Types match, so all fields exist locally (no FieldIndex < 0 checks) + // - Use UnsafeGet at pre-computed offsets, update reader index once per phase + // ========================================================================== + ptr := unsafe.Pointer(value.UnsafeAddr()) + + // Phase 1: Fixed-size primitives (inline unsafe reads with endian handling) + if s.fixedSize > 0 { + baseOffset := buf.ReaderIndex() + data := buf.GetData() + + for _, field := range s.fixedFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + bufOffset := baseOffset + field.WriteOffset + switch field.StaticId { + case ConcreteTypeBool: + *(*bool)(fieldPtr) = data[bufOffset] != 0 + case ConcreteTypeInt8: + *(*int8)(fieldPtr) = int8(data[bufOffset]) + case ConcreteTypeInt16: + if isLittleEndian { + *(*int16)(fieldPtr) = *(*int16)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*int16)(fieldPtr) = int16(binary.LittleEndian.Uint16(data[bufOffset:])) + } + case ConcreteTypeFloat32: + if isLittleEndian { + *(*float32)(fieldPtr) = *(*float32)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*float32)(fieldPtr) = math.Float32frombits(binary.LittleEndian.Uint32(data[bufOffset:])) + } + case ConcreteTypeFloat64: + if isLittleEndian { + *(*float64)(fieldPtr) = *(*float64)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*float64)(fieldPtr) = math.Float64frombits(binary.LittleEndian.Uint64(data[bufOffset:])) + } + } + } + // Update reader index ONCE after all fixed fields + buf.SetReaderIndex(baseOffset + s.fixedSize) + } + + // Phase 2: Varint primitives (must read sequentially - variable length) + // Use unsafe reads when we have enough buffer remaining + if s.maxVarintSize > 0 && buf.remaining() >= s.maxVarintSize { + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.UnsafeReadVarint32() + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.UnsafeReadVarint64() + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.UnsafeReadVarint64()) + } + } + } else if len(s.varintFields) > 0 { + // Slow path with bounds checking + err := ctx.Err() + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.ReadVarint32(err) + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.ReadVarint64(err) + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) + } + } + } + + // Phase 3: Remaining fields (strings, slices, maps, structs, enums) + // No intermediate error checks - trade error path performance for normal path + for _, field := range s.remainingFields { + s.readRemainingField(ctx, ptr, field, value) + } } // readRemainingField reads a non-primitive field (string, slice, map, struct, enum) func (s *structSerializer) readRemainingField(ctx *ReadContext, ptr unsafe.Pointer, field *FieldInfo, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - - // Fast path dispatch using pre-computed StaticId - // ptr must be valid (addressable value) - if ptr != nil { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeString: - if field.RefMode == RefModeTracking { - break // Fall through to slow path for ref tracking - } - // Only read null flag if RefMode requires it (nullable field) - if field.RefMode == RefModeNullOnly { - refFlag := buf.ReadInt8(ctxErr) - if refFlag == NullFlag { - *(*string)(fieldPtr) = "" - return - } - } - *(*string)(fieldPtr) = ctx.ReadString() - return - case ConcreteTypeEnum: - // Enums don't track refs - always use fast path - fieldValue := value.Field(field.FieldIndex) - readEnumField(ctx, field, fieldValue) - return - case ConcreteTypeStringSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]string)(fieldPtr) = ctx.ReadStringSlice(field.RefMode, false) - return - case ConcreteTypeBoolSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]bool)(fieldPtr) = ctx.ReadBoolSlice(field.RefMode, false) - return - case ConcreteTypeInt8Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int8)(fieldPtr) = ctx.ReadInt8Slice(field.RefMode, false) - return - case ConcreteTypeByteSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]byte)(fieldPtr) = ctx.ReadByteSlice(field.RefMode, false) - return - case ConcreteTypeInt16Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int16)(fieldPtr) = ctx.ReadInt16Slice(field.RefMode, false) - return - case ConcreteTypeInt32Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int32)(fieldPtr) = ctx.ReadInt32Slice(field.RefMode, false) - return - case ConcreteTypeInt64Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int64)(fieldPtr) = ctx.ReadInt64Slice(field.RefMode, false) - return - case ConcreteTypeIntSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int)(fieldPtr) = ctx.ReadIntSlice(field.RefMode, false) - return - case ConcreteTypeUintSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]uint)(fieldPtr) = ctx.ReadUintSlice(field.RefMode, false) - return - case ConcreteTypeFloat32Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]float32)(fieldPtr) = ctx.ReadFloat32Slice(field.RefMode, false) - return - case ConcreteTypeFloat64Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]float64)(fieldPtr) = ctx.ReadFloat64Slice(field.RefMode, false) - return - case ConcreteTypeStringStringMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]string)(fieldPtr) = ctx.ReadStringStringMap(field.RefMode, false) - return - case ConcreteTypeStringInt64Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int64)(fieldPtr) = ctx.ReadStringInt64Map(field.RefMode, false) - return - case ConcreteTypeStringInt32Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int32)(fieldPtr) = ctx.ReadStringInt32Map(field.RefMode, false) - return - case ConcreteTypeStringIntMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int)(fieldPtr) = ctx.ReadStringIntMap(field.RefMode, false) - return - case ConcreteTypeStringFloat64Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]float64)(fieldPtr) = ctx.ReadStringFloat64Map(field.RefMode, false) - return - case ConcreteTypeStringBoolMap: - // NOTE: map[string]bool is used to represent SETs in Go xlang mode. - // We CANNOT use the fast path here because it reads MAP format, - // but the data is actually in SET format. Fall through to slow path - // which uses setSerializer to correctly read the SET format. - break - case ConcreteTypeIntIntMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[int]int)(fieldPtr) = ctx.ReadIntIntMap(field.RefMode, false) - return - } - } - - // Slow path: use full serializer - fieldValue := value.Field(field.FieldIndex) - - if field.Serializer != nil { - field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.ReadValue(fieldValue) - } + buf := ctx.Buffer() + ctxErr := ctx.Err() + + // Fast path dispatch using pre-computed StaticId + // ptr must be valid (addressable value) + if ptr != nil { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeString: + if field.RefMode == RefModeTracking { + break // Fall through to slow path for ref tracking + } + // Only read null flag if RefMode requires it (nullable field) + if field.RefMode == RefModeNullOnly { + refFlag := buf.ReadInt8(ctxErr) + if refFlag == NullFlag { + *(*string)(fieldPtr) = "" + return + } + } + *(*string)(fieldPtr) = ctx.ReadString() + return + case ConcreteTypeEnum: + // Enums don't track refs - always use fast path + fieldValue := value.Field(field.FieldIndex) + readEnumField(ctx, field, fieldValue) + return + case ConcreteTypeStringSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]string)(fieldPtr) = ctx.ReadStringSlice(field.RefMode, false) + return + case ConcreteTypeBoolSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]bool)(fieldPtr) = ctx.ReadBoolSlice(field.RefMode, false) + return + case ConcreteTypeInt8Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int8)(fieldPtr) = ctx.ReadInt8Slice(field.RefMode, false) + return + case ConcreteTypeByteSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]byte)(fieldPtr) = ctx.ReadByteSlice(field.RefMode, false) + return + case ConcreteTypeInt16Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int16)(fieldPtr) = ctx.ReadInt16Slice(field.RefMode, false) + return + case ConcreteTypeInt32Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int32)(fieldPtr) = ctx.ReadInt32Slice(field.RefMode, false) + return + case ConcreteTypeInt64Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int64)(fieldPtr) = ctx.ReadInt64Slice(field.RefMode, false) + return + case ConcreteTypeIntSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int)(fieldPtr) = ctx.ReadIntSlice(field.RefMode, false) + return + case ConcreteTypeUintSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]uint)(fieldPtr) = ctx.ReadUintSlice(field.RefMode, false) + return + case ConcreteTypeFloat32Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]float32)(fieldPtr) = ctx.ReadFloat32Slice(field.RefMode, false) + return + case ConcreteTypeFloat64Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]float64)(fieldPtr) = ctx.ReadFloat64Slice(field.RefMode, false) + return + case ConcreteTypeStringStringMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]string)(fieldPtr) = ctx.ReadStringStringMap(field.RefMode, false) + return + case ConcreteTypeStringInt64Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int64)(fieldPtr) = ctx.ReadStringInt64Map(field.RefMode, false) + return + case ConcreteTypeStringInt32Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int32)(fieldPtr) = ctx.ReadStringInt32Map(field.RefMode, false) + return + case ConcreteTypeStringIntMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int)(fieldPtr) = ctx.ReadStringIntMap(field.RefMode, false) + return + case ConcreteTypeStringFloat64Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]float64)(fieldPtr) = ctx.ReadStringFloat64Map(field.RefMode, false) + return + case ConcreteTypeStringBoolMap: + // NOTE: map[string]bool is used to represent SETs in Go xlang mode. + // We CANNOT use the fast path here because it reads MAP format, + // but the data is actually in SET format. Fall through to slow path + // which uses setSerializer to correctly read the SET format. + break + case ConcreteTypeIntIntMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[int]int)(fieldPtr) = ctx.ReadIntIntMap(field.RefMode, false) + return + } + } + + // Slow path: use full serializer + fieldValue := value.Field(field.FieldIndex) + + if field.Serializer != nil { + field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.ReadValue(fieldValue, RefModeTracking, true) + } } // readFieldsInOrder reads fields in the order they appear in s.fields (TypeDef order) // This is used in compatible mode where Java writes fields in TypeDef order func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Value) { - buf := ctx.Buffer() - canUseUnsafe := value.CanAddr() - var ptr unsafe.Pointer - if canUseUnsafe { - ptr = unsafe.Pointer(value.UnsafeAddr()) - } - err := ctx.Err() - - for _, field := range s.fields { - if field.FieldIndex < 0 { - s.skipField(ctx, field) - if ctx.HasError() { - return - } - continue - } - - // Fast path for fixed-size primitive types (no ref flag) - // Use error-aware methods with deferred checking - if canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeBool: - *(*bool)(fieldPtr) = buf.ReadBool(err) - case ConcreteTypeInt8: - *(*int8)(fieldPtr) = buf.ReadInt8(err) - case ConcreteTypeInt16: - *(*int16)(fieldPtr) = buf.ReadInt16(err) - case ConcreteTypeFloat32: - *(*float32)(fieldPtr) = buf.ReadFloat32(err) - case ConcreteTypeFloat64: - *(*float64)(fieldPtr) = buf.ReadFloat64(err) - } - continue - } - - // Fast path for varint primitive types (no ref flag) - // Skip fast path if field has a serializer with a non-primitive type (e.g., NAMED_ENUM) - if canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.ReadVarint32(err) - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.ReadVarint64(err) - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) - } - continue - } - - // Get field value for slow paths - fieldValue := value.Field(field.FieldIndex) - - // Slow path for primitives when not addressable - if !canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { - switch field.StaticId { - case ConcreteTypeBool: - fieldValue.SetBool(buf.ReadBool(err)) - case ConcreteTypeInt8: - fieldValue.SetInt(int64(buf.ReadInt8(err))) - case ConcreteTypeInt16: - fieldValue.SetInt(int64(buf.ReadInt16(err))) - case ConcreteTypeFloat32: - fieldValue.SetFloat(float64(buf.ReadFloat32(err))) - case ConcreteTypeFloat64: - fieldValue.SetFloat(buf.ReadFloat64(err)) - } - continue - } - - if !canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { - switch field.StaticId { - case ConcreteTypeInt32: - fieldValue.SetInt(int64(buf.ReadVarint32(err))) - case ConcreteTypeInt64, ConcreteTypeInt: - fieldValue.SetInt(buf.ReadVarint64(err)) - } - continue - } - - if isEnumField(field) { - readEnumField(ctx, field, fieldValue) - continue - } - - // Slow path for non-primitives (all need ref flag per xlang spec) - if field.Serializer != nil { - // Use pre-computed RefMode and WriteType from field initialization - field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.ReadValue(fieldValue) - } - } + buf := ctx.Buffer() + canUseUnsafe := value.CanAddr() + var ptr unsafe.Pointer + if canUseUnsafe { + ptr = unsafe.Pointer(value.UnsafeAddr()) + } + err := ctx.Err() + + for _, field := range s.fields { + if field.FieldIndex < 0 { + s.skipField(ctx, field) + if ctx.HasError() { + return + } + continue + } + + // Fast path for fixed-size primitive types (no ref flag) + // Use error-aware methods with deferred checking + if canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeBool: + *(*bool)(fieldPtr) = buf.ReadBool(err) + case ConcreteTypeInt8: + *(*int8)(fieldPtr) = buf.ReadInt8(err) + case ConcreteTypeInt16: + *(*int16)(fieldPtr) = buf.ReadInt16(err) + case ConcreteTypeFloat32: + *(*float32)(fieldPtr) = buf.ReadFloat32(err) + case ConcreteTypeFloat64: + *(*float64)(fieldPtr) = buf.ReadFloat64(err) + } + continue + } + + // Fast path for varint primitive types (no ref flag) + // Skip fast path if field has a serializer with a non-primitive type (e.g., NAMED_ENUM) + if canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.ReadVarint32(err) + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.ReadVarint64(err) + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) + } + continue + } + + // Get field value for slow paths + fieldValue := value.Field(field.FieldIndex) + + // Slow path for primitives when not addressable + if !canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { + switch field.StaticId { + case ConcreteTypeBool: + fieldValue.SetBool(buf.ReadBool(err)) + case ConcreteTypeInt8: + fieldValue.SetInt(int64(buf.ReadInt8(err))) + case ConcreteTypeInt16: + fieldValue.SetInt(int64(buf.ReadInt16(err))) + case ConcreteTypeFloat32: + fieldValue.SetFloat(float64(buf.ReadFloat32(err))) + case ConcreteTypeFloat64: + fieldValue.SetFloat(buf.ReadFloat64(err)) + } + continue + } + + if !canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { + switch field.StaticId { + case ConcreteTypeInt32: + fieldValue.SetInt(int64(buf.ReadVarint32(err))) + case ConcreteTypeInt64, ConcreteTypeInt: + fieldValue.SetInt(buf.ReadVarint64(err)) + } + continue + } + + if isEnumField(field) { + readEnumField(ctx, field, fieldValue) + continue + } + + // Slow path for non-primitives (all need ref flag per xlang spec) + if field.Serializer != nil { + // Use pre-computed RefMode and WriteType from field initialization + field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.ReadValue(fieldValue, RefModeTracking, true) + } + } } // skipField skips a field that doesn't exist or is incompatible // Uses context error state for deferred error checking. func (s *structSerializer) skipField(ctx *ReadContext, field *FieldInfo) { - if field.FieldDef.name != "" { - fieldDefIsStructType := isStructFieldType(field.FieldDef.fieldType) - // Use FieldDef's trackingRef and nullable to determine if ref flag was written by Java - // Java writes ref flag based on its FieldDef, not Go's field type - readRefFlag := field.FieldDef.trackingRef || field.FieldDef.nullable - SkipFieldValueWithTypeFlag(ctx, field.FieldDef, readRefFlag, ctx.Compatible() && fieldDefIsStructType) - return - } - // No FieldDef available, read into temp value - tempValue := reflect.New(field.Type).Elem() - if field.Serializer != nil { - readType := ctx.Compatible() && isStructField(field.Type) - refMode := RefModeNone - if field.Referencable { - refMode = RefModeTracking - } - field.Serializer.Read(ctx, refMode, readType, false, tempValue) - } else { - ctx.ReadValue(tempValue) - } + if field.FieldDef.name != "" { + fieldDefIsStructType := isStructFieldType(field.FieldDef.fieldType) + // Use FieldDef's trackingRef and nullable to determine if ref flag was written by Java + // Java writes ref flag based on its FieldDef, not Go's field type + readRefFlag := field.FieldDef.trackingRef || field.FieldDef.nullable + SkipFieldValueWithTypeFlag(ctx, field.FieldDef, readRefFlag, ctx.Compatible() && fieldDefIsStructType) + return + } + // No FieldDef available, read into temp value + tempValue := reflect.New(field.Type).Elem() + if field.Serializer != nil { + readType := ctx.Compatible() && isStructField(field.Type) + refMode := RefModeNone + if field.Referencable { + refMode = RefModeTracking + } + field.Serializer.Read(ctx, refMode, readType, false, tempValue) + } else { + ctx.ReadValue(tempValue, RefModeTracking, true) + } } func (s *structSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again - s.Read(ctx, refMode, false, false, value) + // typeInfo is already read, don't read it again + s.Read(ctx, refMode, false, false, value) } // initFieldsFromContext initializes fields using context's type resolver (for WriteContext) // initFieldsFromTypeResolver initializes fields from local struct type using TypeResolver func (s *structSerializer) initFieldsFromTypeResolver(typeResolver *TypeResolver) error { - // If we have fieldDefs from type_def (remote meta), use them - if len(s.fieldDefs) > 0 { - return s.initFieldsFromDefsWithResolver(typeResolver) - } - - // Otherwise initialize from local struct type - type_ := s.type_ - var fields []*FieldInfo - var fieldNames []string - var serializers []Serializer - var typeIds []TypeId - var nullables []bool - var tagIDs []int - - for i := 0; i < type_.NumField(); i++ { - field := type_.Field(i) - firstRune, _ := utf8.DecodeRuneInString(field.Name) - if unicode.IsLower(firstRune) { - continue // skip unexported fields - } - - // Parse fory struct tag and check for ignore - foryTag := ParseForyTag(field) - if foryTag.Ignore { - continue // skip ignored fields - } - - fieldType := field.Type - - var fieldSerializer Serializer - // For interface{} fields, don't get a serializer - use WriteValue/ReadValue instead - // which will handle polymorphic types dynamically - if fieldType.Kind() != reflect.Interface { - // Get serializer for all non-interface field types - fieldSerializer, _ = typeResolver.getSerializerByType(fieldType, true) - } - - // Use TypeResolver helper methods for arrays and slices - if fieldType.Kind() == reflect.Array && fieldType.Elem().Kind() != reflect.Interface { - fieldSerializer, _ = typeResolver.GetArraySerializer(fieldType) - } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() != reflect.Interface { - fieldSerializer, _ = typeResolver.GetSliceSerializer(fieldType) - } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.Interface { - // For struct fields with interface element types, use sliceDynSerializer - fieldSerializer = mustNewSliceDynSerializer(fieldType.Elem()) - } - - // Get TypeId for the serializer, fallback to deriving from kind - fieldTypeId := typeResolver.getTypeIdByType(fieldType) - if fieldTypeId == 0 { - fieldTypeId = typeIdFromKind(fieldType) - } - // Calculate nullable flag for serialization (wire format): - // - In COMPATIBLE mode: reference types default to nullable=true to match Java's - // wire format where reference types have null flags - // - In SCHEMA_CONSISTENT mode: reference types default to nullable=false to match - // Java's default behavior where reference types are non-nullable unless annotated - // - Primitives (int32, bool, etc.) are always non-nullable - // - Can be overridden by explicit fory tag - // Note: computeHash uses its own nullable calculation for fingerprint matching - internalId := TypeId(fieldTypeId & 0xFF) - isEnum := internalId == ENUM || internalId == NAMED_ENUM - - // Default nullable based on type (reference types are nullable by default) - // Go's codegen always writes null flags for reference types, so reflect must match - // This is consistent with Go's existing behavior and codegen - nullableFlag := fieldType.Kind() == reflect.Ptr || - fieldType.Kind() == reflect.Slice || - fieldType.Kind() == reflect.Map || - fieldType.Kind() == reflect.Interface - if foryTag.NullableSet { - // Override nullable flag if explicitly set in fory tag - nullableFlag = foryTag.Nullable - } - // Primitives are never nullable, regardless of tag - if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum { - nullableFlag = false - } - - // Calculate ref tracking - use tag override if explicitly set - trackRef := typeResolver.TrackRef() - if foryTag.RefSet { - trackRef = foryTag.Ref - } - - // Pre-compute RefMode based on (possibly overridden) trackRef and nullable - // For pointer-to-struct fields, enable ref tracking when trackRef is enabled, - // regardless of nullable flag. This is necessary to detect circular references. - refMode := RefModeNone - isStructPointer := fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.Struct - if trackRef && (nullableFlag || isStructPointer) { - refMode = RefModeTracking - } else if nullableFlag { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(fieldType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(fieldType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] initFieldsFromTypeResolver: field=%s type=%v staticId=%d refMode=%v nullableFlag=%v serializer=%T\n", - SnakeCase(field.Name), fieldType, staticId, refMode, nullableFlag, fieldSerializer) - } - - fieldInfo := &FieldInfo{ - Name: SnakeCase(field.Name), - Offset: field.Offset, - Type: fieldType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: nullableFlag, // Use same logic as TypeDef's nullable flag for consistent ref handling - FieldIndex: i, - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - TagID: foryTag.ID, - HasForyTag: foryTag.HasTag, - TagRefSet: foryTag.RefSet, - TagRef: foryTag.Ref, - TagNullableSet: foryTag.NullableSet, - TagNullable: foryTag.Nullable, - } - fields = append(fields, fieldInfo) - fieldNames = append(fieldNames, fieldInfo.Name) - serializers = append(serializers, fieldSerializer) - typeIds = append(typeIds, fieldTypeId) - nullables = append(nullables, nullableFlag) - tagIDs = append(tagIDs, foryTag.ID) - } - - // Sort fields according to specification using nullable info and tag IDs for consistent ordering - serializers, fieldNames = sortFields(typeResolver, fieldNames, serializers, typeIds, nullables, tagIDs) - order := make(map[string]int, len(fieldNames)) - for idx, name := range fieldNames { - order[name] = idx - } - - sort.SliceStable(fields, func(i, j int) bool { - oi, okI := order[fields[i].Name] - oj, okJ := order[fields[j].Name] - switch { - case okI && okJ: - return oi < oj - case okI: - return true - case okJ: - return false - default: - return false - } - }) - - s.fields = fields - s.groupFields() - return nil + // If we have fieldDefs from type_def (remote meta), use them + if len(s.fieldDefs) > 0 { + return s.initFieldsFromDefsWithResolver(typeResolver) + } + + // Otherwise initialize from local struct type + type_ := s.type_ + var fields []*FieldInfo + var fieldNames []string + var serializers []Serializer + var typeIds []TypeId + var nullables []bool + var tagIDs []int + + for i := 0; i < type_.NumField(); i++ { + field := type_.Field(i) + firstRune, _ := utf8.DecodeRuneInString(field.Name) + if unicode.IsLower(firstRune) { + continue // skip unexported fields + } + + // Parse fory struct tag and check for ignore + foryTag := ParseForyTag(field) + if foryTag.Ignore { + continue // skip ignored fields + } + + fieldType := field.Type + + var fieldSerializer Serializer + // For interface{} fields, don't get a serializer - use WriteValue/ReadValue instead + // which will handle polymorphic types dynamically + if fieldType.Kind() != reflect.Interface { + // Get serializer for all non-interface field types + fieldSerializer, _ = typeResolver.getSerializerByType(fieldType, true) + } + + // Use TypeResolver helper methods for arrays and slices + if fieldType.Kind() == reflect.Array && fieldType.Elem().Kind() != reflect.Interface { + fieldSerializer, _ = typeResolver.GetArraySerializer(fieldType) + } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() != reflect.Interface { + fieldSerializer, _ = typeResolver.GetSliceSerializer(fieldType) + } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.Interface { + // For struct fields with interface element types, use sliceDynSerializer + fieldSerializer = mustNewSliceDynSerializer(fieldType.Elem()) + } + + // Get TypeId for the serializer, fallback to deriving from kind + fieldTypeId := typeResolver.getTypeIdByType(fieldType) + if fieldTypeId == 0 { + fieldTypeId = typeIdFromKind(fieldType) + } + // Calculate nullable flag for serialization (wire format): + // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. + // Only pointer types are nullable by default. + // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, + // so they are nullable by default. + // Can be overridden by explicit fory tag `fory:"nullable"`. + internalId := TypeId(fieldTypeId & 0xFF) + isEnum := internalId == ENUM || internalId == NAMED_ENUM + + // Determine nullable based on mode + // In xlang mode: only pointer types are nullable by default (per xlang spec) + // In native mode: Go's natural semantics - all nil-able types are nullable + // This ensures proper interoperability with Java/other languages in xlang mode. + var nullableFlag bool + if typeResolver.fory.config.IsXlang { + // xlang mode: only pointer types are nullable by default per xlang spec + // Slices and maps are NOT nullable - they serialize as empty when nil + nullableFlag = fieldType.Kind() == reflect.Ptr + } else { + // Native mode: Go's natural semantics - all nil-able types are nullable + nullableFlag = fieldType.Kind() == reflect.Ptr || + fieldType.Kind() == reflect.Slice || + fieldType.Kind() == reflect.Map || + fieldType.Kind() == reflect.Interface + } + if foryTag.NullableSet { + // Override nullable flag if explicitly set in fory tag + nullableFlag = foryTag.Nullable + } + // Primitives are never nullable, regardless of tag + if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum { + nullableFlag = false + } + + // Calculate ref tracking - use tag override if explicitly set + trackRef := typeResolver.TrackRef() + if foryTag.RefSet { + trackRef = foryTag.Ref + } + + // Pre-compute RefMode based on (possibly overridden) trackRef and nullable + // For pointer-to-struct fields, enable ref tracking when trackRef is enabled, + // regardless of nullable flag. This is necessary to detect circular references. + refMode := RefModeNone + isStructPointer := fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.Struct + if trackRef && (nullableFlag || isStructPointer) { + refMode = RefModeTracking + } else if nullableFlag { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(fieldType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(fieldType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] initFieldsFromTypeResolver: field=%s type=%v staticId=%d refMode=%v nullableFlag=%v serializer=%T\n", + SnakeCase(field.Name), fieldType, staticId, refMode, nullableFlag, fieldSerializer) + } + + fieldInfo := &FieldInfo{ + Name: SnakeCase(field.Name), + Offset: field.Offset, + Type: fieldType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: nullableFlag, // Use same logic as TypeDef's nullable flag for consistent ref handling + FieldIndex: i, + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + TagID: foryTag.ID, + HasForyTag: foryTag.HasTag, + TagRefSet: foryTag.RefSet, + TagRef: foryTag.Ref, + TagNullableSet: foryTag.NullableSet, + TagNullable: foryTag.Nullable, + } + fields = append(fields, fieldInfo) + fieldNames = append(fieldNames, fieldInfo.Name) + serializers = append(serializers, fieldSerializer) + typeIds = append(typeIds, fieldTypeId) + nullables = append(nullables, nullableFlag) + tagIDs = append(tagIDs, foryTag.ID) + } + + // Sort fields according to specification using nullable info and tag IDs for consistent ordering + serializers, fieldNames = sortFields(typeResolver, fieldNames, serializers, typeIds, nullables, tagIDs) + order := make(map[string]int, len(fieldNames)) + for idx, name := range fieldNames { + order[name] = idx + } + + sort.SliceStable(fields, func(i, j int) bool { + oi, okI := order[fields[i].Name] + oj, okJ := order[fields[j].Name] + switch { + case okI && okJ: + return oi < oj + case okI: + return true + case okJ: + return false + default: + return false + } + }) + + s.fields = fields + s.groupFields() + return nil } // groupFields categorizes fields into fixedFields, varintFields, and remainingFields. // Also computes pre-computed sizes and WriteOffset for batch buffer reservation. func (s *structSerializer) groupFields() { - s.fixedFields = nil - s.varintFields = nil - s.remainingFields = nil - s.fixedSize = 0 - s.maxVarintSize = 0 - - for _, field := range s.fields { - // Fields with non-primitive serializers (NAMED_ENUM, NAMED_STRUCT, etc.) - // must go to remainingFields to use their serializer's type info writing - hasNonPrimitive := fieldHasNonPrimitiveSerializer(field) - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] groupFields: field=%s TypeId=%d internalId=%d hasNonPrimitive=%v\n", - field.Name, field.TypeId, field.TypeId&0xFF, hasNonPrimitive) - } - if hasNonPrimitive { - s.remainingFields = append(s.remainingFields, field) - } else if isFixedSizePrimitive(field.StaticId, field.Referencable) { - // Compute FixedSize and WriteOffset for this field - field.FixedSize = getFixedSizeByStaticId(field.StaticId) - field.WriteOffset = s.fixedSize - s.fixedSize += field.FixedSize - s.fixedFields = append(s.fixedFields, field) - } else if isVarintPrimitive(field.StaticId, field.Referencable) { - s.maxVarintSize += getVarintMaxSizeByStaticId(field.StaticId) - s.varintFields = append(s.varintFields, field) - } else { - s.remainingFields = append(s.remainingFields, field) - } - } + s.fixedFields = nil + s.varintFields = nil + s.remainingFields = nil + s.fixedSize = 0 + s.maxVarintSize = 0 + + for _, field := range s.fields { + // Fields with non-primitive serializers (NAMED_ENUM, NAMED_STRUCT, etc.) + // must go to remainingFields to use their serializer's type info writing + hasNonPrimitive := fieldHasNonPrimitiveSerializer(field) + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] groupFields: field=%s TypeId=%d internalId=%d hasNonPrimitive=%v\n", + field.Name, field.TypeId, field.TypeId&0xFF, hasNonPrimitive) + } + if hasNonPrimitive { + s.remainingFields = append(s.remainingFields, field) + } else if isFixedSizePrimitive(field.StaticId, field.Referencable) { + // Compute FixedSize and WriteOffset for this field + field.FixedSize = getFixedSizeByStaticId(field.StaticId) + field.WriteOffset = s.fixedSize + s.fixedSize += field.FixedSize + s.fixedFields = append(s.fixedFields, field) + } else if isVarintPrimitive(field.StaticId, field.Referencable) { + s.maxVarintSize += getVarintMaxSizeByStaticId(field.StaticId) + s.varintFields = append(s.varintFields, field) + } else { + s.remainingFields = append(s.remainingFields, field) + } + } } // initFieldsFromDefsWithResolver initializes fields from remote fieldDefs using typeResolver func (s *structSerializer) initFieldsFromDefsWithResolver(typeResolver *TypeResolver) error { - type_ := s.type_ - if type_ == nil { - // Type is not known - we'll create an interface{} placeholder - // This happens when deserializing unknown types in compatible mode - // For now, we'll create fields that discard all data - var fields []*FieldInfo - for _, def := range s.fieldDefs { - fieldSerializer, _ := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - remoteType := remoteTypeInfo.Type - if remoteType == nil { - remoteType = reflect.TypeOf((*interface{})(nil)).Elem() - } - // Get TypeId from FieldType's TypeId method - fieldTypeId := def.fieldType.TypeId() - // Pre-compute RefMode based on trackRef and FieldDef flags - refMode := RefModeNone - if def.trackingRef { - refMode = RefModeTracking - } else if def.nullable { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(remoteType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(remoteType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - - fieldInfo := &FieldInfo{ - Name: def.name, - Offset: 0, - Type: remoteType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: def.nullable, // Use remote nullable flag - FieldIndex: -1, // Mark as non-existent field to discard data - FieldDef: def, // Save original FieldDef for skipping - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - } - fields = append(fields, fieldInfo) - } - s.fields = fields - s.groupFields() - s.typeDefDiffers = true // Unknown type, must use ordered reading - return nil - } - - // Build maps from field names and tag IDs to struct field indices - fieldNameToIndex := make(map[string]int) - fieldNameToOffset := make(map[string]uintptr) - fieldNameToType := make(map[string]reflect.Type) - fieldTagIDToIndex := make(map[int]int) // tag ID -> struct field index - fieldTagIDToOffset := make(map[int]uintptr) // tag ID -> field offset - fieldTagIDToType := make(map[int]reflect.Type) // tag ID -> field type - fieldTagIDToName := make(map[int]string) // tag ID -> snake_case field name - for i := 0; i < type_.NumField(); i++ { - field := type_.Field(i) - - // Parse fory tag and skip ignored fields - foryTag := ParseForyTag(field) - if foryTag.Ignore { - continue - } - - name := SnakeCase(field.Name) - fieldNameToIndex[name] = i - fieldNameToOffset[name] = field.Offset - fieldNameToType[name] = field.Type - - // Also index by tag ID if present - if foryTag.ID >= 0 { - fieldTagIDToIndex[foryTag.ID] = i - fieldTagIDToOffset[foryTag.ID] = field.Offset - fieldTagIDToType[foryTag.ID] = field.Type - fieldTagIDToName[foryTag.ID] = name - } - } - - var fields []*FieldInfo - - for _, def := range s.fieldDefs { - fieldSerializer, err := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) - if err != nil || fieldSerializer == nil { - // If we can't get serializer from typeID, try to get it from the Go type - // This can happen when the type isn't registered in typeIDToTypeInfo - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - if remoteTypeInfo.Type != nil { - fieldSerializer, _ = typeResolver.getSerializerByType(remoteTypeInfo.Type, true) - } - } - - // Get the remote type from fieldDef - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - remoteType := remoteTypeInfo.Type - // Track if type lookup failed - we'll need to skip such fields - // Note: DynamicFieldType.getTypeInfoWithResolver returns interface{} (not nil) when lookup fails - emptyInterfaceType := reflect.TypeOf((*interface{})(nil)).Elem() - typeLookupFailed := remoteType == nil || remoteType == emptyInterfaceType - if remoteType == nil { - remoteType = emptyInterfaceType - } - - // For struct-like fields, even if TypeDef lookup fails, we can try to read - // the field because type resolution happens at read time from the buffer. - // The type name might map to a different local type. - isStructLikeField := isStructFieldType(def.fieldType) - - // Try to find corresponding local field - // First try to match by tag ID (if remote def uses tag ID) - // Then fall back to matching by field name - fieldIndex := -1 - var offset uintptr - var fieldType reflect.Type - var localFieldName string - var localType reflect.Type - var exists bool - - if def.tagID >= 0 { - // Try to match by tag ID - if idx, ok := fieldTagIDToIndex[def.tagID]; ok { - exists = true - fieldIndex = idx // Will be overwritten if types are compatible - localType = fieldTagIDToType[def.tagID] - offset = fieldTagIDToOffset[def.tagID] - localFieldName = fieldTagIDToName[def.tagID] - _ = fieldIndex // Use to avoid compiler warning, will be set properly below - } - } - - // Fall back to name-based matching if tag ID match failed - if !exists && def.name != "" { - if idx, ok := fieldNameToIndex[def.name]; ok { - exists = true - localType = fieldNameToType[def.name] - offset = fieldNameToOffset[def.name] - localFieldName = def.name - _ = idx // Will be set properly below - } - } - - if exists { - idx := fieldNameToIndex[localFieldName] - if def.tagID >= 0 { - idx = fieldTagIDToIndex[def.tagID] - } - // Check if types are compatible - // For primitive types: skip if types don't match - // For struct-like types: allow read even if TypeDef lookup failed, - // because runtime type resolution by name might work - shouldRead := false - isPolymorphicField := def.fieldType.TypeId() == UNKNOWN - defTypeId := def.fieldType.TypeId() - // Check if field is an enum - either by type ID or by serializer type - // The type ID may be a composite value with namespace bits, so check the low 8 bits - internalDefTypeId := defTypeId & 0xFF - isEnumField := internalDefTypeId == NAMED_ENUM || internalDefTypeId == ENUM - if !isEnumField && fieldSerializer != nil { - _, isEnumField = fieldSerializer.(*enumSerializer) - } - if isPolymorphicField && localType.Kind() == reflect.Interface { - // For polymorphic (UNKNOWN) fields with interface{} local type, - // allow reading - the actual type will be determined at runtime - shouldRead = true - fieldType = localType - } else if typeLookupFailed && isEnumField { - // For enum fields with failed TypeDef lookup (NAMED_ENUM stores by namespace/typename, not typeId), - // check if local field is a numeric type (Go enums are int-based) - // Also handle pointer enum fields (*EnumType) - localKind := localType.Kind() - elemKind := localKind - if localKind == reflect.Ptr { - elemKind = localType.Elem().Kind() - } - if isNumericKind(elemKind) { - shouldRead = true - fieldType = localType - // Get the serializer for the base type (the enum type, not the pointer) - baseType := localType - if localKind == reflect.Ptr { - baseType = localType.Elem() - } - fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) - } - } else if typeLookupFailed && isStructLikeField { - // For struct fields with failed TypeDef lookup, check if local field can hold a struct - localKind := localType.Kind() - if localKind == reflect.Ptr { - localKind = localType.Elem().Kind() - } - if localKind == reflect.Struct || localKind == reflect.Interface { - shouldRead = true - fieldType = localType // Use local type for struct fields - } - } else if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) { - // For collection fields with failed type lookup (e.g., List with interface element type), - // check if local type is a slice with interface element type (e.g., []Animal) - // The type lookup fails because sliceConcreteValueSerializer doesn't support interface elements - if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { - shouldRead = true - fieldType = localType - } - } else if !typeLookupFailed && typesCompatible(localType, remoteType) { - shouldRead = true - fieldType = localType - } - - if shouldRead { - fieldIndex = idx - // offset was already set above when matching by tag ID or field name - // For struct-like fields with failed type lookup, get the serializer for the local type - if typeLookupFailed && isStructLikeField && fieldSerializer == nil { - fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) - } - // For collection fields with interface element types, use sliceDynSerializer - if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) && fieldSerializer == nil { - if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { - fieldSerializer = mustNewSliceDynSerializer(localType.Elem()) - } - } - // If local type is *T and remote type is T, we need the serializer for *T - // This handles Java's Integer/Long (nullable boxed types) mapping to Go's *int32/*int64 - if localType.Kind() == reflect.Ptr && localType.Elem() == remoteType { - fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) - } - // For pointer enum fields (*EnumType), get the serializer for the base enum type - // The struct read/write code will handle pointer dereferencing - if isEnumField && localType.Kind() == reflect.Ptr { - baseType := localType.Elem() - fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] pointer enum field %s: localType=%v baseType=%v serializer=%T\n", - def.name, localType, baseType, fieldSerializer) - } - } - // For array fields, use array serializers (not slice serializers) even if typeID maps to slice serializer - // The typeID (INT16_ARRAY, etc.) is shared between arrays and slices, but we need the correct - // serializer based on the actual Go type - if localType.Kind() == reflect.Array { - elemType := localType.Elem() - switch elemType.Kind() { - case reflect.Bool: - fieldSerializer = boolArraySerializer{arrayType: localType} - case reflect.Int8: - fieldSerializer = int8ArraySerializer{arrayType: localType} - case reflect.Int16: - fieldSerializer = int16ArraySerializer{arrayType: localType} - case reflect.Int32: - fieldSerializer = int32ArraySerializer{arrayType: localType} - case reflect.Int64: - fieldSerializer = int64ArraySerializer{arrayType: localType} - case reflect.Uint8: - fieldSerializer = uint8ArraySerializer{arrayType: localType} - case reflect.Float32: - fieldSerializer = float32ArraySerializer{arrayType: localType} - case reflect.Float64: - fieldSerializer = float64ArraySerializer{arrayType: localType} - case reflect.Int: - if reflect.TypeOf(int(0)).Size() == 8 { - fieldSerializer = int64ArraySerializer{arrayType: localType} - } else { - fieldSerializer = int32ArraySerializer{arrayType: localType} - } - } - } - } else { - // Types are incompatible or unknown - use remote type but mark field as not settable - fieldType = remoteType - fieldIndex = -1 - offset = 0 // Don't set offset for incompatible fields - } - } else { - // Field doesn't exist locally, use type from fieldDef - fieldType = remoteType - } - - // Get TypeId from FieldType's TypeId method - fieldTypeId := def.fieldType.TypeId() - // Pre-compute RefMode based on FieldDef flags (trackingRef and nullable) - refMode := RefModeNone - if def.trackingRef { - refMode = RefModeTracking - } else if def.nullable { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(fieldType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(fieldType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - - // Determine field name: use local field name if matched, otherwise use def.name - fieldName := def.name - if localFieldName != "" { - fieldName = localFieldName - } - - fieldInfo := &FieldInfo{ - Name: fieldName, - Offset: offset, - Type: fieldType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: def.nullable, // Use remote nullable flag - FieldIndex: fieldIndex, - FieldDef: def, // Save original FieldDef for skipping - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - TagID: def.tagID, - HasForyTag: def.tagID >= 0, - } - fields = append(fields, fieldInfo) - } - - s.fields = fields - s.groupFields() - - // Compute typeDefDiffers: true if any field doesn't exist locally or has type mismatch - // When typeDefDiffers is false, we can use grouped reading for better performance - s.typeDefDiffers = false - for _, field := range fields { - if field.FieldIndex < 0 { - // Field exists in remote TypeDef but not locally - s.typeDefDiffers = true - break - } - } - - return nil + type_ := s.type_ + if type_ == nil { + // Type is not known - we'll create an interface{} placeholder + // This happens when deserializing unknown types in compatible mode + // For now, we'll create fields that discard all data + var fields []*FieldInfo + for _, def := range s.fieldDefs { + fieldSerializer, _ := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + remoteType := remoteTypeInfo.Type + if remoteType == nil { + remoteType = reflect.TypeOf((*interface{})(nil)).Elem() + } + // Get TypeId from FieldType's TypeId method + fieldTypeId := def.fieldType.TypeId() + // Pre-compute RefMode based on trackRef and FieldDef flags + refMode := RefModeNone + if def.trackingRef { + refMode = RefModeTracking + } else if def.nullable { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(remoteType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(remoteType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + + fieldInfo := &FieldInfo{ + Name: def.name, + Offset: 0, + Type: remoteType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: def.nullable, // Use remote nullable flag + FieldIndex: -1, // Mark as non-existent field to discard data + FieldDef: def, // Save original FieldDef for skipping + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + } + fields = append(fields, fieldInfo) + } + s.fields = fields + s.groupFields() + s.typeDefDiffers = true // Unknown type, must use ordered reading + return nil + } + + // Build maps from field names and tag IDs to struct field indices + fieldNameToIndex := make(map[string]int) + fieldNameToOffset := make(map[string]uintptr) + fieldNameToType := make(map[string]reflect.Type) + fieldTagIDToIndex := make(map[int]int) // tag ID -> struct field index + fieldTagIDToOffset := make(map[int]uintptr) // tag ID -> field offset + fieldTagIDToType := make(map[int]reflect.Type) // tag ID -> field type + fieldTagIDToName := make(map[int]string) // tag ID -> snake_case field name + for i := 0; i < type_.NumField(); i++ { + field := type_.Field(i) + + // Parse fory tag and skip ignored fields + foryTag := ParseForyTag(field) + if foryTag.Ignore { + continue + } + + name := SnakeCase(field.Name) + fieldNameToIndex[name] = i + fieldNameToOffset[name] = field.Offset + fieldNameToType[name] = field.Type + + // Also index by tag ID if present + if foryTag.ID >= 0 { + fieldTagIDToIndex[foryTag.ID] = i + fieldTagIDToOffset[foryTag.ID] = field.Offset + fieldTagIDToType[foryTag.ID] = field.Type + fieldTagIDToName[foryTag.ID] = name + } + } + + var fields []*FieldInfo + + for _, def := range s.fieldDefs { + fieldSerializer, err := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) + if err != nil || fieldSerializer == nil { + // If we can't get serializer from typeID, try to get it from the Go type + // This can happen when the type isn't registered in typeIDToTypeInfo + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + if remoteTypeInfo.Type != nil { + fieldSerializer, _ = typeResolver.getSerializerByType(remoteTypeInfo.Type, true) + } + } + + // Get the remote type from fieldDef + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + remoteType := remoteTypeInfo.Type + // Track if type lookup failed - we'll need to skip such fields + // Note: DynamicFieldType.getTypeInfoWithResolver returns interface{} (not nil) when lookup fails + emptyInterfaceType := reflect.TypeOf((*interface{})(nil)).Elem() + typeLookupFailed := remoteType == nil || remoteType == emptyInterfaceType + if remoteType == nil { + remoteType = emptyInterfaceType + } + + // For struct-like fields, even if TypeDef lookup fails, we can try to read + // the field because type resolution happens at read time from the buffer. + // The type name might map to a different local type. + isStructLikeField := isStructFieldType(def.fieldType) + + // Try to find corresponding local field + // First try to match by tag ID (if remote def uses tag ID) + // Then fall back to matching by field name + fieldIndex := -1 + var offset uintptr + var fieldType reflect.Type + var localFieldName string + var localType reflect.Type + var exists bool + + if def.tagID >= 0 { + // Try to match by tag ID + if idx, ok := fieldTagIDToIndex[def.tagID]; ok { + exists = true + fieldIndex = idx // Will be overwritten if types are compatible + localType = fieldTagIDToType[def.tagID] + offset = fieldTagIDToOffset[def.tagID] + localFieldName = fieldTagIDToName[def.tagID] + _ = fieldIndex // Use to avoid compiler warning, will be set properly below + } + } + + // Fall back to name-based matching if tag ID match failed + if !exists && def.name != "" { + if idx, ok := fieldNameToIndex[def.name]; ok { + exists = true + localType = fieldNameToType[def.name] + offset = fieldNameToOffset[def.name] + localFieldName = def.name + _ = idx // Will be set properly below + } + } + + if exists { + idx := fieldNameToIndex[localFieldName] + if def.tagID >= 0 { + idx = fieldTagIDToIndex[def.tagID] + } + // Check if types are compatible + // For primitive types: skip if types don't match + // For struct-like types: allow read even if TypeDef lookup failed, + // because runtime type resolution by name might work + shouldRead := false + isPolymorphicField := def.fieldType.TypeId() == UNKNOWN + defTypeId := def.fieldType.TypeId() + // Check if field is an enum - either by type ID or by serializer type + // The type ID may be a composite value with namespace bits, so check the low 8 bits + internalDefTypeId := defTypeId & 0xFF + isEnumField := internalDefTypeId == NAMED_ENUM || internalDefTypeId == ENUM + if !isEnumField && fieldSerializer != nil { + _, isEnumField = fieldSerializer.(*enumSerializer) + } + if isPolymorphicField && localType.Kind() == reflect.Interface { + // For polymorphic (UNKNOWN) fields with interface{} local type, + // allow reading - the actual type will be determined at runtime + shouldRead = true + fieldType = localType + } else if typeLookupFailed && isEnumField { + // For enum fields with failed TypeDef lookup (NAMED_ENUM stores by namespace/typename, not typeId), + // check if local field is a numeric type (Go enums are int-based) + // Also handle pointer enum fields (*EnumType) + localKind := localType.Kind() + elemKind := localKind + if localKind == reflect.Ptr { + elemKind = localType.Elem().Kind() + } + if isNumericKind(elemKind) { + shouldRead = true + fieldType = localType + // Get the serializer for the base type (the enum type, not the pointer) + baseType := localType + if localKind == reflect.Ptr { + baseType = localType.Elem() + } + fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) + } + } else if typeLookupFailed && isStructLikeField { + // For struct fields with failed TypeDef lookup, check if local field can hold a struct + localKind := localType.Kind() + if localKind == reflect.Ptr { + localKind = localType.Elem().Kind() + } + if localKind == reflect.Struct || localKind == reflect.Interface { + shouldRead = true + fieldType = localType // Use local type for struct fields + } + } else if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) { + // For collection fields with failed type lookup (e.g., List with interface element type), + // check if local type is a slice with interface element type (e.g., []Animal) + // The type lookup fails because sliceSerializer doesn't support interface elements + if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { + shouldRead = true + fieldType = localType + } + } else if !typeLookupFailed && typesCompatible(localType, remoteType) { + shouldRead = true + fieldType = localType + } + + if shouldRead { + fieldIndex = idx + // offset was already set above when matching by tag ID or field name + // For struct-like fields with failed type lookup, get the serializer for the local type + if typeLookupFailed && isStructLikeField && fieldSerializer == nil { + fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) + } + // For collection fields with interface element types, use sliceDynSerializer + if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) && fieldSerializer == nil { + if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { + fieldSerializer = mustNewSliceDynSerializer(localType.Elem()) + } + } + // If local type is *T and remote type is T, we need the serializer for *T + // This handles Java's Integer/Long (nullable boxed types) mapping to Go's *int32/*int64 + if localType.Kind() == reflect.Ptr && localType.Elem() == remoteType { + fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) + } + // For pointer enum fields (*EnumType), get the serializer for the base enum type + // The struct read/write code will handle pointer dereferencing + if isEnumField && localType.Kind() == reflect.Ptr { + baseType := localType.Elem() + fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] pointer enum field %s: localType=%v baseType=%v serializer=%T\n", + def.name, localType, baseType, fieldSerializer) + } + } + // For array fields, use array serializers (not slice serializers) even if typeID maps to slice serializer + // The typeID (INT16_ARRAY, etc.) is shared between arrays and slices, but we need the correct + // serializer based on the actual Go type + if localType.Kind() == reflect.Array { + elemType := localType.Elem() + switch elemType.Kind() { + case reflect.Bool: + fieldSerializer = boolArraySerializer{arrayType: localType} + case reflect.Int8: + fieldSerializer = int8ArraySerializer{arrayType: localType} + case reflect.Int16: + fieldSerializer = int16ArraySerializer{arrayType: localType} + case reflect.Int32: + fieldSerializer = int32ArraySerializer{arrayType: localType} + case reflect.Int64: + fieldSerializer = int64ArraySerializer{arrayType: localType} + case reflect.Uint8: + fieldSerializer = uint8ArraySerializer{arrayType: localType} + case reflect.Float32: + fieldSerializer = float32ArraySerializer{arrayType: localType} + case reflect.Float64: + fieldSerializer = float64ArraySerializer{arrayType: localType} + case reflect.Int: + if reflect.TypeOf(int(0)).Size() == 8 { + fieldSerializer = int64ArraySerializer{arrayType: localType} + } else { + fieldSerializer = int32ArraySerializer{arrayType: localType} + } + } + } + } else { + // Types are incompatible or unknown - use remote type but mark field as not settable + fieldType = remoteType + fieldIndex = -1 + offset = 0 // Don't set offset for incompatible fields + } + } else { + // Field doesn't exist locally, use type from fieldDef + fieldType = remoteType + } + + // Get TypeId from FieldType's TypeId method + fieldTypeId := def.fieldType.TypeId() + // Pre-compute RefMode based on FieldDef flags (trackingRef and nullable) + refMode := RefModeNone + if def.trackingRef { + refMode = RefModeTracking + } else if def.nullable { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(fieldType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(fieldType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + + // Determine field name: use local field name if matched, otherwise use def.name + fieldName := def.name + if localFieldName != "" { + fieldName = localFieldName + } + + fieldInfo := &FieldInfo{ + Name: fieldName, + Offset: offset, + Type: fieldType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: def.nullable, // Use remote nullable flag + FieldIndex: fieldIndex, + FieldDef: def, // Save original FieldDef for skipping + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + TagID: def.tagID, + HasForyTag: def.tagID >= 0, + } + fields = append(fields, fieldInfo) + } + + s.fields = fields + s.groupFields() + + // Compute typeDefDiffers: true if any field doesn't exist locally or has type mismatch + // When typeDefDiffers is false, we can use grouped reading for better performance + s.typeDefDiffers = false + for _, field := range fields { + if field.FieldIndex < 0 { + // Field exists in remote TypeDef but not locally + s.typeDefDiffers = true + break + } + } + + return nil } // isNonNullablePrimitiveKind returns true for Go kinds that map to Java primitive types // These are the types that cannot be null in Java and should have nullable=0 in hash computation func isNonNullablePrimitiveKind(kind reflect.Kind) bool { - switch kind { - case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64, reflect.Int, reflect.Uint: - return true - default: - return false - } + switch kind { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.Int, reflect.Uint: + return true + default: + return false + } } // isInternalTypeWithoutTypeMeta checks if a type is serialized without type meta per xlang spec. @@ -1605,40 +1606,40 @@ func isNonNullablePrimitiveKind(kind reflect.Kind) bool { // - Map: | ref meta | value data | // Only struct/enum/ext types need type meta: | ref flag | type meta | value data | func isInternalTypeWithoutTypeMeta(t reflect.Type) bool { - kind := t.Kind() - // String type - no type meta needed - if kind == reflect.String { - return true - } - // Slice (list or byte slice) - no type meta needed - if kind == reflect.Slice { - return true - } - // Map type - no type meta needed - if kind == reflect.Map { - return true - } - // Pointer to primitive - no type meta needed - if kind == reflect.Ptr { - elemKind := t.Elem().Kind() - switch elemKind { - case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Int, reflect.Float32, reflect.Float64, reflect.String: - return true - } - } - return false + kind := t.Kind() + // String type - no type meta needed + if kind == reflect.String { + return true + } + // Slice (list or byte slice) - no type meta needed + if kind == reflect.Slice { + return true + } + // Map type - no type meta needed + if kind == reflect.Map { + return true + } + // Pointer to primitive - no type meta needed + if kind == reflect.Ptr { + elemKind := t.Elem().Kind() + switch elemKind { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Int, reflect.Float32, reflect.Float64, reflect.String: + return true + } + } + return false } // isStructField checks if a type is a struct type (directly or via pointer) func isStructField(t reflect.Type) bool { - if t.Kind() == reflect.Struct { - return true - } - if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { - return true - } - return false + if t.Kind() == reflect.Struct { + return true + } + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + return true + } + return false } // isStructFieldType checks if a FieldType represents a type that needs type info written @@ -1646,40 +1647,34 @@ func isStructField(t reflect.Type) bool { // In compatible mode, Java writes type info for struct and ext types, but NOT for enum types // Enum fields only have null flag + ordinal, no type ID func isStructFieldType(ft FieldType) bool { - if ft == nil { - return false - } - typeId := ft.TypeId() - // Check base type IDs that need type info (struct and ext, NOT enum) - switch typeId { - case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, - EXT, NAMED_EXT: - return true - } - // Check for composite type IDs (customId << 8 | baseType) - if typeId > 255 { - baseType := typeId & 0xff - switch TypeId(baseType) { - case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, - EXT, NAMED_EXT: - return true - } - } - return false + if ft == nil { + return false + } + typeId := ft.TypeId() + // Check base type IDs that need type info (struct and ext, NOT enum) + // Always check the internal type ID (low byte) to handle composite type IDs + // which may be negative when stored as int32 (e.g., -2288 = (short)128784) + internalTypeId := TypeId(typeId & 0xFF) + switch internalTypeId { + case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, + EXT, NAMED_EXT: + return true + } + return false } // FieldFingerprintInfo contains the information needed to compute a field's fingerprint. type FieldFingerprintInfo struct { - // FieldID is the tag ID if configured (>= 0), or -1 to use field name - FieldID int - // FieldName is the snake_case field name (used when FieldID < 0) - FieldName string - // TypeID is the Fory type ID for the field - TypeID TypeId - // Ref is true if reference tracking is enabled for this field - Ref bool - // Nullable is true if null flag is written for this field - Nullable bool + // FieldID is the tag ID if configured (>= 0), or -1 to use field name + FieldID int + // FieldName is the snake_case field name (used when FieldID < 0) + FieldName string + // TypeID is the Fory type ID for the field + TypeID TypeId + // Ref is true if reference tracking is enabled for this field + Ref bool + // Nullable is true if null flag is written for this field + Nullable bool } // ComputeStructFingerprint computes the fingerprint string for a struct type. @@ -1703,171 +1698,177 @@ type FieldFingerprintInfo struct { // Different nullable/ref settings will produce different fingerprints, // ensuring schema compatibility is properly validated. func ComputeStructFingerprint(fields []FieldFingerprintInfo) string { - // Sort fields by their identifier (field ID or name) - type fieldWithKey struct { - field FieldFingerprintInfo - sortKey string - } - fieldsWithKeys := make([]fieldWithKey, 0, len(fields)) - for _, field := range fields { - var sortKey string - if field.FieldID >= 0 { - sortKey = fmt.Sprintf("%d", field.FieldID) - } else { - sortKey = field.FieldName - } - fieldsWithKeys = append(fieldsWithKeys, fieldWithKey{field: field, sortKey: sortKey}) - } - - sort.Slice(fieldsWithKeys, func(i, j int) bool { - return fieldsWithKeys[i].sortKey < fieldsWithKeys[j].sortKey - }) - - var sb strings.Builder - for _, fw := range fieldsWithKeys { - // Field identifier - sb.WriteString(fw.sortKey) - sb.WriteString(",") - // Type ID - sb.WriteString(fmt.Sprintf("%d", fw.field.TypeID)) - sb.WriteString(",") - // Ref flag - if fw.field.Ref { - sb.WriteString("1") - } else { - sb.WriteString("0") - } - sb.WriteString(",") - // Nullable flag - if fw.field.Nullable { - sb.WriteString("1") - } else { - sb.WriteString("0") - } - sb.WriteString(";") - } - return sb.String() + // Sort fields by their identifier (field ID or name) + type fieldWithKey struct { + field FieldFingerprintInfo + sortKey string + } + fieldsWithKeys := make([]fieldWithKey, 0, len(fields)) + for _, field := range fields { + var sortKey string + if field.FieldID >= 0 { + sortKey = fmt.Sprintf("%d", field.FieldID) + } else { + sortKey = field.FieldName + } + fieldsWithKeys = append(fieldsWithKeys, fieldWithKey{field: field, sortKey: sortKey}) + } + + sort.Slice(fieldsWithKeys, func(i, j int) bool { + return fieldsWithKeys[i].sortKey < fieldsWithKeys[j].sortKey + }) + + var sb strings.Builder + for _, fw := range fieldsWithKeys { + // Field identifier + sb.WriteString(fw.sortKey) + sb.WriteString(",") + // Type ID + sb.WriteString(fmt.Sprintf("%d", fw.field.TypeID)) + sb.WriteString(",") + // Ref flag + if fw.field.Ref { + sb.WriteString("1") + } else { + sb.WriteString("0") + } + sb.WriteString(",") + // Nullable flag + if fw.field.Nullable { + sb.WriteString("1") + } else { + sb.WriteString("0") + } + sb.WriteString(";") + } + return sb.String() } func (s *structSerializer) computeHash() int32 { - // Build FieldFingerprintInfo for each field - fields := make([]FieldFingerprintInfo, 0, len(s.fields)) - for _, field := range s.fields { - var typeId TypeId - isEnumField := false - if field.Serializer == nil { - typeId = UNKNOWN - } else { - typeId = field.TypeId - // Check if this is an enum serializer (directly or wrapped in ptrToValueSerializer) - if _, ok := field.Serializer.(*enumSerializer); ok { - isEnumField = true - typeId = UNKNOWN - } else if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - isEnumField = true - typeId = UNKNOWN - } - } - // For fixed-size arrays with primitive elements, use primitive array type IDs - if field.Type.Kind() == reflect.Array { - elemKind := field.Type.Elem().Kind() - switch elemKind { - case reflect.Int8: - typeId = INT8_ARRAY - case reflect.Int16: - typeId = INT16_ARRAY - case reflect.Int32: - typeId = INT32_ARRAY - case reflect.Int64: - typeId = INT64_ARRAY - case reflect.Float32: - typeId = FLOAT32_ARRAY - case reflect.Float64: - typeId = FLOAT64_ARRAY - default: - typeId = LIST - } - } else if field.Type.Kind() == reflect.Slice { - typeId = LIST - } else if field.Type.Kind() == reflect.Map { - // map[T]bool is used to represent a Set in Go - if field.Type.Elem().Kind() == reflect.Bool { - typeId = SET - } else { - typeId = MAP - } - } - } - - // Determine nullable flag for xlang compatibility: - // - Default: false for ALL fields (xlang default - aligned with all languages) - // - Primitives are always non-nullable - // - Can be overridden by explicit fory tag - nullable := false // Default to nullable=false for xlang mode - if field.TagNullableSet { - // Use explicit tag value if set - nullable = field.TagNullable - } - // Primitives are never nullable, regardless of tag - if isNonNullablePrimitiveKind(field.Type.Kind()) && !isEnumField { - nullable = false - } - - fields = append(fields, FieldFingerprintInfo{ - FieldID: field.TagID, - FieldName: SnakeCase(field.Name), - TypeID: typeId, - // Ref is based on explicit tag annotation only, NOT runtime ref_tracking config - // This allows fingerprint to be computed at compile time for C++/Rust - Ref: field.TagRefSet && field.TagRef, - Nullable: nullable, - }) - } - - hashString := ComputeStructFingerprint(fields) - data := []byte(hashString) - h1, _ := murmur3.Sum128WithSeed(data, 47) - hash := int32(h1 & 0xFFFFFFFF) - - if DebugOutputEnabled() { - fmt.Printf("[Go][fory-debug] struct %v version fingerprint=\"%s\" version hash=%d\n", s.type_, hashString, hash) - } - - if hash == 0 { - panic(fmt.Errorf("hash for type %v is 0", s.type_)) - } - return hash + // Build FieldFingerprintInfo for each field + fields := make([]FieldFingerprintInfo, 0, len(s.fields)) + for _, field := range s.fields { + var typeId TypeId + isEnumField := false + if field.Serializer == nil { + typeId = UNKNOWN + } else { + typeId = field.TypeId + // Check if this is an enum serializer (directly or wrapped in ptrToValueSerializer) + if _, ok := field.Serializer.(*enumSerializer); ok { + isEnumField = true + typeId = UNKNOWN + } else if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + isEnumField = true + typeId = UNKNOWN + } + } + // For user-defined types (struct, ext types), use UNKNOWN in fingerprint + // This matches Java's behavior where user-defined types return UNKNOWN + // to ensure consistent fingerprint computation across languages + if isUserDefinedType(int16(typeId)) { + typeId = UNKNOWN + } + // For fixed-size arrays with primitive elements, use primitive array type IDs + if field.Type.Kind() == reflect.Array { + elemKind := field.Type.Elem().Kind() + switch elemKind { + case reflect.Int8: + typeId = INT8_ARRAY + case reflect.Int16: + typeId = INT16_ARRAY + case reflect.Int32: + typeId = INT32_ARRAY + case reflect.Int64: + typeId = INT64_ARRAY + case reflect.Float32: + typeId = FLOAT32_ARRAY + case reflect.Float64: + typeId = FLOAT64_ARRAY + default: + typeId = LIST + } + } else if field.Type.Kind() == reflect.Slice { + typeId = LIST + } else if field.Type.Kind() == reflect.Map { + // map[T]bool is used to represent a Set in Go + if field.Type.Elem().Kind() == reflect.Bool { + typeId = SET + } else { + typeId = MAP + } + } + } + + // Determine nullable flag for xlang compatibility: + // - Default: false for ALL fields (xlang default - aligned with all languages) + // - Primitives are always non-nullable + // - Can be overridden by explicit fory tag + nullable := false // Default to nullable=false for xlang mode + if field.TagNullableSet { + // Use explicit tag value if set + nullable = field.TagNullable + } + // Primitives are never nullable, regardless of tag + if isNonNullablePrimitiveKind(field.Type.Kind()) && !isEnumField { + nullable = false + } + + fields = append(fields, FieldFingerprintInfo{ + FieldID: field.TagID, + FieldName: SnakeCase(field.Name), + TypeID: typeId, + // Ref is based on explicit tag annotation only, NOT runtime ref_tracking config + // This allows fingerprint to be computed at compile time for C++/Rust + Ref: field.TagRefSet && field.TagRef, + Nullable: nullable, + }) + } + + hashString := ComputeStructFingerprint(fields) + data := []byte(hashString) + h1, _ := murmur3.Sum128WithSeed(data, 47) + hash := int32(h1 & 0xFFFFFFFF) + + if DebugOutputEnabled() { + fmt.Printf("[Go][fory-debug] struct %v version fingerprint=\"%s\" version hash=%d\n", s.type_, hashString, hash) + } + + if hash == 0 { + panic(fmt.Errorf("hash for type %v is 0", s.type_)) + } + return hash } // GetStructHash returns the struct hash for a given type using the provided TypeResolver. // This is used by codegen serializers to get the hash at runtime. func GetStructHash(type_ reflect.Type, resolver *TypeResolver) int32 { - ser := newStructSerializer(type_, "", nil) - if err := ser.initialize(resolver); err != nil { - panic(fmt.Errorf("failed to initialize struct serializer for hash computation: %v", err)) - } - return ser.structHash + ser := newStructSerializer(type_, "", nil) + if err := ser.initialize(resolver); err != nil { + panic(fmt.Errorf("failed to initialize struct serializer for hash computation: %v", err)) + } + return ser.structHash } // Field sorting helpers type triple struct { - typeID int16 - serializer Serializer - name string - nullable bool - tagID int // -1 = use field name, >=0 = use tag ID for sorting + typeID int16 + serializer Serializer + name string + nullable bool + tagID int // -1 = use field name, >=0 = use tag ID for sorting } // getFieldSortKey returns the sort key for a field. // If tagID >= 0, returns the tag ID as string (for tag-based sorting). // Otherwise returns the snake_case field name. func (t triple) getSortKey() string { - if t.tagID >= 0 { - return fmt.Sprintf("%d", t.tagID) - } - return SnakeCase(t.name) + if t.tagID >= 0 { + return fmt.Sprintf("%d", t.tagID) + } + return SnakeCase(t.name) } // sortFields sorts fields with nullable information to match Java's field ordering. @@ -1875,315 +1876,315 @@ func (t triple) getSortKey() string { // In Go, this corresponds to non-pointer primitives vs pointer-to-primitive. // When tagIDs are provided (>= 0), fields are sorted by tag ID instead of field name. func sortFields( - typeResolver *TypeResolver, - fieldNames []string, - serializers []Serializer, - typeIds []TypeId, - nullables []bool, - tagIDs []int, + typeResolver *TypeResolver, + fieldNames []string, + serializers []Serializer, + typeIds []TypeId, + nullables []bool, + tagIDs []int, ) ([]Serializer, []string) { - var ( - typeTriples []triple - others []triple - userDefined []triple - ) - - for i, name := range fieldNames { - ser := serializers[i] - tagID := TagIDUseFieldName // default: use field name - if tagIDs != nil && i < len(tagIDs) { - tagID = tagIDs[i] - } - if ser == nil { - others = append(others, triple{UNKNOWN, nil, name, nullables[i], tagID}) - continue - } - typeTriples = append(typeTriples, triple{typeIds[i], ser, name, nullables[i], tagID}) - } - // Java orders: primitives, boxed, finals, others, collections, maps - // primitives = non-nullable primitive types (int, long, etc.) - // boxed = nullable boxed types (Integer, Long, etc. which are pointers in Go) - var primitives, boxed, collection, setFields, maps, otherInternalTypeFields []triple - - for _, t := range typeTriples { - switch { - case isPrimitiveType(t.typeID): - // Separate non-nullable primitives from nullable (boxed) primitives - if t.nullable { - boxed = append(boxed, t) - } else { - primitives = append(primitives, t) - } - case isListType(t.typeID), isPrimitiveArrayType(t.typeID): - collection = append(collection, t) - case isSetType(t.typeID): - setFields = append(setFields, t) - case isMapType(t.typeID): - maps = append(maps, t) - case isUserDefinedType(t.typeID): - userDefined = append(userDefined, t) - case t.typeID == UNKNOWN: - others = append(others, t) - default: - otherInternalTypeFields = append(otherInternalTypeFields, t) - } - } - // Sort primitives (non-nullable) - same logic as boxed - // Java sorts by: compressed types last, then by size (largest first), then by type ID (descending) - sortPrimitiveSlice := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - ai, aj := s[i], s[j] - compressI := ai.typeID == INT32 || ai.typeID == INT64 || - ai.typeID == VAR_INT32 || ai.typeID == VAR_INT64 - compressJ := aj.typeID == INT32 || aj.typeID == INT64 || - aj.typeID == VAR_INT32 || aj.typeID == VAR_INT64 - if compressI != compressJ { - return !compressI && compressJ - } - szI, szJ := getPrimitiveTypeSize(ai.typeID), getPrimitiveTypeSize(aj.typeID) - if szI != szJ { - return szI > szJ - } - // Tie-breaker: type ID descending (higher type ID first), then field name - if ai.typeID != aj.typeID { - return ai.typeID > aj.typeID - } - return ai.getSortKey() < aj.getSortKey() - }) - } - sortPrimitiveSlice(primitives) - sortPrimitiveSlice(boxed) - sortByTypeIDThenName := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - if s[i].typeID != s[j].typeID { - return s[i].typeID < s[j].typeID - } - return s[i].getSortKey() < s[j].getSortKey() - }) - } - sortTuple := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - return s[i].getSortKey() < s[j].getSortKey() - }) - } - sortByTypeIDThenName(otherInternalTypeFields) - sortTuple(others) - sortTuple(collection) - sortTuple(setFields) - sortTuple(maps) - sortTuple(userDefined) - - // Java order: primitives, boxed, finals, collections, maps, others - // finals = String and other monomorphic types (otherInternalTypeFields) - // others = userDefined types (structs, enums) and unknown types - all := make([]triple, 0, len(fieldNames)) - all = append(all, primitives...) - all = append(all, boxed...) - all = append(all, otherInternalTypeFields...) // finals (String, etc.) - all = append(all, collection...) - all = append(all, setFields...) - all = append(all, maps...) - all = append(all, userDefined...) // others (structs, enums) - all = append(all, others...) // unknown types - - outSer := make([]Serializer, len(all)) - outNam := make([]string, len(all)) - for i, t := range all { - outSer[i] = t.serializer - outNam[i] = t.name - } - return outSer, outNam + var ( + typeTriples []triple + others []triple + userDefined []triple + ) + + for i, name := range fieldNames { + ser := serializers[i] + tagID := TagIDUseFieldName // default: use field name + if tagIDs != nil && i < len(tagIDs) { + tagID = tagIDs[i] + } + if ser == nil { + others = append(others, triple{UNKNOWN, nil, name, nullables[i], tagID}) + continue + } + typeTriples = append(typeTriples, triple{typeIds[i], ser, name, nullables[i], tagID}) + } + // Java orders: primitives, boxed, finals, others, collections, maps + // primitives = non-nullable primitive types (int, long, etc.) + // boxed = nullable boxed types (Integer, Long, etc. which are pointers in Go) + var primitives, boxed, collection, setFields, maps, otherInternalTypeFields []triple + + for _, t := range typeTriples { + switch { + case isPrimitiveType(t.typeID): + // Separate non-nullable primitives from nullable (boxed) primitives + if t.nullable { + boxed = append(boxed, t) + } else { + primitives = append(primitives, t) + } + case isListType(t.typeID), isPrimitiveArrayType(t.typeID): + collection = append(collection, t) + case isSetType(t.typeID): + setFields = append(setFields, t) + case isMapType(t.typeID): + maps = append(maps, t) + case isUserDefinedType(t.typeID): + userDefined = append(userDefined, t) + case t.typeID == UNKNOWN: + others = append(others, t) + default: + otherInternalTypeFields = append(otherInternalTypeFields, t) + } + } + // Sort primitives (non-nullable) - same logic as boxed + // Java sorts by: compressed types last, then by size (largest first), then by type ID (descending) + sortPrimitiveSlice := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + ai, aj := s[i], s[j] + compressI := ai.typeID == INT32 || ai.typeID == INT64 || + ai.typeID == VAR_INT32 || ai.typeID == VAR_INT64 + compressJ := aj.typeID == INT32 || aj.typeID == INT64 || + aj.typeID == VAR_INT32 || aj.typeID == VAR_INT64 + if compressI != compressJ { + return !compressI && compressJ + } + szI, szJ := getPrimitiveTypeSize(ai.typeID), getPrimitiveTypeSize(aj.typeID) + if szI != szJ { + return szI > szJ + } + // Tie-breaker: type ID descending (higher type ID first), then field name + if ai.typeID != aj.typeID { + return ai.typeID > aj.typeID + } + return ai.getSortKey() < aj.getSortKey() + }) + } + sortPrimitiveSlice(primitives) + sortPrimitiveSlice(boxed) + sortByTypeIDThenName := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + if s[i].typeID != s[j].typeID { + return s[i].typeID < s[j].typeID + } + return s[i].getSortKey() < s[j].getSortKey() + }) + } + sortTuple := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + return s[i].getSortKey() < s[j].getSortKey() + }) + } + sortByTypeIDThenName(otherInternalTypeFields) + sortTuple(others) + sortTuple(collection) + sortTuple(setFields) + sortTuple(maps) + sortTuple(userDefined) + + // Java order: primitives, boxed, finals, collections, maps, others + // finals = String and other monomorphic types (otherInternalTypeFields) + // others = userDefined types (structs, enums) and unknown types + all := make([]triple, 0, len(fieldNames)) + all = append(all, primitives...) + all = append(all, boxed...) + all = append(all, otherInternalTypeFields...) // finals (String, etc.) + all = append(all, collection...) + all = append(all, setFields...) + all = append(all, maps...) + all = append(all, userDefined...) // others (structs, enums) + all = append(all, others...) // unknown types + + outSer := make([]Serializer, len(all)) + outNam := make([]string, len(all)) + for i, t := range all { + outSer[i] = t.serializer + outNam[i] = t.name + } + return outSer, outNam } func typesCompatible(actual, expected reflect.Type) bool { - if actual == nil || expected == nil { - return false - } - if actual == expected { - return true - } - // interface{} can accept any value - if actual.Kind() == reflect.Interface && actual.NumMethod() == 0 { - return true - } - if actual.AssignableTo(expected) || expected.AssignableTo(actual) { - return true - } - if actual.Kind() == reflect.Ptr && actual.Elem() == expected { - return true - } - if expected.Kind() == reflect.Ptr && expected.Elem() == actual { - return true - } - if actual.Kind() == expected.Kind() { - switch actual.Kind() { - case reflect.Slice, reflect.Array: - return elementTypesCompatible(actual.Elem(), expected.Elem()) - case reflect.Map: - return elementTypesCompatible(actual.Key(), expected.Key()) && elementTypesCompatible(actual.Elem(), expected.Elem()) - } - } - if (actual.Kind() == reflect.Array && expected.Kind() == reflect.Slice) || - (actual.Kind() == reflect.Slice && expected.Kind() == reflect.Array) { - return true - } - return false + if actual == nil || expected == nil { + return false + } + if actual == expected { + return true + } + // interface{} can accept any value + if actual.Kind() == reflect.Interface && actual.NumMethod() == 0 { + return true + } + if actual.AssignableTo(expected) || expected.AssignableTo(actual) { + return true + } + if actual.Kind() == reflect.Ptr && actual.Elem() == expected { + return true + } + if expected.Kind() == reflect.Ptr && expected.Elem() == actual { + return true + } + if actual.Kind() == expected.Kind() { + switch actual.Kind() { + case reflect.Slice, reflect.Array: + return elementTypesCompatible(actual.Elem(), expected.Elem()) + case reflect.Map: + return elementTypesCompatible(actual.Key(), expected.Key()) && elementTypesCompatible(actual.Elem(), expected.Elem()) + } + } + if (actual.Kind() == reflect.Array && expected.Kind() == reflect.Slice) || + (actual.Kind() == reflect.Slice && expected.Kind() == reflect.Array) { + return true + } + return false } func elementTypesCompatible(actual, expected reflect.Type) bool { - if actual == nil || expected == nil { - return false - } - if actual == expected || actual.AssignableTo(expected) || expected.AssignableTo(actual) { - return true - } - if actual.Kind() == reflect.Ptr { - return elementTypesCompatible(actual, expected.Elem()) - } - return false + if actual == nil || expected == nil { + return false + } + if actual == expected || actual.AssignableTo(expected) || expected.AssignableTo(actual) { + return true + } + if actual.Kind() == reflect.Ptr { + return elementTypesCompatible(actual, expected.Elem()) + } + return false } // typeIdFromKind derives a TypeId from a reflect.Type's kind // This is used when the type is not registered in typesInfo func typeIdFromKind(type_ reflect.Type) TypeId { - switch type_.Kind() { - case reflect.Bool: - return BOOL - case reflect.Int8: - return INT8 - case reflect.Int16: - return INT16 - case reflect.Int32: - return INT32 - case reflect.Int64, reflect.Int: - return INT64 - case reflect.Uint8: - return UINT8 - case reflect.Uint16: - return UINT16 - case reflect.Uint32: - return UINT32 - case reflect.Uint64, reflect.Uint: - return UINT64 - case reflect.Float32: - return FLOAT - case reflect.Float64: - return DOUBLE - case reflect.String: - return STRING - case reflect.Slice: - // For slices, return the appropriate primitive array type ID based on element type - elemKind := type_.Elem().Kind() - switch elemKind { - case reflect.Bool: - return BOOL_ARRAY - case reflect.Int8: - return INT8_ARRAY - case reflect.Int16: - return INT16_ARRAY - case reflect.Int32: - return INT32_ARRAY - case reflect.Int64, reflect.Int: - return INT64_ARRAY - case reflect.Float32: - return FLOAT32_ARRAY - case reflect.Float64: - return FLOAT64_ARRAY - default: - // Non-primitive slices use LIST - return LIST - } - case reflect.Array: - // For arrays, return the appropriate primitive array type ID based on element type - elemKind := type_.Elem().Kind() - switch elemKind { - case reflect.Bool: - return BOOL_ARRAY - case reflect.Int8: - return INT8_ARRAY - case reflect.Int16: - return INT16_ARRAY - case reflect.Int32: - return INT32_ARRAY - case reflect.Int64, reflect.Int: - return INT64_ARRAY - case reflect.Float32: - return FLOAT32_ARRAY - case reflect.Float64: - return FLOAT64_ARRAY - default: - // Non-primitive arrays use LIST - return LIST - } - case reflect.Map: - // map[T]bool is used to represent a Set in Go - if type_.Elem().Kind() == reflect.Bool { - return SET - } - return MAP - case reflect.Struct: - return NAMED_STRUCT - case reflect.Ptr: - // For pointer types, get the type ID of the element type - return typeIdFromKind(type_.Elem()) - default: - return UNKNOWN - } + switch type_.Kind() { + case reflect.Bool: + return BOOL + case reflect.Int8: + return INT8 + case reflect.Int16: + return INT16 + case reflect.Int32: + return INT32 + case reflect.Int64, reflect.Int: + return INT64 + case reflect.Uint8: + return UINT8 + case reflect.Uint16: + return UINT16 + case reflect.Uint32: + return UINT32 + case reflect.Uint64, reflect.Uint: + return UINT64 + case reflect.Float32: + return FLOAT + case reflect.Float64: + return DOUBLE + case reflect.String: + return STRING + case reflect.Slice: + // For slices, return the appropriate primitive array type ID based on element type + elemKind := type_.Elem().Kind() + switch elemKind { + case reflect.Bool: + return BOOL_ARRAY + case reflect.Int8: + return INT8_ARRAY + case reflect.Int16: + return INT16_ARRAY + case reflect.Int32: + return INT32_ARRAY + case reflect.Int64, reflect.Int: + return INT64_ARRAY + case reflect.Float32: + return FLOAT32_ARRAY + case reflect.Float64: + return FLOAT64_ARRAY + default: + // Non-primitive slices use LIST + return LIST + } + case reflect.Array: + // For arrays, return the appropriate primitive array type ID based on element type + elemKind := type_.Elem().Kind() + switch elemKind { + case reflect.Bool: + return BOOL_ARRAY + case reflect.Int8: + return INT8_ARRAY + case reflect.Int16: + return INT16_ARRAY + case reflect.Int32: + return INT32_ARRAY + case reflect.Int64, reflect.Int: + return INT64_ARRAY + case reflect.Float32: + return FLOAT32_ARRAY + case reflect.Float64: + return FLOAT64_ARRAY + default: + // Non-primitive arrays use LIST + return LIST + } + case reflect.Map: + // map[T]bool is used to represent a Set in Go + if type_.Elem().Kind() == reflect.Bool { + return SET + } + return MAP + case reflect.Struct: + return NAMED_STRUCT + case reflect.Ptr: + // For pointer types, get the type ID of the element type + return typeIdFromKind(type_.Elem()) + default: + return UNKNOWN + } } // skipStructSerializer is a serializer that skips unknown struct data // It reads and discards field data based on fieldDefs from remote TypeDef type skipStructSerializer struct { - fieldDefs []FieldDef + fieldDefs []FieldDef } func (s *skipStructSerializer) WriteData(ctx *WriteContext, value reflect.Value) { - ctx.SetError(SerializationError("skipStructSerializer does not support WriteData - unknown struct type")) + ctx.SetError(SerializationError("skipStructSerializer does not support WriteData - unknown struct type")) } func (s *skipStructSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - ctx.SetError(SerializationError("skipStructSerializer does not support Write - unknown struct type")) + ctx.SetError(SerializationError("skipStructSerializer does not support Write - unknown struct type")) } func (s *skipStructSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { - // Skip all fields based on fieldDefs from remote TypeDef - for _, fieldDef := range s.fieldDefs { - isStructType := isStructFieldType(fieldDef.fieldType) - // Use trackingRef from FieldDef for ref flag decision - SkipFieldValueWithTypeFlag(ctx, fieldDef, fieldDef.trackingRef, ctx.Compatible() && isStructType) - if ctx.HasError() { - return - } - } + // Skip all fields based on fieldDefs from remote TypeDef + for _, fieldDef := range s.fieldDefs { + isStructType := isStructFieldType(fieldDef.fieldType) + // Use trackingRef from FieldDef for ref flag decision + SkipFieldValueWithTypeFlag(ctx, fieldDef, fieldDef.trackingRef, ctx.Compatible() && isStructType) + if ctx.HasError() { + return + } + } } func (s *skipStructSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found, nothing to skip - return - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return - } - } - if ctx.HasError() { - return - } - s.ReadData(ctx, nil, value) + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) + if refErr != nil { + ctx.SetError(FromError(refErr)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found, nothing to skip + return + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return + } + } + if ctx.HasError() { + return + } + s.ReadData(ctx, nil, value) } func (s *skipStructSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again - just skip data - s.Read(ctx, refMode, false, false, value) + // typeInfo is already read, don't read it again - just skip data + s.Read(ctx, refMode, false, false, value) } diff --git a/go/fory/tests/generator_test.go b/go/fory/tests/generator_test.go index f5454bfad5..bbfbf5526e 100644 --- a/go/fory/tests/generator_test.go +++ b/go/fory/tests/generator_test.go @@ -135,12 +135,14 @@ func TestDynamicSliceDemo(t *testing.T) { func TestDynamicSliceDemoWithNilAndEmpty(t *testing.T) { // Test with nil and empty dynamic slices + // Use WithXlang(false) for native Go mode where nil slices are preserved original := &DynamicSliceDemo{ DynamicSlice: nil, // nil slice } // SerializeWithCallback using generated code - f := fory.NewFory(fory.WithRefTracking(true)) + // WithXlang(false) enables native Go mode where nil slices are preserved as nil + f := fory.NewFory(fory.WithXlang(false), fory.WithRefTracking(true)) data, err := f.Marshal(original) require.NoError(t, err, "Serialization should not fail") require.NotEmpty(t, data, "Serialized data should not be empty") @@ -174,6 +176,7 @@ func TestDynamicSliceDemoWithNilAndEmpty(t *testing.T) { // TestMapDemo tests basic map serialization and deserialization (including nil maps) func TestMapDemo(t *testing.T) { // Create test instance with various map types (including nil) + // Use WithXlang(false) for native Go mode where nil maps are preserved instance := &MapDemo{ StringMap: map[string]string{ "key1": "value1", @@ -188,7 +191,8 @@ func TestMapDemo(t *testing.T) { } // SerializeWithCallback with codegen - f := fory.NewFory(fory.WithRefTracking(true)) + // WithXlang(false) enables native Go mode where nil maps are preserved as nil + f := fory.NewFory(fory.WithXlang(false), fory.WithRefTracking(true)) data, err := f.Marshal(instance) require.NoError(t, err, "Serialization failed") diff --git a/go/fory/tests/structs_fory_gen.go b/go/fory/tests/structs_fory_gen.go index e2ede9369e..e3538214d9 100644 --- a/go/fory/tests/structs_fory_gen.go +++ b/go/fory/tests/structs_fory_gen.go @@ -1,6 +1,6 @@ // Code generated by forygen. DO NOT EDIT. -// source: structs.go -// generated at: 2026-01-01T05:48:22+08:00 +// source: /Users/chaokunyang/Desktop/dev/fory/go/fory/tests/structs.go +// generated at: 2026-01-03T14:03:09+08:00 package fory @@ -71,11 +71,13 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, // Field: DynamicSlice ([]interface{}) // Dynamic slice []interface{} handling - manual serialization { - if v.DynamicSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - sliceLen := len(v.DynamicSlice) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + sliceLen := 0 + if v.DynamicSlice != nil { + sliceLen = len(v.DynamicSlice) + } buf.WriteVaruint32(uint32(sliceLen)) if sliceLen > 0 { // WriteData collection flags for dynamic slice []interface{} @@ -83,7 +85,25 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, buf.WriteInt8(1) // CollectionTrackingRef only // WriteData each element using WriteValue for _, elem := range v.DynamicSlice { - ctx.WriteValue(reflect.ValueOf(elem)) + ctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true) + } + } + } else { + // Native Go mode: slices are nullable, write null flag + if v.DynamicSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + sliceLen := len(v.DynamicSlice) + buf.WriteVaruint32(uint32(sliceLen)) + if sliceLen > 0 { + // WriteData collection flags for dynamic slice []interface{} + // Only CollectionTrackingRef is set (no declared type, may have different types) + buf.WriteInt8(1) // CollectionTrackingRef only + // WriteData each element using WriteValue + for _, elem := range v.DynamicSlice { + ctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true) + } } } } @@ -157,10 +177,9 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v // Field: DynamicSlice ([]interface{}) // Dynamic slice []interface{} handling - manual deserialization { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.DynamicSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag sliceLen := int(buf.ReadVaruint32(err)) if sliceLen == 0 { v.DynamicSlice = make([]interface{}, 0) @@ -171,7 +190,27 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v v.DynamicSlice = make([]interface{}, sliceLen) // ReadData each element using ReadValue for i := range v.DynamicSlice { - ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem()) + ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem(), fory.RefModeTracking, true) + } + } + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.DynamicSlice = nil + } else { + sliceLen := int(buf.ReadVaruint32(err)) + if sliceLen == 0 { + v.DynamicSlice = make([]interface{}, 0) + } else { + // ReadData collection flags (ignore for now) + _ = buf.ReadInt8(err) + // Create slice with proper capacity + v.DynamicSlice = make([]interface{}, sliceLen) + // ReadData each element using ReadValue + for i := range v.DynamicSlice { + ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem(), fory.RefModeTracking, true) + } } } } @@ -264,11 +303,13 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem // WriteData fields in sorted order // Field: IntMap (map[int]int) { - if v.IntMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.IntMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.IntMap != nil { + mapLen = len(v.IntMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -300,15 +341,56 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.IntMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.IntMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.IntMap { + buf.WriteInt64(int64(mapKey)) + buf.WriteInt64(int64(mapValue)) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.IntMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } // Field: MixedMap (map[string]int) { - if v.MixedMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.MixedMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.MixedMap != nil { + mapLen = len(v.MixedMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -343,15 +425,59 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.MixedMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.MixedMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + if isRefTracking { + kvHeader |= 0x1 // track key ref + } + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.MixedMap { + ctx.WriteString(mapKey) + buf.WriteInt64(int64(mapValue)) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.MixedMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } // Field: StringMap (map[string]string) { - if v.StringMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.StringMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.StringMap != nil { + mapLen = len(v.StringMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -389,6 +515,51 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.StringMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.StringMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + if isRefTracking { + kvHeader |= 0x1 // track key ref + } + if isRefTracking { + kvHeader |= 0x8 // track value ref + } + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.StringMap { + ctx.WriteString(mapKey) + ctx.WriteString(mapValue) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.StringMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } return nil @@ -459,10 +630,9 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) // ReadData fields in same order as write // Field: IntMap (map[int]int) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.IntMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.IntMap = make(map[int]int) @@ -491,14 +661,48 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.IntMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.IntMap = make(map[int]int) + } else { + v.IntMap = make(map[int]int, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey int + mapKey = int(buf.ReadInt64(err)) + var mapValue int + mapValue = int(buf.ReadInt64(err)) + v.IntMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } // Field: MixedMap (map[string]int) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.MixedMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.MixedMap = make(map[string]int) @@ -527,14 +731,48 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.MixedMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.MixedMap = make(map[string]int) + } else { + v.MixedMap = make(map[string]int, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey string + mapKey = ctx.ReadString() + var mapValue int + mapValue = int(buf.ReadInt64(err)) + v.MixedMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } // Field: StringMap (map[string]string) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.StringMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.StringMap = make(map[string]string) @@ -563,6 +801,41 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.StringMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.StringMap = make(map[string]string) + } else { + v.StringMap = make(map[string]string, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey string + mapKey = ctx.ReadString() + var mapValue string + mapValue = ctx.ReadString() + v.StringMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } @@ -652,33 +925,62 @@ func (g *SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *Slic // WriteData fields in sorted order // Field: BoolSlice ([]bool) - if v.BoolSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteBoolSlice(buf, v.BoolSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteBoolSlice(buf, v.BoolSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.BoolSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteBoolSlice(buf, v.BoolSlice) + } + } } // Field: FloatSlice ([]float64) - if v.FloatSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteFloat64Slice(buf, v.FloatSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteFloat64Slice(buf, v.FloatSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.FloatSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteFloat64Slice(buf, v.FloatSlice) + } + } } // Field: IntSlice ([]int32) - if v.IntSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteInt32Slice(buf, v.IntSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteInt32Slice(buf, v.IntSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.IntSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteInt32Slice(buf, v.IntSlice) + } + } } // Field: StringSlice ([]string) { - if v.StringSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - sliceLen := len(v.StringSlice) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + sliceLen := 0 + if v.StringSlice != nil { + sliceLen = len(v.StringSlice) + } buf.WriteVaruint32(uint32(sliceLen)) if sliceLen > 0 { collectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType @@ -693,6 +995,28 @@ func (g *SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *Slic ctx.WriteString(elem) } } + } else { + // Native Go mode: slices are nullable, write null flag + if v.StringSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + sliceLen := len(v.StringSlice) + buf.WriteVaruint32(uint32(sliceLen)) + if sliceLen > 0 { + collectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType + if ctx.TrackRef() { + collectFlag |= 1 // CollectionTrackingRef for referencable element type + } + buf.WriteInt8(int8(collectFlag)) + for _, elem := range v.StringSlice { + if ctx.TrackRef() { + buf.WriteInt8(-1) // NotNullValueFlag for element + } + ctx.WriteString(elem) + } + } + } } } return nil @@ -763,37 +1087,57 @@ func (g *SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *SliceD // ReadData fields in same order as write // Field: BoolSlice ([]bool) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.BoolSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.BoolSlice = fory.ReadBoolSlice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.BoolSlice = nil + } else { + v.BoolSlice = fory.ReadBoolSlice(buf, err) + } } } // Field: FloatSlice ([]float64) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.FloatSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.FloatSlice = fory.ReadFloat64Slice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.FloatSlice = nil + } else { + v.FloatSlice = fory.ReadFloat64Slice(buf, err) + } } } // Field: IntSlice ([]int32) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.IntSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.IntSlice = fory.ReadInt32Slice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.IntSlice = nil + } else { + v.IntSlice = fory.ReadInt32Slice(buf, err) + } } } // Field: StringSlice ([]string) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.StringSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag sliceLen := int(buf.ReadVaruint32(err)) if sliceLen == 0 { v.StringSlice = make([]string, 0) @@ -826,6 +1170,45 @@ func (g *SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *SliceD } } } + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.StringSlice = nil + } else { + sliceLen := int(buf.ReadVaruint32(err)) + if sliceLen == 0 { + v.StringSlice = make([]string, 0) + } else { + collectFlag := buf.ReadInt8(err) + // Check if CollectionIsDeclElementType is set (bit 2, value 4) + hasDeclType := (collectFlag & 4) != 0 + // Check if CollectionTrackingRef is set (bit 0, value 1) + trackRefs := (collectFlag & 1) != 0 + v.StringSlice = make([]string, sliceLen) + if hasDeclType { + // Elements are written directly without type IDs + for i := 0; i < sliceLen; i++ { + if trackRefs { + _ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag) + } + v.StringSlice[i] = ctx.ReadString() + } + } else { + // Need to read type ID once if CollectionIsSameType is set + if (collectFlag & 8) != 0 { + // ReadData element type ID once for all elements + _ = buf.ReadVaruint32(err) + } + for i := 0; i < sliceLen; i++ { + if trackRefs { + _ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag) + } + v.StringSlice[i] = ctx.ReadString() + } + } + } + } } } diff --git a/go/fory/tests/xlang/xlang_test_main.go b/go/fory/tests/xlang/xlang_test_main.go index 5d36efeadb..b6d195bc1e 100644 --- a/go/fory/tests/xlang/xlang_test_main.go +++ b/go/fory/tests/xlang/xlang_test_main.go @@ -1862,6 +1862,162 @@ func testNullableFieldCompatibleNull() { writeFile(dataFile, serialized) } +// ============================================================================ +// Reference Tracking Test Types +// ============================================================================ + +// RefInnerSchemaConsistent - Inner struct for ref tracking tests in SCHEMA_CONSISTENT mode +// Matches Java's RefInnerSchemaConsistent (type id 501) +type RefInnerSchemaConsistent struct { + Id int32 + Name string +} + +// RefOuterSchemaConsistent - Outer struct for ref tracking tests in SCHEMA_CONSISTENT mode +// Both fields point to the same RefInnerSchemaConsistent instance +// Matches Java's RefOuterSchemaConsistent (type id 502) +type RefOuterSchemaConsistent struct { + Inner1 *RefInnerSchemaConsistent `fory:"ref,nullable"` + Inner2 *RefInnerSchemaConsistent `fory:"ref,nullable"` +} + +// RefInnerCompatible - Inner struct for ref tracking tests in COMPATIBLE mode +// Matches Java's RefInnerCompatible (type id 503) +type RefInnerCompatible struct { + Id int32 + Name string +} + +// RefOuterCompatible - Outer struct for ref tracking tests in COMPATIBLE mode +// Both fields point to the same RefInnerCompatible instance +// Matches Java's RefOuterCompatible (type id 504) +type RefOuterCompatible struct { + Inner1 *RefInnerCompatible `fory:"ref,nullable"` + Inner2 *RefInnerCompatible `fory:"ref,nullable"` +} + +func getRefOuterSchemaConsistent(obj interface{}) RefOuterSchemaConsistent { + switch v := obj.(type) { + case RefOuterSchemaConsistent: + return v + case *RefOuterSchemaConsistent: + return *v + default: + panic(fmt.Sprintf("expected RefOuterSchemaConsistent, got %T", obj)) + } +} + +func getRefOuterCompatible(obj interface{}) RefOuterCompatible { + switch v := obj.(type) { + case RefOuterCompatible: + return v + case *RefOuterCompatible: + return *v + default: + panic(fmt.Sprintf("expected RefOuterCompatible, got %T", obj)) + } +} + +// ============================================================================ +// Reference Tracking Tests +// ============================================================================ + +func testRefSchemaConsistent() { + dataFile := getDataFile() + data := readFile(dataFile) + + f := fory.New(fory.WithXlang(true), fory.WithCompatible(false), fory.WithRefTracking(true)) + f.Register(RefInnerSchemaConsistent{}, 501) + f.Register(RefOuterSchemaConsistent{}, 502) + + buf := fory.NewByteBuffer(data) + var obj interface{} + err := f.DeserializeWithCallbackBuffers(buf, &obj, nil) + if err != nil { + panic(fmt.Sprintf("Failed to deserialize: %v", err)) + } + + result := getRefOuterSchemaConsistent(obj) + + // Verify both fields have the expected values + if result.Inner1 == nil { + panic("Inner1 is nil") + } + if result.Inner2 == nil { + panic("Inner2 is nil") + } + assertEqual(int32(42), result.Inner1.Id, "Inner1.Id") + assertEqual("shared_inner", result.Inner1.Name, "Inner1.Name") + assertEqual(int32(42), result.Inner2.Id, "Inner2.Id") + assertEqual("shared_inner", result.Inner2.Name, "Inner2.Name") + + // Verify reference identity is preserved + if result.Inner1 != result.Inner2 { + panic("Reference tracking failed: Inner1 and Inner2 should point to the same object") + } + fmt.Println("Reference identity verified: Inner1 == Inner2") + + // Re-serialize with same shared reference + outer := &RefOuterSchemaConsistent{ + Inner1: result.Inner1, + Inner2: result.Inner1, // Use same reference + } + serialized, err := f.Serialize(outer) + if err != nil { + panic(fmt.Sprintf("Failed to serialize: %v", err)) + } + + writeFile(dataFile, serialized) +} + +func testRefCompatible() { + dataFile := getDataFile() + data := readFile(dataFile) + + f := fory.New(fory.WithXlang(true), fory.WithCompatible(true), fory.WithRefTracking(true)) + f.Register(RefInnerCompatible{}, 503) + f.Register(RefOuterCompatible{}, 504) + + buf := fory.NewByteBuffer(data) + var obj interface{} + err := f.DeserializeWithCallbackBuffers(buf, &obj, nil) + if err != nil { + panic(fmt.Sprintf("Failed to deserialize: %v", err)) + } + + result := getRefOuterCompatible(obj) + + // Verify both fields have the expected values + if result.Inner1 == nil { + panic("Inner1 is nil") + } + if result.Inner2 == nil { + panic("Inner2 is nil") + } + assertEqual(int32(99), result.Inner1.Id, "Inner1.Id") + assertEqual("compatible_shared", result.Inner1.Name, "Inner1.Name") + assertEqual(int32(99), result.Inner2.Id, "Inner2.Id") + assertEqual("compatible_shared", result.Inner2.Name, "Inner2.Name") + + // Verify reference identity is preserved + if result.Inner1 != result.Inner2 { + panic("Reference tracking failed: Inner1 and Inner2 should point to the same object") + } + fmt.Println("Reference identity verified: Inner1 == Inner2") + + // Re-serialize with same shared reference + outer := &RefOuterCompatible{ + Inner1: result.Inner1, + Inner2: result.Inner1, // Use same reference + } + serialized, err := f.Serialize(outer) + if err != nil { + panic(fmt.Sprintf("Failed to serialize: %v", err)) + } + + writeFile(dataFile, serialized) +} + // ============================================================================ // Main // ============================================================================ @@ -1957,6 +2113,10 @@ func main() { testNullableFieldCompatibleNotNull() case "test_nullable_field_compatible_null": testNullableFieldCompatibleNull() + case "test_ref_schema_consistent": + testRefSchemaConsistent() + case "test_ref_compatible": + testRefCompatible() default: panic(fmt.Sprintf("Unknown test case: %s", *caseName)) } diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 3090429693..78895f9c9a 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -431,20 +431,31 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { if err != nil { return nil, fmt.Errorf("failed to build field type for field %s: %w", fieldName, err) } - // Determine nullable based on Go type capability: - // - Pointer types (*T): can hold nil → nullable=true by default - // - Slices, maps, interfaces: can hold nil → nullable=true by default - // - Primitive types (int32, bool, etc.): cannot be nil → nullable=false - // Can be overridden by explicit fory tag + // Determine nullable based on mode: + // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. + // Only pointer types are nullable by default. + // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, + // so they are nullable by default. + // Can be overridden by explicit fory tag `fory:"nullable"` typeId := ft.TypeId() internalId := TypeId(typeId & 0xFF) isEnumField := internalId == ENUM || internalId == NAMED_ENUM - // Default nullable based on whether Go type can be nil - // Pointer types, slices, maps, interfaces can hold nil → nullable=true by default - nullableFlag := field.Type.Kind() == reflect.Ptr || - field.Type.Kind() == reflect.Slice || - field.Type.Kind() == reflect.Map || - field.Type.Kind() == reflect.Interface + // Determine nullable based on mode + // In xlang mode: only pointer types are nullable by default (per xlang spec) + // In native mode: Go's natural semantics - all nil-able types are nullable + // This ensures proper interoperability with Java/other languages in xlang mode. + var nullableFlag bool + if fory.config.IsXlang { + // xlang mode: only pointer types are nullable by default per xlang spec + // Slices and maps are NOT nullable - they serialize as empty when nil + nullableFlag = field.Type.Kind() == reflect.Ptr + } else { + // Native mode: Go's natural semantics - all nil-able types are nullable + nullableFlag = field.Type.Kind() == reflect.Ptr || + field.Type.Kind() == reflect.Slice || + field.Type.Kind() == reflect.Map || + field.Type.Kind() == reflect.Interface + } // Override nullable flag if explicitly set in fory tag if foryTag.NullableSet { nullableFlag = foryTag.Nullable @@ -455,10 +466,21 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { } // Calculate ref tracking - use tag override if explicitly set + // In xlang mode, registered types (primitives, strings) don't use ref tracking + // because they are value types, not reference types. trackingRef := fory.config.TrackRef if foryTag.RefSet { trackingRef = foryTag.Ref } + // Disable ref tracking for simple types (primitives, strings) in xlang mode + // These types don't benefit from ref tracking and Java doesn't expect ref flags for them + if fory.config.IsXlang && trackingRef { + // Check if this is a simple field type (primitives, strings, enums, etc.) + // SimpleFieldType represents built-in types that don't need ref tracking + if _, ok := ft.(*SimpleFieldType); ok { + trackingRef = false + } + } fieldInfo := FieldDef{ name: fieldName, @@ -592,8 +614,10 @@ func (b *BaseFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInf // This is called for top-level field types where flags are NOT embedded in the type ID func readFieldType(buffer *ByteBuffer, err *Error) (FieldType, error) { typeId := buffer.ReadVaruint32Small7(err) + // Use internal type ID (low byte) for switch, but store the full typeId + internalTypeId := TypeId(typeId & 0xFF) - switch typeId { + switch internalTypeId { case LIST, SET: // For nested types, flags ARE embedded in the type ID elementType, etErr := readFieldTypeWithFlags(buffer, err) @@ -626,8 +650,10 @@ func readFieldTypeWithFlags(buffer *ByteBuffer, err *Error) (FieldType, error) { // trackingRef := (rawValue & 0b1) != 0 // Not used currently // nullable := (rawValue & 0b10) != 0 // Not used currently typeId := rawValue >> 2 + // Use internal type ID (low byte) for switch, but store the full typeId + internalTypeId := TypeId(typeId & 0xFF) - switch typeId { + switch internalTypeId { case LIST, SET: elementType, etErr := readFieldTypeWithFlags(buffer, err) if etErr != nil { @@ -782,6 +808,7 @@ func (m *MapFieldType) getTypeInfo(f *Fory) (TypeInfo, error) { mapSerializer := &mapSerializer{ keySerializer: keyInfo.Serializer, valueSerializer: valueInfo.Serializer, + hasGenerics: true, } return TypeInfo{Type: mapType, Serializer: mapSerializer}, nil } @@ -802,6 +829,7 @@ func (m *MapFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo mapSerializer := &mapSerializer{ keySerializer: keyInfo.Serializer, valueSerializer: valueInfo.Serializer, + hasGenerics: true, } return TypeInfo{Type: mapType, Serializer: mapSerializer}, nil } diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 4511a0bb0e..18ed629fe1 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -299,6 +299,11 @@ func (r *TypeResolver) Compatible() bool { return r.fory.config.Compatible } +// IsXlang returns whether xlang (cross-language) mode is enabled +func (r *TypeResolver) IsXlang() bool { + return r.isXlang +} + func (r *TypeResolver) initialize() { serializers := []struct { reflect.Type @@ -748,17 +753,11 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, typeString := value.Type() typePtr := typePointer(typeString) if cachedInfo, ok := r.typePointerCache[typePtr]; ok { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] getTypeInfo: found in cache type=%v serializer=%T\n", typeString, cachedInfo.Serializer) - } return cachedInfo, nil } // Slow path: map lookup by reflect.Type if info, ok := r.typesInfo[typeString]; ok { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] getTypeInfo: found in typesInfo type=%v serializer=%T\n", typeString, info.Serializer) - } if info.Serializer == nil { /* Lazy initialize serializer if not created yet @@ -965,7 +964,7 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, serializer = &arrayConcreteValueSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(type_.Elem()), + referencable: isRefType(type_.Elem(), r.isXlang), } } } @@ -1348,7 +1347,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } // Always use xlang mode (LIST typeId) for non-primitive slices - return newSliceConcreteValueSerializer(type_, elemSerializer) + return newSliceSerializer(type_, elemSerializer, r.isXlang) } case reflect.Array: elem := type_.Elem() @@ -1388,7 +1387,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return &arrayConcreteValueSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(type_.Elem()), + referencable: isRefType(type_.Elem(), r.isXlang), }, nil } case reflect.Map: @@ -1412,16 +1411,19 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } } + // Determine key/value referencability using isRefType which handles xlang mode + keyReferencable := isRefType(type_.Key(), r.isXlang) + valueReferencable := isRefType(type_.Elem(), r.isXlang) return &mapSerializer{ type_: type_, keySerializer: keySerializer, valueSerializer: valueSerializer, - keyReferencable: nullable(type_.Key()), - valueReferencable: nullable(type_.Elem()), - mapInStruct: mapInStruct, + keyReferencable: keyReferencable, + valueReferencable: valueReferencable, + hasGenerics: mapInStruct, }, nil } else { - return mapSerializer{mapInStruct: mapInStruct}, nil + return mapSerializer{hasGenerics: mapInStruct}, nil } case reflect.Struct: serializer := r.typeToSerializers[type_] @@ -1452,7 +1454,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s // GetSliceSerializer returns the appropriate serializer for a slice type. // For primitive element types (bool, int8, int16, int32, int64, uint8, float32, float64), // it returns the dedicated primitive slice serializer that uses ARRAY protocol. -// For non-primitive element types, it returns sliceConcreteValueSerializer (LIST protocol). +// For non-primitive element types, it returns sliceSerializer (LIST protocol). func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, error) { if sliceType.Kind() != reflect.Slice { return nil, fmt.Errorf("expected slice type but got %s", sliceType.Kind()) @@ -1481,12 +1483,12 @@ func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, e case reflect.Uint: return uintSliceSerializer{}, nil } - // For non-primitive element types, use sliceConcreteValueSerializer + // For non-primitive element types, use sliceSerializer elemSerializer, err := r.getSerializerByType(elemType, false) if err != nil { return nil, err } - return newSliceConcreteValueSerializer(sliceType, elemSerializer) + return newSliceSerializer(sliceType, elemSerializer, r.isXlang) } // GetSetSerializer returns the setSerializer for a map[T]bool type (used to represent sets in Go). @@ -1502,7 +1504,7 @@ func (r *TypeResolver) GetSetSerializer(setType reflect.Type) (Serializer, error // GetArraySerializer returns the appropriate serializer for an array type. // For primitive element types, it returns the dedicated primitive array serializer (ARRAY protocol). -// For non-primitive element types, it returns sliceConcreteValueSerializer (LIST protocol). +// For non-primitive element types, it returns sliceSerializer (LIST protocol). func (r *TypeResolver) GetArraySerializer(arrayType reflect.Type) (Serializer, error) { if arrayType.Kind() != reflect.Array { return nil, fmt.Errorf("expected array type but got %s", arrayType.Kind()) @@ -1533,12 +1535,12 @@ func (r *TypeResolver) GetArraySerializer(arrayType reflect.Type) (Serializer, e } return int32ArraySerializer{arrayType: arrayType}, nil } - // For non-primitive element types, use sliceConcreteValueSerializer + // For non-primitive element types, use sliceSerializer elemSerializer, err := r.getSerializerByType(elemType, false) if err != nil { return nil, err } - return newSliceConcreteValueSerializer(arrayType, elemSerializer) + return newSliceSerializer(arrayType, elemSerializer, r.isXlang) } func isDynamicType(type_ reflect.Type) bool { diff --git a/go/fory/writer.go b/go/fory/writer.go index 08a7bfc496..510f4351ad 100644 --- a/go/fory/writer.go +++ b/go/fory/writer.go @@ -33,6 +33,7 @@ type WriteContext struct { buffer *ByteBuffer refWriter *RefWriter trackRef bool // Cached flag to avoid indirection + xlang bool // Cross-language serialization mode compatible bool // Schema evolution compatibility mode depth int maxDepth int @@ -43,6 +44,11 @@ type WriteContext struct { err Error // Accumulated error state for deferred checking } +// IsXlang returns whether cross-language serialization mode is enabled +func (c *WriteContext) IsXlang() bool { + return c.xlang +} + // NewWriteContext creates a new write context func NewWriteContext(trackRef bool, maxDepth int) *WriteContext { return &WriteContext{ @@ -559,10 +565,12 @@ func (c *WriteContext) WriteBufferObject(bufferObject BufferObject) { // If out-of-band, we just write false (already done above) and the data is handled externally } -// WriteValue writes a polymorphic value with reference tracking and type info. +// WriteValue writes a polymorphic value with configurable reference tracking and type info. // This is used when the concrete type is not known at compile time. -// Each serializer's Write method handles reference tracking internally. -func (c *WriteContext) WriteValue(value reflect.Value) { +// Parameters: +// - refMode: controls reference tracking behavior (RefModeNone, RefModeTracking, RefModeNullOnly) +// - writeType: if true, writes type info before the value +func (c *WriteContext) WriteValue(value reflect.Value, refMode RefMode, writeType bool) { // Handle interface values by getting their concrete element if value.Kind() == reflect.Interface { if !value.IsValid() || value.IsNil() { @@ -604,5 +612,5 @@ func (c *WriteContext) WriteValue(value reflect.Value) { } // Use serializer's Write method which handles ref tracking and type info internally - typeInfo.Serializer.Write(c, RefModeTracking, true, false, value) + typeInfo.Serializer.Write(c, refMode, writeType, false, value) } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 186d1c342e..2126a51731 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -484,44 +484,6 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } } - /** Write object class and data without tracking ref. */ - public void writeNullable(MemoryBuffer buffer, Object obj) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj); - } - } - - public void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - serializer.write(buffer, obj); - } - } - - /** Write object class and data without tracking ref. */ - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj, classInfo); - } - } - /** * Serialize a not-null and non-reference object to buffer. * @@ -534,6 +496,12 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { writeData(buffer, classInfo, obj); } + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + depth++; + serializer.write(buffer, obj); + depth--; + } + public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { classResolver.writeClassInfo(buffer, classInfo); Serializer serializer = classInfo.getSerializer(); @@ -549,6 +517,21 @@ public void xwriteRef(MemoryBuffer buffer, Object obj) { } } + public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + if (!refResolver.writeRefOrNull(buffer, obj)) { + ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder); + xtypeResolver.writeClassInfo(buffer, obj); + xwriteData(buffer, classInfo, obj); + } + } + + public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { + if (!refResolver.writeRefOrNull(buffer, obj)) { + xtypeResolver.writeClassInfo(buffer, obj); + xwriteData(buffer, classInfo, obj); + } + } + public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { if (serializer.needToWriteRef()) { if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -578,6 +561,13 @@ public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { xwriteData(buffer, classInfo, obj); } + public void xwriteNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + depth++; + serializer.xwrite(buffer, obj); + depth--; + ; + } + public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { switch (classInfo.getXtypeId()) { case Types.BOOL: @@ -591,14 +581,11 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { break; case Types.INT32: case Types.VAR_INT32: - // TODO(chaokunyang) support other encoding buffer.writeVarInt32((Integer) obj); break; case Types.INT64: case Types.VAR_INT64: - // TODO(chaokunyang) support other encoding case Types.SLI_INT64: - // TODO(chaokunyang) support varint encoding buffer.writeVarInt64((Long) obj); break; case Types.FLOAT32: @@ -983,6 +970,10 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { return readDataInternal(buffer, classResolver.readClassInfo(buffer, classInfoHolder)); } + public Object readNonRef(MemoryBuffer buffer, ClassInfo classInfo) { + return readDataInternal(buffer, classInfo); + } + /** Read object class and data without tracking ref. */ public Object readNullable(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); @@ -1064,6 +1055,19 @@ public Object xreadRef(MemoryBuffer buffer) { } } + public Object xreadRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + RefResolver refResolver = this.refResolver; + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = readDataInternal(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } + } + public Object xreadRef(MemoryBuffer buffer, Serializer serializer) { if (serializer.needToWriteRef()) { RefResolver refResolver = this.refResolver; diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java index 05ff0d7e5e..aae0721967 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java @@ -28,6 +28,26 @@ @Target({ElementType.FIELD, ElementType.METHOD}) public @interface ForyField { + /** Controls polymorphism behavior for struct fields in cross-language serialization. */ + enum Morphic { + /** + * Auto-detect based on declared type (default). + * + *
    + *
  • Xlang mode: only interface/abstract class are treated as POLYMORPHIC, concrete classes + * are treated as FINAL (no type info written) + *
  • Java native mode: all classes without {@code final} modifier are treated as POLYMORPHIC + *
+ */ + AUTO, + + /** Treat as final/sealed - no type info written, uses declared type's serializer directly. */ + FINAL, + + /** Treat as polymorphic - type info written to support subtypes at runtime. */ + POLYMORPHIC + } + /** * Field tag ID for schema evolution mode. * @@ -55,4 +75,18 @@ * defaults) */ boolean ref() default false; + + /** + * Controls polymorphism behavior for this field in cross-language serialization. + * + *
    + *
  • {@link Morphic#AUTO} (default): Interface/abstract types are polymorphic, concrete types + * are final + *
  • {@link Morphic#FINAL}: No type info written, uses declared type's serializer + *
  • {@link Morphic#POLYMORPHIC}: Type info written to support runtime subtypes + *
+ * + *

Default: AUTO (concrete struct types are final, interface/abstract are polymorphic) + */ + Morphic morphic() default Morphic.AUTO; } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 914a7ced48..7b7a04dd4e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -411,23 +411,23 @@ protected Expression serializeField( if (useRefTracking) { return new If( not(writeRefOrNull(buffer, fieldValue)), - serializeForNotNullForField(fieldValue, buffer, typeRef, null, false)); + serializeForNotNullForField(fieldValue, buffer, descriptor, null, false)); } else { // if typeToken is not final, ref tracking of subclass will be ignored too. if (typeRef.isPrimitive()) { - return serializeForNotNullForField(fieldValue, buffer, typeRef, null, false); + return serializeForNotNullForField(fieldValue, buffer, descriptor, null, false); } if (nullable) { Expression action = new ListExpression( new Invoke(buffer, "writeByte", Literal.ofByte(Fory.NOT_NULL_VALUE_FLAG)), - serializeForNotNullForField(fieldValue, buffer, typeRef, null, false)); + serializeForNotNullForField(fieldValue, buffer, descriptor, null, false)); return new If( eqNull(fieldValue), new Invoke(buffer, "writeByte", Literal.ofByte(Fory.NULL_FLAG)), action); } else { - return serializeForNotNullForField(fieldValue, buffer, typeRef, null, false); + return serializeForNotNullForField(fieldValue, buffer, descriptor, null, false); } } } @@ -435,9 +435,10 @@ protected Expression serializeField( private Expression serializeForNotNullForField( Expression inputObject, Expression buffer, - TypeRef typeRef, + Descriptor descriptor, Expression serializer, boolean generateNewMethod) { + TypeRef typeRef = descriptor.getTypeRef(); Class clz = getRawType(typeRef); if (isPrimitive(clz) || isBoxed(clz)) { return serializePrimitive(inputObject, buffer, clz); @@ -452,7 +453,7 @@ private Expression serializeForNotNullForField( } else if (useMapSerialization(typeRef)) { action = serializeForMap(buffer, inputObject, typeRef, serializer, generateNewMethod); } else { - action = serializeForNotNullObjectForField(inputObject, buffer, typeRef, serializer); + action = serializeForNotNullObjectForField(inputObject, buffer, descriptor, serializer); } return action; } @@ -483,12 +484,13 @@ private Expression serializePrimitive(Expression inputObject, Expression buffer, } private Expression serializeForNotNullObjectForField( - Expression inputObject, Expression buffer, TypeRef typeRef, Expression serializer) { + Expression inputObject, Expression buffer, Descriptor descriptor, Expression serializer) { + TypeRef typeRef = descriptor.getTypeRef(); Class clz = getRawType(typeRef); if (serializer != null) { return new Invoke(serializer, writeMethodName, buffer, inputObject); } - if (isMonomorphic(clz)) { + if (isMonomorphic(descriptor)) { // Use descriptor to get the appropriate serializer serializer = getSerializerForField(clz); return new Invoke(serializer, writeMethodName, buffer, inputObject); @@ -613,12 +615,18 @@ protected boolean useMapSerialization(Class type) { * the method can still return false. For example, we return false in meta share mode to write * class defs for the non-inner final types. */ - protected abstract boolean isMonomorphic(Class clz); + protected boolean isMonomorphic(Class clz) { + return typeResolver(r -> r.isMonomorphic(clz)); + } protected boolean isMonomorphic(TypeRef typeRef) { return isMonomorphic(typeRef.getRawType()); } + protected boolean isMonomorphic(Descriptor descriptor) { + return typeResolver(r -> r.isMonomorphic(descriptor)); + } + protected Expression serializeForNotNullObject( Expression inputObject, Expression buffer, TypeRef typeRef, Expression serializer) { Class clz = getRawType(typeRef); @@ -1803,10 +1811,11 @@ protected Expression deserializeField( boolean typeNeedsRef = needWriteRef(typeRef); if (useRefTracking) { - return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, typeRef, null)); + return readRef( + buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (!nullable) { - Expression value = deserializeForNotNullForField(buffer, typeRef, null); + Expression value = deserializeForNotNullForField(buffer, descriptor, null); if (typeNeedsRef) { // When a field explicitly disables ref tracking (@ForyField(trackingRef=false)) @@ -1828,7 +1837,7 @@ protected Expression deserializeField( buffer, typeRef, callback, - () -> deserializeForNotNullForField(buffer, typeRef, null), + () -> deserializeForNotNullForField(buffer, descriptor, null), true, localFieldType); @@ -1842,7 +1851,8 @@ protected Expression deserializeField( } private Expression deserializeForNotNullForField( - Expression buffer, TypeRef typeRef, Expression serializer) { + Expression buffer, Descriptor descriptor, Expression serializer) { + TypeRef typeRef = descriptor.getTypeRef(); Class cls = getRawType(typeRef); if (isPrimitive(cls) || isBoxed(cls)) { return deserializePrimitive(buffer, cls); @@ -1859,7 +1869,7 @@ private Expression deserializeForNotNullForField( if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); } - if (isMonomorphic(cls)) { + if (isMonomorphic(descriptor)) { // Use descriptor to get the appropriate serializer serializer = getSerializerForField(cls); Class returnType = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index ada1a19fe9..865ded99cc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -91,23 +91,18 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) this.classDef = classDef; Collection descriptors = fory( - f -> - MetaSharedSerializer.consolidateFields( - f.isCrossLanguage() ? f.getXtypeResolver() : f.getClassResolver(), - beanClass, - classDef)); + f -> MetaSharedSerializer.consolidateFields(f._getTypeResolver(), beanClass, classDef)); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.debugOutputEnabled()) { LOG.info("========== sorted descriptors for {} ==========", classDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } objectCodecOptimizer = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 6e999d54d4..1e528094b4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -111,12 +111,11 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { List sortedDescriptors = grouper.getSortedDescriptors(); for (Descriptor d : sortedDescriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } classVersionHash = @@ -155,12 +154,6 @@ protected void addCommonImports() { ctx.addImport(Generated.GeneratedObjectSerializer.class); } - /** Mark non-inner registered final types as non-final to write class def for those types. */ - @Override - protected boolean isMonomorphic(Class clz) { - return typeResolver(r -> r.isMonomorphic(clz)); - } - /** * Return an expression that serialize java bean of type {@link CodecBuilder#beanClass} to buffer. */ @@ -180,7 +173,7 @@ public Expression buildEncodeExpression() { addGroupExpressions( objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer); addGroupExpressions( - objectCodecOptimizer.finalWriteGroups, numGroups, expressions, bean, buffer); + objectCodecOptimizer.buildInWriteGroups, numGroups, expressions, bean, buffer); for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(serializeGroup(Collections.singletonList(descriptor), bean, buffer, false)); @@ -210,7 +203,7 @@ private void addGroupExpressions( private int getNumGroups(ObjectCodecOptimizer objectCodecOptimizer) { return objectCodecOptimizer.boxedWriteGroups.size() - + objectCodecOptimizer.finalWriteGroups.size() + + objectCodecOptimizer.buildInWriteGroups.size() + objectCodecOptimizer.otherWriteGroups.size() + objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors().size() + objectCodecOptimizer.descriptorGrouper.getMapDescriptors().size(); @@ -473,7 +466,7 @@ public Expression buildDecodeExpression() { deserializeReadGroup( objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer); deserializeReadGroup( - objectCodecOptimizer.finalReadGroups, numGroups, expressions, bean, buffer); + objectCodecOptimizer.buildInReadGroups, numGroups, expressions, bean, buffer); for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false)); } @@ -611,7 +604,7 @@ private Expression checkClassVersion(Expression buffer) { "checkClassVersion", PRIMITIVE_VOID_TYPE, false, - foryRef, + beanClassExpr(), inlineInvoke(buffer, readIntFunc(), PRIMITIVE_INT_TYPE), Objects.requireNonNull(classVersionHash)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java index 992370f759..95c5efe7a8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java @@ -70,8 +70,8 @@ public class ObjectCodecOptimizer extends ExpressionOptimizer { final List> primitiveGroups = new ArrayList<>(); final List> boxedWriteGroups = new ArrayList<>(); final List> boxedReadGroups = new ArrayList<>(); - final List> finalWriteGroups = new ArrayList<>(); - final List> finalReadGroups = new ArrayList<>(); + final List> buildInWriteGroups = new ArrayList<>(); + final List> buildInReadGroups = new ArrayList<>(); final List> otherWriteGroups = new ArrayList<>(); final List> otherReadGroups = new ArrayList<>(); @@ -117,9 +117,9 @@ private void buildGroups() { boxedReadWeight, boxedReadGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getFinalDescriptors()), 9, finalWriteGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 9, buildInWriteGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getFinalDescriptors()), 5, finalReadGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 5, buildInReadGroups), MutableTuple3.of( new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 4, otherReadGroups), MutableTuple3.of( diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index ce6bc390a4..d84490b3b2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -1866,8 +1866,13 @@ public ExprCode doGenCode(CodegenContext ctx) { falseEval.value()); } codeBuilder.append(StringUtils.stripBlankLines(ifCode)); - return new ExprCode( - codeBuilder.toString(), Code.isNullVariable(isNull), Code.variable(rawType, value)); + if (nullable) { + return new ExprCode( + codeBuilder.toString(), Code.isNullVariable(isNull), Code.variable(rawType, value)); + } else { + // When not nullable, return FalseLiteral instead of a variable that was never declared + return new ExprCode(codeBuilder.toString(), FalseLiteral, Code.variable(rawType, value)); + } } else { String ifCode; if (falseExpr != null) { @@ -2762,4 +2767,40 @@ public String toString() { return String.format("%s = %s", from, to); } } + + /** Expression for Java instanceof check. */ + class InstanceOf extends ValueExpression { + private Expression target; + private final TypeRef checkType; + + public InstanceOf(Expression target, TypeRef checkType) { + super(new Expression[] {target}); + this.target = target; + this.checkType = checkType; + this.inlineCall = true; + } + + @Override + public TypeRef type() { + return PRIMITIVE_BOOLEAN_TYPE; + } + + @Override + public ExprCode doGenCode(CodegenContext ctx) { + ExprCode targetCode = target.genCode(ctx); + String value = String.format("(%s instanceof %s)", targetCode.value(), ctx.type(checkType)); + String code = StringUtils.isBlank(targetCode.code()) ? null : targetCode.code(); + return new ExprCode(code, FalseLiteral, Code.variable(boolean.class, value)); + } + + @Override + public boolean nullable() { + return false; + } + + @Override + public String toString() { + return String.format("%s instanceof %s", target, checkType); + } + } } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index b3f062a1cc..8b057c5e3f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -74,7 +74,7 @@ static List buildFields(Fory fory, Class cls, boolean resolveParent) { .getBoxedDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); descriptorGrouper - .getFinalDescriptors() + .getBuildInDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); descriptorGrouper .getOtherDescriptors() diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 95f90637f0..c026bdeaa7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -78,6 +78,7 @@ import org.apache.fory.Fory; import org.apache.fory.ForyCopyable; import org.apache.fory.annotation.CodegenInvoke; +import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; @@ -687,6 +688,22 @@ public String getTypeAlias(Class cls) { return cls.getName(); } + @Override + public boolean isMonomorphic(Descriptor descriptor) { + ForyField foryField = descriptor.getForyField(); + if (foryField != null) { + switch (foryField.morphic()) { + case POLYMORPHIC: + return false; + case FINAL: + return true; + default: + return isMonomorphic(descriptor.getRawType()); + } + } + return isMonomorphic(descriptor.getRawType()); + } + /** * Mark non-inner registered final types as non-final to write class def for those types. Note if * a class is registered but not an inner class with inner serializer, it will still be taken as @@ -717,6 +734,10 @@ public boolean isMonomorphic(Class clz) { return ReflectionUtils.isMonomorphic(clz); } + public boolean isBuildIn(Descriptor descriptor) { + return isMonomorphic(descriptor); + } + /** Returns true if cls is fory inner registered class. */ boolean isInnerClass(Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); @@ -1794,15 +1815,6 @@ public void resetRead() {} public void resetWrite() {} - private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); - - @CodegenInvoke - public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { - Map map = - extRegistry.classGenericTypes.computeIfAbsent(cls, this::buildGenericMap); - return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); - } - @Override public GenericType buildGenericType(TypeRef typeRef) { return GenericType.build( @@ -1930,7 +1942,7 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - fory.getClassResolver()::isMonomorphic, + this::isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator, diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 24600e1695..73bd5f7ec9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import org.apache.fory.Fory; +import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; @@ -97,6 +98,7 @@ public abstract class TypeResolver { static final String SET_META__CONTEXT_MSG = "Meta context must be set before serialization, " + "please set meta context by SerializationContext.setMetaContext"; + private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); final Fory fory; final boolean metaContextShareEnabled; @@ -223,6 +225,10 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract boolean isRegisteredByName(Class cls); + public abstract boolean isBuildIn(Descriptor descriptor); + + public abstract boolean isMonomorphic(Descriptor descriptor); + public abstract boolean isMonomorphic(Class clz); public abstract ClassInfo getClassInfo(Class cls); @@ -479,6 +485,13 @@ final Class loadClass( public abstract GenericType buildGenericType(Type type); + @CodegenInvoke + public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { + Map map = + extRegistry.classGenericTypes.computeIfAbsent(cls, this::buildGenericMap); + return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); + } + public abstract void initialize(); public abstract ClassDef getTypeDef(Class cls, boolean resolveParent); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 08abaa2011..a4fc4079e7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; @@ -433,6 +434,28 @@ public boolean isRegisteredByName(Class cls) { } } + @Override + public boolean isMonomorphic(Descriptor descriptor) { + ForyField foryField = descriptor.getForyField(); + ForyField.Morphic morphic = foryField != null ? foryField.morphic() : ForyField.Morphic.AUTO; + switch (morphic) { + case POLYMORPHIC: + return false; + case FINAL: + return true; + default: + Class rawType = descriptor.getRawType(); + if (rawType.isEnum()) { + return true; + } + byte xtypeId = getXtypeId(rawType); + if (fory.isCompatible()) { + return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; + } + return xtypeId != Types.UNKNOWN; + } + } + @Override public boolean isMonomorphic(Class clz) { if (TypeUtils.unwrap(clz).isPrimitive() || clz.isEnum() || clz == String.class) { @@ -459,6 +482,12 @@ public boolean isMonomorphic(Class clz) { return false; } + public boolean isBuildIn(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + byte xtypeId = getXtypeId(rawType); + return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; + } + @Override public ClassInfo getClassInfo(Class cls) { ClassInfo classInfo = classInfoMap.get(cls); @@ -920,19 +949,7 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - clz -> { - ClassInfo classInfo = getClassInfo(clz, false); - if (classInfo == null || clz.isEnum()) { - return false; - } - byte foryTypeId = (byte) (classInfo.xtypeId & 0xff); - if (foryTypeId == 0 - || foryTypeId == Types.UNKNOWN - || Types.isUserDefinedType(foryTypeId)) { - return false; - } - return foryTypeId != Types.LIST && foryTypeId != Types.SET && foryTypeId != Types.MAP; - }, + this::isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator, @@ -952,9 +969,7 @@ public DescriptorGrouper createDescriptorGrouper( .sort(); } - private static final int UNKNOWN_TYPE_ID = Types.UNKNOWN; - - private int getXtypeId(Class cls) { + private byte getXtypeId(Class cls) { if (isSet(cls)) { return Types.SET; } @@ -967,8 +982,8 @@ private int getXtypeId(Class cls) { if (isMap(cls)) { return Types.MAP; } - if (fory.getXtypeResolver().isRegistered(cls)) { - return fory.getXtypeResolver().getClassInfo(cls).getXtypeId(); + if (isRegistered(cls)) { + return (byte) (getClassInfo(cls).getXtypeId() & 0xff); } else { if (cls.isEnum()) { return Types.ENUM; @@ -979,7 +994,7 @@ private int getXtypeId(Class cls) { if (ReflectionUtils.isMonomorphic(cls)) { throw new UnsupportedOperationException(cls + " is not supported for xlang serialization"); } - return UNKNOWN_TYPE_ID; + return Types.UNKNOWN; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index b11e071ed7..6c0a0ee193 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -23,12 +23,9 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import org.apache.fory.Fory; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.FieldAccessor; @@ -37,19 +34,12 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.serializer.converter.FieldConverter; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.FinalObjectTypeStub; -import org.apache.fory.type.GenericType; import org.apache.fory.type.Generics; -import org.apache.fory.type.TypeUtils; -import org.apache.fory.util.StringUtils; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -57,9 +47,10 @@ public abstract class AbstractObjectSerializer extends Serializer { protected final RefResolver refResolver; protected final ClassResolver classResolver; + protected final TypeResolver typeResolver; protected final boolean isRecord; protected final ObjectCreator objectCreator; - private InternalFieldInfo[] fieldInfos; + private FieldGroups.SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; public AbstractObjectSerializer(Fory fory, Class type) { @@ -70,203 +61,129 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec super(fory, type); this.refResolver = fory.getRefResolver(); this.classResolver = fory.getClassResolver(); + this.typeResolver = fory._getTypeResolver(); this.isRecord = RecordUtils.isRecord(type); this.objectCreator = objectCreator; } - /** - * Read final object field value. Note that primitive field value can't be read by this method, - * because primitive field doesn't write null flag. - */ - static Object readFinalObjectFieldValue( + static void writeOtherFieldValue( SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - FinalTypeField fieldInfo, - boolean isFinal, - MemoryBuffer buffer) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); - binding.incReadDepth(); - Object fieldValue; - if (isFinal) { + MemoryBuffer buffer, + FieldGroups.SerializationFieldInfo fieldInfo, + Object fieldValue) { + if (fieldInfo.useDeclaredTypeInfo) { switch (fieldInfo.refMode) { case NONE: - fieldValue = binding.read(buffer, serializer); + binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); break; case NULL_ONLY: - fieldValue = binding.readNullable(buffer, serializer); + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); + } break; case TRACKING: - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - fieldValue = binding.readRef(buffer, serializer); + binding.writeRef(buffer, fieldValue, fieldInfo.serializer); break; default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } } else { switch (fieldInfo.refMode) { case NONE: - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); + binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); break; case NULL_ONLY: - { - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - binding.decDepth(); - return null; - } - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); } break; case TRACKING: - { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); - refResolver.setReadObject(nextReadRefId, fieldValue); - } else { - fieldValue = refResolver.getReadObject(); - } - } + binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); break; default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } } - binding.decDepth(); - return fieldValue; - } - - /** - * Read a non-container field value that is not a final type. Handles enum types, reference - * tracking, and nullable fields according to xlang serialization protocol. - * - * @param binding the serialization binding for read operations - * @param fieldInfo the field metadata including type info and nullability - * @param buffer the buffer to read from - * @return the deserialized field value, or null if the field is nullable and was null - */ - static Object readOtherFieldValue( - SerializationBinding binding, GenericTypeField fieldInfo, MemoryBuffer buffer) { - // Note: Enum has special handling for xlang compatibility - no type info for enum fields - if (fieldInfo.genericType.getCls().isEnum()) { - // Only read null flag when the field is nullable (for xlang compatibility) - if (fieldInfo.nullable && buffer.readByte() == Fory.NULL_FLAG) { - return null; - } - return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); - } - Object fieldValue; - switch (fieldInfo.refMode) { - case NONE: - binding.preserveRefId(-1); - fieldValue = binding.readNonRef(buffer, fieldInfo); - break; - case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } - fieldValue = binding.readNonRef(buffer, fieldInfo); - } - break; - case TRACKING: - fieldValue = binding.readRef(buffer, fieldInfo); - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - return fieldValue; } - /** - * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, - * and pushes/pops generic type information for proper deserialization of parameterized types. - * - * @param binding the serialization binding for read operations - * @param generics the generics context for tracking parameterized types - * @param fieldInfo the field metadata including generic type info and nullability - * @param buffer the buffer to read from - * @return the deserialized container field value, or null if the field is nullable and was null - */ - static Object readContainerFieldValue( + static void writeContainerFieldValue( SerializationBinding binding, + RefResolver refResolver, + TypeResolver typeResolver, Generics generics, - GenericTypeField fieldInfo, - MemoryBuffer buffer) { - Object fieldValue; + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object fieldValue) { switch (fieldInfo.refMode) { case NONE: - binding.preserveRefId(-1); generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + binding.writeContainerFieldValue( + buffer, + fieldValue, + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); generics.popGenericType(); break; case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + binding.writeContainerFieldValue( + buffer, + fieldValue, + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); generics.popGenericType(); } break; case TRACKING: - generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); - generics.popGenericType(); + if (!refResolver.writeRefOrNull(buffer, fieldValue)) { + ClassInfo classInfo = + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); + generics.pushGenericType(fieldInfo.genericType); + binding.writeContainerFieldValue(buffer, fieldValue, classInfo); + generics.popGenericType(); + } break; default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } - return fieldValue; } /** - * Write a primitive field value to buffer using the field accessor. + * Write a primitive field value to buffer using direct memory offset access. * * @param fory the fory instance for compression settings * @param buffer the buffer to write to * @param targetObject the object containing the field - * @param fieldAccessor the accessor to get the field value + * @param fieldOffset the memory offset of the field * @param classId the class ID of the primitive type * @return true if classId is not a primitive type and needs further write handling */ static boolean writePrimitiveFieldValue( - Fory fory, - MemoryBuffer buffer, - Object targetObject, - FieldAccessor fieldAccessor, - short classId) { - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); - } + Fory fory, MemoryBuffer buffer, Object targetObject, long fieldOffset, short classId) { switch (classId) { case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: - buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); + buffer.writeBoolean(Platform.getBoolean(targetObject, fieldOffset)); return false; case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: - buffer.writeByte((Byte) fieldAccessor.get(targetObject)); + buffer.writeByte(Platform.getByte(targetObject, fieldOffset)); return false; case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: - buffer.writeChar((Character) fieldAccessor.get(targetObject)); + buffer.writeChar(Platform.getChar(targetObject, fieldOffset)); return false; case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: - buffer.writeInt16((Short) fieldAccessor.get(targetObject)); + buffer.writeInt16(Platform.getShort(targetObject, fieldOffset)); return false; case ClassResolver.PRIMITIVE_INT_CLASS_ID: { - int fieldValue = (Integer) fieldAccessor.get(targetObject); + int fieldValue = Platform.getInt(targetObject, fieldOffset); if (fory.compressInt()) { buffer.writeVarInt32(fieldValue); } else { @@ -275,16 +192,16 @@ static boolean writePrimitiveFieldValue( return false; } case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: - buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); + buffer.writeFloat32(Platform.getFloat(targetObject, fieldOffset)); return false; case ClassResolver.PRIMITIVE_LONG_CLASS_ID: { - long fieldValue = (long) fieldAccessor.get(targetObject); + long fieldValue = Platform.getLong(targetObject, fieldOffset); fory.writeInt64(buffer, fieldValue); return false; } case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: - buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); + buffer.writeFloat64(Platform.getDouble(targetObject, fieldOffset)); return false; default: return true; @@ -292,33 +209,41 @@ static boolean writePrimitiveFieldValue( } /** - * Write a primitive field value to buffer using direct memory offset access. + * Write a primitive field value to buffer using the field accessor. * * @param fory the fory instance for compression settings * @param buffer the buffer to write to * @param targetObject the object containing the field - * @param fieldOffset the memory offset of the field + * @param fieldAccessor the accessor to get the field value * @param classId the class ID of the primitive type * @return true if classId is not a primitive type and needs further write handling */ static boolean writePrimitiveFieldValue( - Fory fory, MemoryBuffer buffer, Object targetObject, long fieldOffset, short classId) { + Fory fory, + MemoryBuffer buffer, + Object targetObject, + FieldAccessor fieldAccessor, + short classId) { + long fieldOffset = fieldAccessor.getFieldOffset(); + if (fieldOffset != -1) { + return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); + } switch (classId) { case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: - buffer.writeBoolean(Platform.getBoolean(targetObject, fieldOffset)); + buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); return false; case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: - buffer.writeByte(Platform.getByte(targetObject, fieldOffset)); + buffer.writeByte((Byte) fieldAccessor.get(targetObject)); return false; case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: - buffer.writeChar(Platform.getChar(targetObject, fieldOffset)); + buffer.writeChar((Character) fieldAccessor.get(targetObject)); return false; case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: - buffer.writeInt16(Platform.getShort(targetObject, fieldOffset)); + buffer.writeInt16((Short) fieldAccessor.get(targetObject)); return false; case ClassResolver.PRIMITIVE_INT_CLASS_ID: { - int fieldValue = Platform.getInt(targetObject, fieldOffset); + int fieldValue = (Integer) fieldAccessor.get(targetObject); if (fory.compressInt()) { buffer.writeVarInt32(fieldValue); } else { @@ -327,16 +252,16 @@ static boolean writePrimitiveFieldValue( return false; } case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: - buffer.writeFloat32(Platform.getFloat(targetObject, fieldOffset)); + buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); return false; case ClassResolver.PRIMITIVE_LONG_CLASS_ID: { - long fieldValue = Platform.getLong(targetObject, fieldOffset); + long fieldValue = (long) fieldAccessor.get(targetObject); fory.writeInt64(buffer, fieldValue); return false; } case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: - buffer.writeFloat64(Platform.getDouble(targetObject, fieldOffset)); + buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); return false; default: return true; @@ -527,6 +452,164 @@ static boolean writeBasicNullableObjectFieldValue( } } + /** + * Read final object field value. Note that primitive field value can't be read by this method, + * because primitive field doesn't write null flag. + */ + static Object readFinalObjectFieldValue( + SerializationBinding binding, + RefResolver refResolver, + TypeResolver typeResolver, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + Serializer serializer = fieldInfo.classInfo.getSerializer(); + binding.incReadDepth(); + Object fieldValue; + if (fieldInfo.useDeclaredTypeInfo) { + switch (fieldInfo.refMode) { + case NONE: + fieldValue = binding.read(buffer, serializer); + break; + case NULL_ONLY: + fieldValue = binding.readNullable(buffer, serializer); + break; + case TRACKING: + // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still + // consistent with jit serializer. + fieldValue = binding.readRef(buffer, serializer); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + } else { + switch (fieldInfo.refMode) { + case NONE: + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + break; + case NULL_ONLY: + { + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + binding.decDepth(); + return null; + } + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + } + break; + case TRACKING: + { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + refResolver.setReadObject(nextReadRefId, fieldValue); + } else { + fieldValue = refResolver.getReadObject(); + } + } + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + } + binding.decDepth(); + return fieldValue; + } + + /** + * Read a non-container field value that is not a final type. Handles enum types, reference + * tracking, and nullable fields according to xlang serialization protocol. + * + * @param binding the serialization binding for read operations + * @param fieldInfo the field metadata including type info and nullability + * @param buffer the buffer to read from + * @return the deserialized field value, or null if the field is nullable and was null + */ + static Object readOtherFieldValue( + SerializationBinding binding, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + // Note: Enum has special handling for xlang compatibility - no type info for enum fields + if (fieldInfo.genericType.getCls().isEnum()) { + // Only read null flag when the field is nullable (for xlang compatibility) + if (fieldInfo.nullable && buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); + } + Object fieldValue; + switch (fieldInfo.refMode) { + case NONE: + binding.preserveRefId(-1); + fieldValue = binding.readNonRef(buffer, fieldInfo); + break; + case NULL_ONLY: + { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; + } + fieldValue = binding.readNonRef(buffer, fieldInfo); + } + break; + case TRACKING: + fieldValue = binding.readRef(buffer, fieldInfo); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + return fieldValue; + } + + /** + * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, + * and pushes/pops generic type information for proper deserialization of parameterized types. + * + * @param binding the serialization binding for read operations + * @param generics the generics context for tracking parameterized types + * @param fieldInfo the field metadata including generic type info and nullability + * @param buffer the buffer to read from + * @return the deserialized container field value, or null if the field is nullable and was null + */ + static Object readContainerFieldValue( + SerializationBinding binding, + Generics generics, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + Object fieldValue; + switch (fieldInfo.refMode) { + case NONE: + binding.preserveRefId(-1); + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); + break; + case NULL_ONLY: + { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; + } + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); + } + break; + case TRACKING: + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); + generics.popGenericType(); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + return fieldValue; + } + /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. @@ -856,13 +939,13 @@ private T copyRecord(T originObj) { } private Object[] copyFields(T originObj) { - InternalFieldInfo[] fieldInfos = this.fieldInfos; + FieldGroups.SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); } Object[] fieldValues = new Object[fieldInfos.length]; for (int i = 0; i < fieldInfos.length; i++) { - InternalFieldInfo fieldInfo = fieldInfos[i]; + FieldGroups.SerializationFieldInfo fieldInfo = fieldInfos[i]; FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); if (fieldOffset != -1) { @@ -877,11 +960,11 @@ private Object[] copyFields(T originObj) { } private void copyFields(T originObj, T newObj) { - InternalFieldInfo[] fieldInfos = this.fieldInfos; + FieldGroups.SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); } - for (InternalFieldInfo fieldInfo : fieldInfos) { + for (FieldGroups.SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); // record class won't go to this path; @@ -930,8 +1013,8 @@ private void copyFields(T originObj, T newObj) { } public static void copyFields( - Fory fory, InternalFieldInfo[] fieldInfos, Object originObj, Object newObj) { - for (InternalFieldInfo fieldInfo : fieldInfos) { + Fory fory, FieldGroups.SerializationFieldInfo[] fieldInfos, Object originObj, Object newObj) { + for (FieldGroups.SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); // record class won't go to this path; @@ -1012,7 +1095,7 @@ private Object copyField(Object targetObject, long fieldOffset, short classId) { } } - private InternalFieldInfo[] buildFieldsInfo() { + private FieldGroups.SerializationFieldInfo[] buildFieldsInfo() { List descriptors = new ArrayList<>(); if (RecordUtils.isRecord(type)) { RecordComponent[] components = RecordUtils.getRecordComponents(type); @@ -1037,12 +1120,8 @@ private InternalFieldInfo[] buildFieldsInfo() { } DescriptorGrouper descriptorGrouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, descriptorGrouper); - fieldInfos = new InternalFieldInfo[descriptors.size()]; - System.arraycopy(infos.f0.f0, 0, fieldInfos, 0, infos.f0.f0.length); - System.arraycopy(infos.f1, 0, fieldInfos, infos.f0.f0.length, infos.f1.length); - System.arraycopy(infos.f2, 0, fieldInfos, fieldInfos.length - infos.f2.length, infos.f2.length); + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + fieldInfos = fieldGroups.allFields; if (isRecord) { List fieldNames = Arrays.stream(fieldInfos) @@ -1053,206 +1132,7 @@ private InternalFieldInfo[] buildFieldsInfo() { return fieldInfos; } - public static InternalFieldInfo[] buildFieldsInfo(Fory fory, List fields) { - List descriptors = new ArrayList<>(); - for (Field field : fields) { - if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) { - descriptors.add(new Descriptor(field, TypeRef.of(field.getGenericType()), null, null)); - } - } - DescriptorGrouper descriptorGrouper = - fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, descriptorGrouper); - InternalFieldInfo[] fieldInfos = new InternalFieldInfo[descriptors.size()]; - System.arraycopy(infos.f0.f0, 0, fieldInfos, 0, infos.f0.f0.length); - System.arraycopy(infos.f1, 0, fieldInfos, infos.f0.f0.length, infos.f1.length); - System.arraycopy(infos.f2, 0, fieldInfos, fieldInfos.length - infos.f2.length, infos.f2.length); - return fieldInfos; - } - protected T newBean() { return objectCreator.newInstance(); } - - static Tuple3, GenericTypeField[], GenericTypeField[]> - buildFieldInfos(Fory fory, DescriptorGrouper grouper) { - // When a type is both Collection/Map and final, add it to collection/map fields to keep - // consistent with jit. - Collection primitives = grouper.getPrimitiveDescriptors(); - Collection boxed = grouper.getBoxedDescriptors(); - Collection finals = grouper.getFinalDescriptors(); - FinalTypeField[] finalFields = - new FinalTypeField[primitives.size() + boxed.size() + finals.size()]; - int cnt = 0; - for (Descriptor d : primitives) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - for (Descriptor d : boxed) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - // TODO(chaokunyang) Support Pojo generics besides Map/Collection subclass - // when it's supported in BaseObjectCodecBuilder. - for (Descriptor d : finals) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - boolean[] isFinal = new boolean[finalFields.length]; - for (int i = 0; i < isFinal.length; i++) { - ClassInfo classInfo = finalFields[i].classInfo; - isFinal[i] = classInfo != null && fory.getClassResolver().isMonomorphic(classInfo.getCls()); - } - cnt = 0; - GenericTypeField[] otherFields = new GenericTypeField[grouper.getOtherDescriptors().size()]; - for (Descriptor descriptor : grouper.getOtherDescriptors()) { - GenericTypeField genericTypeField = new GenericTypeField(fory, descriptor); - otherFields[cnt++] = genericTypeField; - } - cnt = 0; - Collection collections = grouper.getCollectionDescriptors(); - Collection maps = grouper.getMapDescriptors(); - GenericTypeField[] containerFields = new GenericTypeField[collections.size() + maps.size()]; - for (Descriptor d : collections) { - containerFields[cnt++] = new GenericTypeField(fory, d); - } - for (Descriptor d : maps) { - containerFields[cnt++] = new GenericTypeField(fory, d); - } - return Tuple3.of(Tuple2.of(finalFields, isFinal), otherFields, containerFields); - } - - public static class InternalFieldInfo { - protected final TypeRef typeRef; - protected final short classId; - protected final String qualifiedFieldName; - protected final FieldAccessor fieldAccessor; - protected final FieldConverter fieldConverter; - protected final RefMode refMode; - protected final boolean nullable; - protected final boolean trackingRef; - protected final boolean isPrimitive; - - private InternalFieldInfo(Fory fory, Descriptor d, short classId) { - this.typeRef = d.getTypeRef(); - this.classId = classId; - this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); - if (d.getField() != null) { - this.fieldAccessor = FieldAccessor.createAccessor(d.getField()); - isPrimitive = d.getField().getType().isPrimitive(); - } else { - this.fieldAccessor = null; - isPrimitive = d.getTypeRef().getRawType().isPrimitive(); - } - fieldConverter = d.getFieldConverter(); - nullable = d.isNullable(); - // descriptor.isTrackingRef() already includes the needToWriteRef check - trackingRef = d.isTrackingRef(); - refMode = RefMode.of(trackingRef, nullable); - } - - @Override - public String toString() { - String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); - return "InternalFieldInfo{" - + "fieldName='" - + rsplit[1] - + ", typeRef=" - + typeRef - + ", classId=" - + classId - + ", fieldAccessor=" - + fieldAccessor - + ", nullable=" - + nullable - + '}'; - } - } - - static final class FinalTypeField extends InternalFieldInfo { - final ClassInfo classInfo; - - private FinalTypeField(Fory fory, Descriptor d) { - super(fory, d, getRegisteredClassId(fory, d)); - // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. - if (typeRef.getRawType() == FinalObjectTypeStub.class) { - // `FinalObjectTypeStub` has no fields, using its `classInfo` - // will make deserialization failed. - classInfo = null; - } else { - classInfo = SerializationUtils.getClassInfo(fory, typeRef.getRawType()); - if (!fory.isShareMeta() - && !fory.isCompatible() - && classInfo.getSerializer() instanceof ReplaceResolveSerializer) { - // overwrite replace resolve serializer for final field - classInfo.setSerializer(new FinalFieldReplaceResolveSerializer(fory, classInfo.getCls())); - } - } - } - } - - static final class GenericTypeField extends InternalFieldInfo { - final GenericType genericType; - final ClassInfoHolder classInfoHolder; - final boolean isArray; - final ClassInfo containerClassInfo; - - private GenericTypeField(Fory fory, Descriptor d) { - super(fory, d, getRegisteredClassId(fory, d)); - // TODO support generics in Pojo, see ComplexObjectSerializer.getGenericTypes - ClassResolver classResolver = fory.getClassResolver(); - GenericType t = classResolver.buildGenericType(typeRef); - Class cls = t.getCls(); - if (t.getTypeParametersCount() > 0) { - boolean skip = - Arrays.stream(t.getTypeParameters()).allMatch(p -> p.getCls() == Object.class); - if (skip) { - t = new GenericType(t.getTypeRef(), t.isMonomorphic()); - } - } - genericType = t; - classInfoHolder = classResolver.nilClassInfoHolder(); - isArray = cls.isArray(); - if (!fory.isCrossLanguage()) { - containerClassInfo = null; - } else { - if (classResolver.isMap(cls) - || classResolver.isCollection(cls) - || classResolver.isSet(cls)) { - containerClassInfo = fory.getXtypeResolver().getClassInfo(cls); - } else { - containerClassInfo = null; - } - } - } - - @Override - public String toString() { - String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); - return "GenericTypeField{" - + "fieldName=" - + rsplit[1] - + ", genericType=" - + genericType - + ", classInfoHolder=" - + classInfoHolder - + ", trackingRef=" - + trackingRef - + ", typeRef=" - + typeRef - + ", classId=" - + classId - + ", nullable=" - + nullable - + '}'; - } - } - - private static short getRegisteredClassId(Fory fory, Descriptor d) { - Field field = d.getField(); - Class cls = d.getTypeRef().getRawType(); - if (TypeUtils.unwrap(cls).isPrimitive() && field != null) { - return fory.getClassResolver().getRegisteredClassId(field.getType()); - } - Short classId = fory.getClassResolver().getRegisteredClassId(cls); - return classId == null ? ClassResolver.NO_CLASS_ID : classId; - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java new file mode 100644 index 0000000000..dbcc3362d6 --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.fory.Fory; +import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.ClassInfoHolder; +import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.RefMode; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.converter.FieldConverter; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.FinalObjectTypeStub; +import org.apache.fory.type.GenericType; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.StringUtils; + +public class FieldGroups { + public final SerializationFieldInfo[] buildInFields; + public final SerializationFieldInfo[] userTypeFields; + public final SerializationFieldInfo[] containerFields; + public final SerializationFieldInfo[] allFields; + + public FieldGroups( + SerializationFieldInfo[] buildInFields, + SerializationFieldInfo[] containerFields, + SerializationFieldInfo[] userTypeFields) { + this.buildInFields = buildInFields; + this.userTypeFields = userTypeFields; + this.containerFields = containerFields; + int len = buildInFields.length + userTypeFields.length + containerFields.length; + SerializationFieldInfo[] fields = new SerializationFieldInfo[len]; + System.arraycopy(buildInFields, 0, fields, 0, buildInFields.length); + System.arraycopy(containerFields, 0, fields, buildInFields.length, containerFields.length); + System.arraycopy( + userTypeFields, + 0, + fields, + buildInFields.length + containerFields.length, + userTypeFields.length); + allFields = fields; + } + + public static FieldGroups buildFieldsInfo(Fory fory, List fields) { + List descriptors = new ArrayList<>(); + for (Field field : fields) { + if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) { + descriptors.add(new Descriptor(field, TypeRef.of(field.getGenericType()), null, null)); + } + } + DescriptorGrouper descriptorGrouper = + fory.getClassResolver().createDescriptorGrouper(descriptors, false); + return buildFieldInfos(fory, descriptorGrouper); + } + + public static FieldGroups buildFieldInfos(Fory fory, DescriptorGrouper grouper) { + // When a type is both Collection/Map and final, add it to collection/map fields to keep + // consistent with jit. + Collection primitives = grouper.getPrimitiveDescriptors(); + Collection boxed = grouper.getBoxedDescriptors(); + Collection buildIn = grouper.getBuildInDescriptors(); + SerializationFieldInfo[] allBuildIn = + new SerializationFieldInfo[primitives.size() + boxed.size() + buildIn.size()]; + int cnt = 0; + for (Descriptor d : primitives) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : boxed) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : buildIn) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + cnt = 0; + SerializationFieldInfo[] otherFields = + new SerializationFieldInfo[grouper.getOtherDescriptors().size()]; + for (Descriptor descriptor : grouper.getOtherDescriptors()) { + SerializationFieldInfo genericTypeField = new SerializationFieldInfo(fory, descriptor); + otherFields[cnt++] = genericTypeField; + } + cnt = 0; + Collection collections = grouper.getCollectionDescriptors(); + Collection maps = grouper.getMapDescriptors(); + SerializationFieldInfo[] containerFields = + new SerializationFieldInfo[collections.size() + maps.size()]; + for (Descriptor d : collections) { + containerFields[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : maps) { + containerFields[cnt++] = new SerializationFieldInfo(fory, d); + } + return new FieldGroups(allBuildIn, containerFields, otherFields); + } + + static short getRegisteredClassId(Fory fory, Descriptor d) { + Field field = d.getField(); + Class cls = d.getTypeRef().getRawType(); + if (TypeUtils.unwrap(cls).isPrimitive() && field != null) { + return fory.getClassResolver().getRegisteredClassId(field.getType()); + } + Short classId = fory.getClassResolver().getRegisteredClassId(cls); + return classId == null ? ClassResolver.NO_CLASS_ID : classId; + } + + public static final class SerializationFieldInfo { + public final Descriptor descriptor; + public final TypeRef typeRef; + public final short classId; + public final ClassInfo classInfo; + public final Serializer serializer; + public final String qualifiedFieldName; + public final FieldAccessor fieldAccessor; + public final FieldConverter fieldConverter; + public final RefMode refMode; + public final boolean nullable; + public final boolean trackingRef; + public final boolean isPrimitive; + // Use declared type for serialization/deserialization + public final boolean useDeclaredTypeInfo; + + public final GenericType genericType; + public final ClassInfoHolder classInfoHolder; + public final boolean isArray; + public final ClassInfo containerClassInfo; + + SerializationFieldInfo(Fory fory, Descriptor d) { + this.descriptor = d; + this.typeRef = d.getTypeRef(); + this.classId = getRegisteredClassId(fory, d); + TypeResolver resolver = fory._getTypeResolver(); + // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. + if (typeRef.getRawType() == FinalObjectTypeStub.class) { + // `FinalObjectTypeStub` has no fields, using its `classInfo` + // will make deserialization failed. + classInfo = null; + } else { + if (resolver.isMonomorphic(descriptor)) { + classInfo = SerializationUtils.getClassInfo(fory, typeRef.getRawType()); + if (!fory.isShareMeta() + && !fory.isCompatible() + && classInfo.getSerializer() instanceof ReplaceResolveSerializer) { + // overwrite replace resolve serializer for final field + classInfo.setSerializer( + new FinalFieldReplaceResolveSerializer(fory, classInfo.getCls())); + } + } else { + classInfo = null; + } + } + useDeclaredTypeInfo = classInfo != null && resolver.isMonomorphic(descriptor); + if (classInfo != null) { + serializer = classInfo.getSerializer(); + } else { + serializer = null; + } + + this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); + if (d.getField() != null) { + this.fieldAccessor = FieldAccessor.createAccessor(d.getField()); + isPrimitive = d.getField().getType().isPrimitive(); + } else { + this.fieldAccessor = null; + isPrimitive = d.getTypeRef().getRawType().isPrimitive(); + } + fieldConverter = d.getFieldConverter(); + nullable = d.isNullable(); + // descriptor.isTrackingRef() already includes the needToWriteRef check + trackingRef = d.isTrackingRef(); + refMode = RefMode.of(trackingRef, nullable); + + GenericType t = resolver.buildGenericType(typeRef); + Class cls = t.getCls(); + if (t.getTypeParametersCount() > 0) { + boolean skip = + Arrays.stream(t.getTypeParameters()).allMatch(p -> p.getCls() == Object.class); + if (skip) { + t = new GenericType(t.getTypeRef(), t.isMonomorphic()); + } + } + genericType = t; + classInfoHolder = resolver.nilClassInfoHolder(); + isArray = cls.isArray(); + if (!fory.isCrossLanguage()) { + containerClassInfo = null; + } else { + if (resolver.isMap(cls) || resolver.isCollection(cls) || resolver.isSet(cls)) { + containerClassInfo = resolver.getClassInfo(cls); + } else { + containerClassInfo = null; + } + } + } + + @Override + public String toString() { + String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); + return "InternalFieldInfo{" + + "fieldName='" + + rsplit[1] + + ", typeRef=" + + typeRef + + ", classId=" + + classId + + ", fieldAccessor=" + + fieldAccessor + + ", nullable=" + + nullable + + '}'; + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index cad3dcb1b0..af0da89a9d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -23,8 +23,7 @@ import org.apache.fory.Fory; import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectArray; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; +import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; @@ -34,6 +33,7 @@ import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; @@ -55,10 +55,9 @@ public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase { private final ClassDef layerClassDef; private final Class layerMarkerClass; - private final ObjectSerializer.FinalTypeField[] finalFields; - private final boolean[] isFinal; - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final ClassInfoHolder classInfoHolder; private final SerializationBinding binding; private final TypeResolver typeResolver; @@ -83,16 +82,10 @@ public MetaSharedLayerSerializer( // Build field infos from layerClassDef Collection descriptors = layerClassDef.getDescriptors(typeResolver, type); DescriptorGrouper descriptorGrouper = typeResolver.createDescriptorGrouper(descriptors, false); - - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - infos = AbstractObjectSerializer.buildFieldInfos(fory, descriptorGrouper); - this.finalFields = infos.f0.f0; - this.isFinal = infos.f0.f1; - this.otherFields = infos.f1; - this.containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + this.buildInFields = fieldGroups.buildInFields; + this.otherFields = fieldGroups.userTypeFields; + this.containerFields = fieldGroups.containerFields; } @Override @@ -129,8 +122,7 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -145,7 +137,7 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { fory, buffer, fieldValue, classId); if (writeBasicObjectResult) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinal[i]) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { if (!fieldInfo.trackingRef) { binding.writeNullable(buffer, fieldValue, serializer, nullable); } else { @@ -168,19 +160,19 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { private void writeContainerFields(MemoryBuffer buffer, T value) { Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); } } private void writeOtherFields(MemoryBuffer buffer, T value) { - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - ObjectSerializer.writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -208,7 +200,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { // Read fields in order: final, container, other readFinalFields(buffer, obj); readContainerFields(buffer, obj); - readOtherFields(buffer, obj); + readUserTypeFields(buffer, obj); return obj; } @@ -235,11 +227,8 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; ClassResolver classResolver = this.classResolver; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinalField = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { boolean nullable = fieldInfo.nullable; @@ -253,7 +242,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory, buffer, obj, fieldAccessor, classId))) { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -263,7 +252,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } } @@ -272,7 +261,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { private void readContainerFields(MemoryBuffer buffer, T obj) { Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -282,8 +271,8 @@ private void readContainerFields(MemoryBuffer buffer, T obj) { } } - private void readOtherFields(MemoryBuffer buffer, T obj) { - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + private void readUserTypeFields(MemoryBuffer buffer, T obj) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -314,7 +303,7 @@ public Class getLayerMarkerClass() { /** Returns the number of fields in this layer. */ public int getNumFields() { - return finalFields.length + containerFields.length + otherFields.length; + return buildInFields.length + containerFields.length + otherFields.length; } /** @@ -331,30 +320,26 @@ public void writeFieldsValues(MemoryBuffer buffer, Object[] vals) { // Write field values from array int index = 0; // Write final fields - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { Object fieldValue = vals[index++]; - writeFieldValueFromArray(buffer, fieldInfo, fieldValue, isFinal[i]); + writeFieldValueFromArray(buffer, fieldInfo, fieldValue); } // Write container fields Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = vals[index++]; - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); } // Write other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = vals[index++]; - ObjectSerializer.writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } private void writeFieldValueFromArray( - MemoryBuffer buffer, - ObjectSerializer.FinalTypeField fieldInfo, - Object fieldValue, - boolean isFinalField) { + MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object fieldValue) { short classId = fieldInfo.classId; boolean nullable = fieldInfo.nullable; @@ -395,7 +380,7 @@ private void writeFieldValueFromArray( // Handle objects boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinalField) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { if (!fieldInfo.trackingRef) { binding.writeNullable(buffer, fieldValue, serializer, nullable); } else { @@ -426,28 +411,23 @@ public void readFields(MemoryBuffer buffer, Object[] vals) { } // Read field values into array int index = 0; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - // Read final fields - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinalField = !metaShareEnabled || this.isFinal[i]; - vals[index++] = readFieldValueToArray(buffer, fieldInfo, isFinalField); + for (SerializationFieldInfo fieldInfo : buildInFields) { + vals[index++] = readFieldValueToArray(buffer, fieldInfo); } // Read container fields Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { vals[index++] = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); } // Read other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { vals[index++] = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); } } - private Object readFieldValueToArray( - MemoryBuffer buffer, ObjectSerializer.FinalTypeField fieldInfo, boolean isFinalField) { + private Object readFieldValueToArray(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { short classId = fieldInfo.classId; // Handle primitives @@ -458,7 +438,7 @@ private Object readFieldValueToArray( // Handle objects return AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } /** @@ -471,32 +451,17 @@ private Object readFieldValueToArray( */ @Override @SuppressWarnings("rawtypes") - public void setFieldValuesFromPutFields( - Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, Object[] vals) { + public void setFieldValuesFromPutFields(Object obj, ObjectIntMap fieldIndexMap, Object[] vals) { // Set final fields - for (ObjectSerializer.FinalTypeField fieldInfo : finalFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - fieldAccessor.set(obj, vals[index]); - } - } - } - // Set container fields - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - fieldAccessor.set(obj, vals[index]); - } - } - } + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, buildInFields); + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, containerFields); + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, otherFields); + } + + private void setFieldValuesFromPutFields( + Object obj, ObjectIntMap fieldIndexMap, Object[] vals, SerializationFieldInfo[] fieldInfos) { // Set other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { String fieldName = fieldAccessor.getField().getName(); @@ -523,29 +488,20 @@ public Object[] getFieldValuesForPutFields( Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, int arraySize) { Object[] vals = new Object[arraySize]; // Get final fields - for (ObjectSerializer.FinalTypeField fieldInfo : finalFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - vals[index] = fieldAccessor.get(obj); - } - } - } + getFieldValuesForPutFields(obj, fieldIndexMap, vals, buildInFields); // Get container fields - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - vals[index] = fieldAccessor.get(obj); - } - } - } + getFieldValuesForPutFields(obj, fieldIndexMap, vals, containerFields); // Get other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + getFieldValuesForPutFields(obj, fieldIndexMap, vals, otherFields); + return vals; + } + + private void getFieldValuesForPutFields( + Object obj, + ObjectIntMap fieldIndexMap, + Object[] vals, + SerializationFieldInfo[] buildInFields) { + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { String fieldName = fieldAccessor.getField().getName(); @@ -555,6 +511,5 @@ public Object[] getFieldValuesForPutFields( } } } - return vals; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 8b8bfdf925..60c4044572 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -25,8 +25,6 @@ import java.util.stream.Collectors; import org.apache.fory.Fory; import org.apache.fory.builder.MetaSharedCodecBuilder; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.logging.Logger; @@ -39,6 +37,7 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; @@ -65,21 +64,12 @@ * @see MetaSharedCodecBuilder * @see ObjectSerializer */ -@SuppressWarnings({"unchecked"}) public class MetaSharedSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(MetaSharedSerializer.class); - private final ObjectSerializer.FinalTypeField[] finalFields; - - /** - * Whether write class def for non-inner final types. - * - * @see ClassResolver#isMonomorphic(Class) - */ - private final boolean[] isFinal; - - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] containerFields; + private final SerializationFieldInfo[] otherFields; private final RecordInfo recordInfo; private Serializer serializer; private final ClassInfoHolder classInfoHolder; @@ -109,24 +99,18 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { "========== MetaSharedSerializer sorted descriptors for {} ==========", type.getName()); for (Descriptor d : descriptorGrouper.getSortedDescriptors()) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } // d.getField() may be null if not exists in this class when meta share enabled. - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - infos = AbstractObjectSerializer.buildFieldInfos(fory, descriptorGrouper); - finalFields = infos.f0.f0; - isFinal = infos.f0.f1; - otherFields = infos.f1; - containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + buildInFields = fieldGroups.buildInFields; + containerFields = fieldGroups.containerFields; + otherFields = fieldGroups.userTypeFields; classInfoHolder = this.classResolver.nilClassInfoHolder(); if (isRecord) { List fieldNames = @@ -141,15 +125,13 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { boolean hasDefaultValues = false; DefaultValueUtils.DefaultValueField[] defaultValueFields = new DefaultValueUtils.DefaultValueField[0]; - DefaultValueUtils.DefaultValueSupport defaultValueSupport = null; + DefaultValueUtils.DefaultValueSupport defaultValueSupport; if (fory.getConfig().isScalaOptimizationEnabled()) { defaultValueSupport = DefaultValueUtils.getScalaDefaultValueSupport(); - if (defaultValueSupport != null) { - hasDefaultValues = defaultValueSupport.hasDefaultValues(type); - defaultValueFields = - defaultValueSupport.buildDefaultValueFields( - fory, type, descriptorGrouper.getSortedDescriptors()); - } + hasDefaultValues = defaultValueSupport.hasDefaultValues(type); + defaultValueFields = + defaultValueSupport.buildDefaultValueFields( + fory, type, descriptorGrouper.getSortedDescriptors()); } if (!hasDefaultValues) { DefaultValueUtils.DefaultValueSupport kotlinDefaultValueSupport = @@ -168,6 +150,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { @Override public void write(MemoryBuffer buffer, T value) { if (serializer == null) { + // xlang mode will register class and create serializer in advance, it won't go to here. serializer = this.classResolver.createSerializerSafe(type, () -> new ObjectSerializer<>(fory, type)); } @@ -183,7 +166,7 @@ public void xwrite(MemoryBuffer buffer, T value) { public T read(MemoryBuffer buffer) { if (isRecord) { Object[] fieldValues = - new Object[finalFields.length + otherFields.length + containerFields.length]; + new Object[buildInFields.length + otherFields.length + containerFields.length]; readFields(buffer, fieldValues); fieldValues = RecordUtils.remapping(recordInfo, fieldValues); T t = objectCreator.newInstanceWithArguments(fieldValues); @@ -193,14 +176,11 @@ public T read(MemoryBuffer buffer) { T obj = newInstance(); Fory fory = this.fory; RefResolver refResolver = this.refResolver; - ClassResolver classResolver = this.classResolver; + TypeResolver typeResolver = this.typeResolver; SerializationBinding binding = this.binding; refResolver.reference(obj); // read order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = this.finalFields; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; if (fieldAccessor != null) { @@ -222,7 +202,7 @@ public T read(MemoryBuffer buffer) { assert fieldInfo.classInfo != null; Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, typeResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -231,19 +211,19 @@ public T read(MemoryBuffer buffer) { if (skipPrimitiveFieldValueFailed(fory, fieldInfo.classId, buffer)) { if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. - fory.readRef(buffer, classInfoHolder); + binding.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, typeResolver, fieldInfo, buffer); } } } else { - compatibleRead(buffer, fieldInfo, isFinal, obj); + compatibleRead(buffer, fieldInfo, obj); } } } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -251,7 +231,7 @@ public T read(MemoryBuffer buffer) { fieldAccessor.putObject(obj, fieldValue); } } - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -261,8 +241,7 @@ public T read(MemoryBuffer buffer) { return obj; } - private void compatibleRead( - MemoryBuffer buffer, FinalTypeField fieldInfo, boolean isFinal, Object obj) { + private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object obj) { Object fieldValue; short classId = fieldInfo.classId; if (classId >= ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID @@ -271,7 +250,7 @@ private void compatibleRead( } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } fieldInfo.fieldConverter.set(obj, fieldValue); } @@ -298,10 +277,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { ClassResolver classResolver = this.classResolver; SerializationBinding binding = this.binding; // read order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = this.finalFields; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { if (fieldInfo.fieldAccessor != null) { assert fieldInfo.classInfo != null; short classId = fieldInfo.classId; @@ -312,7 +288,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { } else { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); fields[counter++] = fieldValue; } } else { @@ -323,19 +299,19 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { fory.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } // remapping will handle those extra fields from peers. fields[counter++] = null; } } - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); fields[counter++] = fieldValue; } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); fields[counter++] = fieldValue; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 4213343dc4..83366f1286 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -19,7 +19,7 @@ package org.apache.fory.serializer; -import static org.apache.fory.serializer.ObjectSerializer.writeOtherFieldValue; +import static org.apache.fory.serializer.AbstractObjectSerializer.writeOtherFieldValue; import static org.apache.fory.serializer.SerializationUtils.getTypeResolver; import java.util.ArrayList; @@ -29,8 +29,6 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; import org.apache.fory.collection.MapEntry; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.resolver.ClassInfo; @@ -40,6 +38,7 @@ import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.NonexistentClass.NonexistentEnum; import org.apache.fory.serializer.Serializers.CrossLanguageCompatibleSerializer; import org.apache.fory.type.Descriptor; @@ -51,22 +50,15 @@ public final class NonexistentClassSerializers { private static final class ClassFieldsInfo { - private final ObjectSerializer.FinalTypeField[] finalFields; - private final boolean[] isFinal; - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final int classVersionHash; - private ClassFieldsInfo( - ObjectSerializer.FinalTypeField[] finalFields, - boolean[] isFinal, - ObjectSerializer.GenericTypeField[] otherFields, - ObjectSerializer.GenericTypeField[] containerFields, - int classVersionHash) { - this.finalFields = finalFields; - this.isFinal = isFinal; - this.otherFields = otherFields; - this.containerFields = containerFields; + private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { + this.buildInFields = fieldGroups.buildInFields; + this.otherFields = fieldGroups.userTypeFields; + this.containerFields = fieldGroups.containerFields; this.classVersionHash = classVersionHash; } } @@ -76,7 +68,6 @@ public static final class NonexistentClassSerializer extends Serializer { private final ClassInfoHolder classInfoHolder; private final LongMap fieldsInfoMap; private final SerializationBinding binding; - private final TypeResolver typeResolver; public NonexistentClassSerializer(Fory fory, ClassDef classDef) { super(fory, NonexistentClass.NonexistentMetaShared.class); @@ -84,7 +75,6 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { classInfoHolder = fory.getClassResolver().nilClassInfoHolder(); fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); - typeResolver = fory._getTypeResolver(); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); } @@ -124,16 +114,13 @@ public void write(MemoryBuffer buffer, Object v) { buffer.writeInt32(fieldsInfo.classVersionHash); } // write order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields; - boolean[] isFinal = fieldsInfo.isFinal; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); ClassInfo classInfo = fieldInfo.classInfo; if (classResolver.isPrimitive(fieldInfo.classId)) { classInfo.getSerializer().write(buffer, fieldValue); } else { - if (isFinal[i]) { + if (fieldInfo.useDeclaredTypeInfo) { // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still // consistent with jit serializer. Serializer serializer = classInfo.getSerializer(); @@ -144,14 +131,14 @@ public void write(MemoryBuffer buffer, Object v) { } } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, classResolver, generics, fieldInfo, buffer, fieldValue); } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -165,17 +152,12 @@ private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { resolver, NonexistentClass.NonexistentSkip.class, classDef); DescriptorGrouper grouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - tuple = AbstractObjectSerializer.buildFieldInfos(fory, grouper); + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, grouper); int classVersionHash = 0; if (fory.checkClassVersion()) { classVersionHash = ObjectSerializer.computeStructHash(fory, grouper); } - fieldsInfo = - new ClassFieldsInfo(tuple.f0.f0, tuple.f0.f1, tuple.f1, tuple.f2, classVersionHash); + fieldsInfo = new ClassFieldsInfo(fieldGroups, classVersionHash); fieldsInfoMap.put(classDef.getId(), fieldsInfo); } return fieldsInfo; @@ -192,10 +174,7 @@ public Object read(MemoryBuffer buffer) { List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); - ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields; - boolean[] isFinal = fieldsInfo.isFinal; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { Object fieldValue; if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. @@ -206,18 +185,18 @@ public Object read(MemoryBuffer buffer) { } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal[i], buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 43cd434ef1..376ed5c7d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -25,24 +25,21 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.fory.Fory; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.exception.ForyException; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.struct.Fingerprint; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; import org.apache.fory.util.MurmurHash3; -import org.apache.fory.util.Preconditions; import org.apache.fory.util.Utils; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -66,17 +63,9 @@ public final class ObjectSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(ObjectSerializer.class); private final RecordInfo recordInfo; - private final FinalTypeField[] finalFields; - - /** - * Whether write class def for non-inner final types. - * - * @see ClassResolver#isMonomorphic(Class) - */ - private final boolean[] isFinal; - - private final GenericTypeField[] otherFields; - private final GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final int classVersionHash; private final SerializationBinding binding; private final TypeResolver typeResolver; @@ -116,12 +105,11 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { LOG.info("========== ObjectSerializer sorted descriptors for {} ==========", cls.getName()); for (Descriptor d : descriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } if (isRecord) { @@ -136,12 +124,10 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { } else { classVersionHash = 0; } - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, grouper); - finalFields = infos.f0.f0; - isFinal = infos.f0.f1; - otherFields = infos.f1; - containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, grouper); + buildInFields = fieldGroups.buildInFields; + otherFields = fieldGroups.userTypeFields; + containerFields = fieldGroups.containerFields; } @Override @@ -149,19 +135,26 @@ public void write(MemoryBuffer buffer, T value) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; if (fory.checkClassVersion()) { + if (fory.getConfig().isForyDebugOutputEnabled()) { + LOG.info( + "[Java][fory-debug] Writing struct hash for {} at position {}: hash={}", + type.getSimpleName(), + buffer.writerIndex(), + classVersionHash); + } buffer.writeInt32(classVersionHash); } // write order: primitive,boxed,final,other,collection,map - writeFinalFields(buffer, value, fory, refResolver, typeResolver); + writeBuildInFields(buffer, value, fory, refResolver, typeResolver); writeContainerFields(buffer, value, fory, refResolver, typeResolver); writeOtherFields(buffer, value); } private void writeOtherFields(MemoryBuffer buffer, T value) { - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -170,12 +163,10 @@ public void xwrite(MemoryBuffer buffer, T value) { write(buffer, value); } - private void writeFinalFields( + private void writeBuildInFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { - FinalTypeField[] finalFields = this.finalFields; boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -187,7 +178,7 @@ private void writeFinalFields( : writeBasicObjectFieldValue(fory, buffer, fieldValue, classId); if (needWrite) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinal[i]) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { switch (fieldInfo.refMode) { case NONE: binding.write(buffer, serializer, fieldValue); @@ -237,7 +228,7 @@ private void writeFinalFields( private void writeContainerFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); writeContainerFieldValue( @@ -245,97 +236,12 @@ private void writeContainerFields( } } - static void writeContainerFieldValue( - SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - Generics generics, - GenericTypeField fieldInfo, - MemoryBuffer buffer, - Object fieldValue) { - switch (fieldInfo.refMode) { - case NONE: - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - } - break; - case TRACKING: - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - ClassInfo classInfo = - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue(buffer, fieldValue, classInfo); - generics.popGenericType(); - } - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } - - static void writeOtherFieldValue( - SerializationBinding binding, - TypeResolver typeResolver, - MemoryBuffer buffer, - GenericTypeField fieldInfo, - Object fieldValue) { - // Enum has special handling for xlang compatibility - no ref tracking for enums - if (fieldInfo.genericType.getCls().isEnum()) { - if (fieldValue == null) { - Preconditions.checkArgument( - fieldInfo.nullable, "Enum field value is null but not nullable"); - buffer.writeByte(Fory.NULL_FLAG); - } else { - // Only write null flag when the field is nullable (for xlang compatibility) - if (fieldInfo.nullable) { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - } - fieldInfo.genericType.getSerializer(typeResolver).write(buffer, fieldValue); - } - return; - } - switch (fieldInfo.refMode) { - case NONE: - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } - break; - case TRACKING: - binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } - @Override public T read(MemoryBuffer buffer) { if (isRecord) { Object[] fields = readFields(buffer); fields = RecordUtils.remapping(recordInfo, fields); - T obj = (T) objectCreator.newInstanceWithArguments(fields); + T obj = objectCreator.newInstanceWithArguments(fields); Arrays.fill(recordInfo.getRecordComponents(), null); return obj; } @@ -355,34 +261,29 @@ public Object[] readFields(MemoryBuffer buffer) { TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); - checkClassVersion(fory, hash, classVersionHash); + checkClassVersion(type, hash, classVersionHash); } Object[] fieldValues = - new Object[finalFields.length + otherFields.length + containerFields.length]; + new Object[buildInFields.length + otherFields.length + containerFields.length]; int counter = 0; // read order: primitive,boxed,final,other,collection,map - FinalTypeField[] finalFields = this.finalFields; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { short classId = fieldInfo.classId; if (classId >= ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID && classId <= ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID) { fieldValues[counter++] = Serializers.readPrimitiveValue(fory, buffer, classId); } else { Object fieldValue = - readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, isFinal, buffer); + readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } } Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } @@ -395,14 +296,10 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); - checkClassVersion(fory, hash, classVersionHash); + checkClassVersion(type, hash, classVersionHash); } // read order: primitive,boxed,final,other,collection,map - FinalTypeField[] finalFields = this.finalFields; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -411,18 +308,17 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, classId) : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, classId))) { Object fieldValue = - readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, isFinal, buffer); + readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); @@ -450,12 +346,12 @@ public static int computeStructHash(Fory fory, DescriptorGrouper grouper) { return hash; } - public static void checkClassVersion(Fory fory, int readHash, int classVersionHash) { + public static void checkClassVersion(Class cls, int readHash, int classVersionHash) { if (readHash != classVersionHash) { throw new ForyException( String.format( "Read class %s version %s is not consistent with %s", - fory.getClassResolver().getCurrentReadClass(), readHash, classVersionHash)); + cls, readHash, classVersionHash)); } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 3fb1c84179..4ce0ebaf0c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -20,9 +20,10 @@ package org.apache.fory.serializer; import static org.apache.fory.Fory.NOT_NULL_VALUE_FLAG; -import static org.apache.fory.serializer.AbstractObjectSerializer.GenericTypeField; import org.apache.fory.Fory; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; @@ -30,12 +31,15 @@ import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; // This polymorphic interface has cost, do not expose it as a public class // If it's used in other packages in fory, duplicate it in those packages. @SuppressWarnings({"rawtypes", "unchecked"}) // noinspection Duplicates abstract class SerializationBinding { + private static final Logger LOG = LoggerFactory.getLogger(SerializationBinding.class); + protected final Fory fory; protected final RefResolver refResolver; protected final TypeResolver typeResolver; @@ -56,7 +60,7 @@ abstract class SerializationBinding { abstract void writeNonRef(MemoryBuffer buffer, Object obj); - abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer); abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); @@ -83,7 +87,7 @@ abstract void writeContainerFieldValue( abstract T readRef(MemoryBuffer buffer, Serializer serializer); - abstract Object readRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readRef(MemoryBuffer buffer, SerializationFieldInfo field); abstract Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); @@ -93,16 +97,16 @@ abstract void writeContainerFieldValue( abstract Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); - abstract Object readNonRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field); abstract Object readNullable(MemoryBuffer buffer, Serializer serializer); abstract Object readNullable( MemoryBuffer buffer, Serializer serializer, boolean nullable); - abstract Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field); + abstract Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field); - abstract Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo); + abstract Object readContainerFieldValueRef(MemoryBuffer buffer, SerializationFieldInfo fieldInfo); public int preserveRefId(int refId) { return refResolver.preserveRefId(refId); @@ -162,7 +166,10 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.readRef(buffer, field.classInfo.getSerializer()); + } return fory.readRef(buffer, field.classInfoHolder); } @@ -187,7 +194,10 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { } @Override - public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.readNonRef(buffer, field.classInfo); + } return fory.readNonRef(buffer, field.classInfoHolder); } @@ -207,12 +217,13 @@ public Object readNullable( } @Override - public Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field) { return fory.readNonRef(buffer, field.classInfoHolder); } @Override - public Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo) { + public Object readContainerFieldValueRef( + MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { RefResolver refResolver = fory.getRefResolver(); int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { @@ -241,8 +252,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.writeNonRef(buffer, obj, classInfo); + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + fory.writeNonRef(buffer, obj, serializer); } @Override @@ -332,17 +343,17 @@ public void writeRef(MemoryBuffer buffer, T obj) { @Override public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { - fory.xwriteRef(buffer, obj, serializer); + fory.writeRef(buffer, obj, serializer); } @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - fory.xwriteRef(buffer, obj); + fory.xwriteRef(buffer, obj, classInfoHolder); } @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.xwriteRef(buffer, obj); + fory.xwriteRef(buffer, obj, classInfo); } @Override @@ -351,20 +362,29 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.isArray) { fory.getGenerics().pushGenericType(field.genericType); - Object o = fory.xreadRef(buffer); + Object o; + if (field.useDeclaredTypeInfo) { + o = fory.xreadRef(buffer, field.serializer); + } else { + o = fory.xreadRef(buffer); + } fory.getGenerics().popGenericType(); return o; } else { - return fory.xreadRef(buffer); + if (field.useDeclaredTypeInfo) { + return fory.xreadRef(buffer, field.serializer); + } else { + return fory.xreadRef(buffer); + } } } @Override public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - return fory.xreadRef(buffer); + return fory.xreadRef(buffer, classInfoHolder); } @Override @@ -383,14 +403,23 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { } @Override - public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.isArray) { fory.getGenerics().pushGenericType(field.genericType); - Object o = fory.xreadNonRef(buffer); + Object o; + if (field.useDeclaredTypeInfo) { + o = fory.xreadNonRef(buffer, field.serializer); + } else { + o = fory.xreadNonRef(buffer); + } fory.getGenerics().popGenericType(); return o; } else { - return fory.xreadNonRef(buffer); + if (field.useDeclaredTypeInfo) { + return fory.xreadNonRef(buffer, field.serializer); + } else { + return fory.xreadNonRef(buffer); + } } } @@ -410,12 +439,12 @@ public Object readNullable( } @Override - public Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field) { return fory.xreadNonRef(buffer, field.containerClassInfo); } @Override - public Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValueRef(MemoryBuffer buffer, SerializationFieldInfo field) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { Object o = fory.xreadNonRef(buffer, field.containerClassInfo); @@ -442,8 +471,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.xwriteNonRef(buffer, obj, classInfo); + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + fory.xwriteNonRef(buffer, obj, serializer); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 923974e75a..98247cff92 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -43,7 +43,8 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.serializer.AbstractObjectSerializer; -import org.apache.fory.serializer.AbstractObjectSerializer.InternalFieldInfo; +import org.apache.fory.serializer.FieldGroups; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.JavaSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializer; import org.apache.fory.serializer.ObjectSerializer; @@ -131,7 +132,7 @@ public static class ChildCollectionSerializer ArrayList.class, LinkedList.class, ArrayDeque.class, Vector.class, HashSet.class // PriorityQueue/TreeSet/ConcurrentSkipListSet need comparator as constructor argument ); - protected InternalFieldInfo[] fieldInfos; + protected SerializationFieldInfo[] fieldInfos; protected final Serializer[] slotsSerializers; public ChildCollectionSerializer(Fory fory, Class cls) { @@ -159,7 +160,7 @@ public Collection newCollection(Collection originCollection) { Collection newCollection = super.newCollection(originCollection); if (fieldInfos == null) { List fields = ReflectionUtils.getFieldsWithoutSuperClasses(type, superClasses); - fieldInfos = AbstractObjectSerializer.buildFieldsInfo(fory, fields); + fieldInfos = FieldGroups.buildFieldsInfo(fory, fields).allFields; } AbstractObjectSerializer.copyFields(fory, fieldInfos, originCollection, newCollection); return newCollection; @@ -193,7 +194,7 @@ public static class ChildMapSerializer extends MapSerializer { // TreeMap/ConcurrentSkipListMap need comparator as constructor argument ); private final Serializer[] slotsSerializers; - private InternalFieldInfo[] fieldInfos; + private SerializationFieldInfo[] fieldInfos; public ChildMapSerializer(Fory fory, Class cls) { super(fory, cls); @@ -221,7 +222,7 @@ public Map newMap(Map originMap) { Map newMap = super.newMap(originMap); if (fieldInfos == null || fieldInfos.length == 0) { List fields = ReflectionUtils.getFieldsWithoutSuperClasses(type, superClasses); - fieldInfos = AbstractObjectSerializer.buildFieldsInfo(fory, fields); + fieldInfos = FieldGroups.buildFieldsInfo(fory, fields).allFields; } AbstractObjectSerializer.copyFields(fory, fieldInfos, originMap, newMap); return newMap; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java index 0a0e18880a..4589065265 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java @@ -607,6 +607,7 @@ private void readSameTypeElements( private void readDifferentTypeElements( Fory fory, MemoryBuffer buffer, int flags, T collection, int numElements) { if ((flags & CollectionFlags.TRACKING_REF) == CollectionFlags.TRACKING_REF) { + Preconditions.checkState(fory.trackingRef(), "Reference tracking is not enabled"); for (int i = 0; i < numElements; i++) { collection.add(binding.readRef(buffer)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index e1ff58fd7a..79b984e375 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -52,6 +52,7 @@ import org.apache.fory.type.GenericType; import org.apache.fory.type.Generics; import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.Preconditions; /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) @@ -80,6 +81,7 @@ public abstract class MapLikeSerializer extends Serializer { private int numElements; private final TypeResolver typeResolver; protected final SerializationBinding binding; + private boolean trackRef; public MapLikeSerializer(Fory fory, Class cls) { this(fory, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); @@ -92,6 +94,7 @@ public MapLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook) { public MapLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook, boolean immutable) { super(fory, cls, immutable); this.typeResolver = fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); + trackRef = fory.trackingRef(); this.supportCodegenHook = supportCodegenHook; keyClassInfoWriteCache = typeResolver.nilClassInfoHolder(); keyClassInfoReadCache = typeResolver.nilClassInfoHolder(); @@ -773,6 +776,9 @@ private long readJavaChunk( // noinspection Duplicates boolean trackKeyRef = (chunkHeader & TRACKING_KEY_REF) != 0; boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0; + if (trackKeyRef || trackValueRef) { + Preconditions.checkState(trackRef, "Ref tracking is not enabled"); + } boolean keyIsDeclaredType = (chunkHeader & KEY_DECL_TYPE) != 0; boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0; int chunkSize = buffer.readUnsignedByte(); @@ -818,6 +824,9 @@ private long readJavaChunkGeneric( // noinspection Duplicates boolean trackKeyRef = (chunkHeader & TRACKING_KEY_REF) != 0; boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0; + if (trackKeyRef || trackValueRef) { + Preconditions.checkState(trackRef, "Ref tracking is not enabled"); + } boolean keyIsDeclaredType = (chunkHeader & KEY_DECL_TYPE) != 0; boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0; int chunkSize = buffer.readUnsignedByte(); diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index bd9c6aa8cb..158c02968f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -261,6 +261,18 @@ public ForyField getForyField() { return foryField; } + /** + * Returns the morphic setting for this field. + * + * @return the morphic setting from @ForyField annotation, or AUTO if not specified + */ + public ForyField.Morphic getMorphic() { + if (foryField != null) { + return foryField.morphic(); + } + return ForyField.Morphic.AUTO; + } + /** Try not use {@link TypeRef#getRawType()} since it's expensive. */ public Class getRawType() { Class type = this.type; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java index f7e86d91df..da25315d9b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java @@ -82,7 +82,7 @@ public static String getFieldSortKey(Descriptor descriptor) { return c; }; private final Collection descriptors; - private final Predicate> isMonomorphic; + private final Predicate isBuildIn; private final Function descriptorUpdater; private final boolean descriptorsGroupedOrdered; private boolean sorted = false; @@ -167,13 +167,13 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole private final Collection collectionDescriptors; // The key/value type should be final. private final Collection mapDescriptors; - private final Collection finalDescriptors; + private final Collection buildInDescriptors; private Collection otherDescriptors; /** * Create a descriptor grouper. * - * @param isMonomorphic whether the class is monomorphic. + * @param isBuildIn whether the class is build-in types. * @param descriptors descriptors may have field with same name. * @param descriptorsGroupedOrdered whether the descriptors are grouped and ordered. * @param descriptorUpdater create a new descriptor from original one. @@ -181,14 +181,14 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole * @param comparator comparator for non-primitive fields. */ private DescriptorGrouper( - Predicate> isMonomorphic, + Predicate isBuildIn, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdater, Comparator primitiveComparator, Comparator comparator) { this.descriptors = descriptors; - this.isMonomorphic = isMonomorphic; + this.isBuildIn = isBuildIn; this.descriptorUpdater = descriptorUpdater; this.descriptorsGroupedOrdered = descriptorsGroupedOrdered; this.primitiveDescriptors = @@ -198,7 +198,7 @@ private DescriptorGrouper( this.collectionDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); this.mapDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); - this.finalDescriptors = + this.buildInDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); this.otherDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); @@ -232,8 +232,8 @@ public DescriptorGrouper sort() { collectionDescriptors.add(descriptorUpdater.apply(descriptor)); } else if (TypeUtils.isMap(descriptor.getRawType())) { mapDescriptors.add(descriptorUpdater.apply(descriptor)); - } else if (isMonomorphic.test(descriptor.getRawType())) { - finalDescriptors.add(descriptorUpdater.apply(descriptor)); + } else if (isBuildIn.test(descriptor)) { + buildInDescriptors.add(descriptorUpdater.apply(descriptor)); } else { otherDescriptors.add(descriptorUpdater.apply(descriptor)); } @@ -247,7 +247,7 @@ public List getSortedDescriptors() { List descriptors = new ArrayList<>(getNumDescriptors()); descriptors.addAll(getPrimitiveDescriptors()); descriptors.addAll(getBoxedDescriptors()); - descriptors.addAll(getFinalDescriptors()); + descriptors.addAll(getBuildInDescriptors()); descriptors.addAll(getCollectionDescriptors()); descriptors.addAll(getMapDescriptors()); descriptors.addAll(getOtherDescriptors()); @@ -274,9 +274,9 @@ public Collection getMapDescriptors() { return mapDescriptors; } - public Collection getFinalDescriptors() { + public Collection getBuildInDescriptors() { Preconditions.checkArgument(sorted); - return finalDescriptors; + return buildInDescriptors; } public Collection getOtherDescriptors() { @@ -297,7 +297,7 @@ private static Descriptor createDescriptor(Descriptor d) { } public static DescriptorGrouper createDescriptorGrouper( - Predicate> isMonomorphic, + Predicate isBuildIn, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdator, @@ -305,7 +305,7 @@ public static DescriptorGrouper createDescriptorGrouper( boolean compressLong, Comparator comparator) { return new DescriptorGrouper( - isMonomorphic, + isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator == null ? DescriptorGrouper::createDescriptor : descriptorUpdator, @@ -319,7 +319,7 @@ public int getNumDescriptors() { + boxedDescriptors.size() + collectionDescriptors.size() + mapDescriptors.size() - + finalDescriptors.size() + + buildInDescriptors.size() + otherDescriptors.size(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java b/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java index 1a6cfaead0..1f0e133b58 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java @@ -35,6 +35,12 @@ public static T checkNotNull(T o, String errorMessage) { return o; } + public static void checkState(boolean expression, String errorMessage) { + if (!expression) { + throw new IllegalStateException(errorMessage); + } + } + public static void checkState(boolean expression) { if (!expression) { throw new IllegalStateException(); diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 39968373da..44df2f95cc 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -452,10 +452,9 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableCollectionSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableMapSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers,\ - org.apache.fory.serializer.AbstractObjectSerializer$ForyFieldInfo,\ org.apache.fory.serializer.ObjectSerializer,\ - org.apache.fory.serializer.ObjectSerializer$FinalTypeField,\ - org.apache.fory.serializer.ObjectSerializer$GenericTypeField,\ + org.apache.fory.serializer.FieldGroups,\ + org.apache.fory.serializer.FieldGroups.SerializationFieldInfo,\ org.apache.fory.serializer.LazySerializer,\ org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\ org.apache.fory.serializer.shim.ShimDispatcher,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java deleted file mode 100644 index c753607a49..0000000000 --- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.fory; - -import com.google.common.collect.ImmutableMap; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.testng.SkipException; -import org.testng.annotations.Test; - -/** Executes cross-language tests against the Rust implementation. */ -@Test -public class RustXlangTest extends XlangTestBase { - private static final String RUST_EXECUTABLE = "cargo"; - private static final String RUST_MODULE = "test_cross_language"; - - private static final List RUST_BASE_COMMAND = - Arrays.asList( - RUST_EXECUTABLE, - "test", - "--test", - RUST_MODULE, - "", - "--", - "--nocapture", - "--ignored", - "--exact"); - - private static final int RUST_TESTCASE_INDEX = 4; - - @Override - protected void ensurePeerReady() { - String enabled = System.getenv("FORY_RUST_JAVA_CI"); - if (!"1".equals(enabled)) { - throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); - } - boolean rustInstalled = true; - try { - Process process = new ProcessBuilder("rustc", "--version").start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - rustInstalled = false; - } - } catch (IOException | InterruptedException e) { - rustInstalled = false; - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - } - if (!rustInstalled) { - throw new SkipException("Skipping RustXlangTest: rust not installed"); - } - } - - @Override - protected CommandContext buildCommandContext(String caseName, Path dataFile) { - List command = new ArrayList<>(RUST_BASE_COMMAND); - command.set(RUST_TESTCASE_INDEX, caseName); - ImmutableMap env = - envBuilder(dataFile) - .put("RUSTFLAGS", "-Awarnings") - .put("RUST_BACKTRACE", "1") - .put("ENABLE_FORY_DEBUG_OUTPUT", "1") - .put("FORY_PANIC_ON_ERROR", "1") - .build(); - return new CommandContext(command, env, new File("../../rust")); - } - - // ============================================================================ - // Test methods - duplicated from XlangTestBase for Maven Surefire discovery - // ============================================================================ - - @Test - public void testBuffer() throws java.io.IOException { - super.testBuffer(); - } - - @Test - public void testBufferVar() throws java.io.IOException { - super.testBufferVar(); - } - - @Test - public void testMurmurHash3() throws java.io.IOException { - super.testMurmurHash3(); - } - - @Test - public void testStringSerializer() throws Exception { - super.testStringSerializer(); - } - - @Test - public void testCrossLanguageSerializer() throws Exception { - super.testCrossLanguageSerializer(); - } - - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); - } - - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); - } - - @Test - public void testList() throws java.io.IOException { - super.testList(); - } - - @Test - public void testMap() throws java.io.IOException { - super.testMap(); - } - - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); - } - - @Test - public void testItem() throws java.io.IOException { - super.testItem(); - } - - @Test - public void testColor() throws java.io.IOException { - super.testColor(); - } - - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); - } - - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); - } - - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); - } - - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); - } - - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); - } - - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); - } - - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); - } - - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); - } - - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); - } - - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); - } - - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); - } - - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); - } - - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); - } - - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); - } - - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); - } - - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { - super.testEnumSchemaEvolutionCompatible(); - } - - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNotNull(); - } - - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNull(); - } - - @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); - } - - @Override - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { - super.testNullableFieldCompatibleNull(); - } - - @Test - public void testUnionXlang() throws java.io.IOException { - super.testUnionXlang(); - } -} diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index 58fc6555d2..1f96d17869 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -178,7 +178,7 @@ public void testGrouper() { new TypeRef>() {}, "c" + index++, -1, "TestClass", false)); DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, + d -> ReflectionUtils.isMonomorphic(d.getRawType()), descriptors, false, null, @@ -244,7 +244,7 @@ public void testGrouper() { } { List> classes = - grouper.getFinalDescriptors().stream() + grouper.getBuildInDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); assertEquals(classes, Arrays.asList(String.class, Instant.class, Instant.class)); @@ -262,7 +262,7 @@ public void testGrouper() { public void testCompressedPrimitiveGrouper() { DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, + d -> ReflectionUtils.isMonomorphic(d.getRawType()), createDescriptors(), false, null, diff --git a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java similarity index 59% rename from java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index aed21ab137..3f7ba9a191 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.List; import java.util.concurrent.TimeUnit; +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; @@ -46,7 +47,7 @@ public class CPPXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_CPP_JAVA_CI"); if (!"1".equals(enabled)) { - // throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); + throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); } boolean bazelAvailable = true; try { @@ -161,142 +162,144 @@ public void testCrossLanguageSerializer() throws Exception { super.testCrossLanguageSerializer(); } - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); } - @Test - public void testList() throws java.io.IOException { - super.testList(); + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); } - @Test - public void testMap() throws java.io.IOException { - super.testMap(); + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); } - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); } - @Test - public void testItem() throws java.io.IOException { - super.testItem(); + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); } - @Test - public void testColor() throws java.io.IOException { - super.testColor(); + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); } - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); } - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); } - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); } - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); } - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); } - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); } - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); } - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); } - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { - super.testEnumSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testEnumSchemaEvolutionCompatible(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { // C++ has proper std::optional support and sends actual null values, // unlike Rust which sends default values. Override with C++-specific expectations. String caseName = "test_nullable_field_compatible_null"; @@ -304,7 +307,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -362,10 +365,20 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Assert.assertEquals(result, obj); } - @Test @Override - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { // Skip: C++ doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: C++ Union xlang support not implemented"); } + + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testRefSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java similarity index 66% rename from java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index 25b14831ec..e6b60ccd06 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; @@ -125,119 +126,119 @@ public void testCrossLanguageSerializer() throws Exception { super.testCrossLanguageSerializer(); } - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); } - @Test - public void testList() throws java.io.IOException { - super.testList(); + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); } - @Test - public void testMap() throws java.io.IOException { - super.testMap(); + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); } - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); } - @Test - public void testItem() throws java.io.IOException { - super.testItem(); + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); } - @Test - public void testColor() throws java.io.IOException { - super.testColor(); + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); } - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); } - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); } - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); } - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); } - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); } - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); } - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); } - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); } - @Test @Override - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { // Go-specific override: Go writes null for nil pointers (nullable=true by default) String caseName = "test_enum_schema_evolution_compatible"; // Fory for TwoEnumFieldStruct @@ -302,34 +303,28 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - // Go's codegen always writes null flags for slice/map/interface fields, - // which is incompatible with Java's SCHEMA_CONSISTENT mode that expects no null flags. - // TODO: Update Go code generator to respect nullable flag in SCHEMA_CONSISTENT mode. - throw new SkipException( - "Skipping: Go codegen always writes null flags, incompatible with SCHEMA_CONSISTENT mode"); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - // Go's codegen always writes null flags for slice/map/interface fields, - // which is incompatible with Java's SCHEMA_CONSISTENT mode that expects no null flags. - // TODO: Update Go code generator to respect nullable flag in SCHEMA_CONSISTENT mode. - throw new SkipException( - "Skipping: Go codegen always writes null flags, incompatible with SCHEMA_CONSISTENT mode"); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } - @Test @Override - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { // Go-specific override: Unlike Rust which has non-nullable reference types (Vec), // Go's slices and maps can be nil and default to nullable in COMPATIBLE mode. // So Go sends null for nil values, not empty collections like Rust does. @@ -338,7 +333,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -420,21 +415,35 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { expected.nullableFloat1 = 0.0f; expected.nullableDouble1 = 0.0; expected.nullableBool1 = false; - // Nullable group 2 - Go's nullable reference fields: + // Nullable group 2 - Go's reference fields: // - string (not a pointer): defaults to "" (empty string) when nil in Go - // - slices/maps: can be nil, so Go sends null + // - slices/maps: Go struct doesn't have fory:"nullable" tag, so they're non-nullable + // and are read as empty collections, not nil expected.nullableString2 = ""; - expected.nullableList2 = null; - expected.nullableSet2 = null; - expected.nullableMap2 = null; + expected.nullableList2 = new ArrayList<>(); + expected.nullableSet2 = new HashSet<>(); + expected.nullableMap2 = new HashMap<>(); Assert.assertEquals(result, expected); } - @Test @Override - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { // Skip: Go doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Go Union xlang support not implemented"); } + + @Override + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + // Run the test to debug hash mismatch + super.testRefSchemaConsistent(enableCodegen); + } + + @Override + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 903f63f68d..e03353093b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -20,13 +20,13 @@ package org.apache.fory.xlang; import lombok.Data; -import org.apache.fory.CrossLanguageTest.Bar; -import org.apache.fory.CrossLanguageTest.Foo; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.test.bean.BeanB; +import org.apache.fory.xlang.PyCrossLanguageTest.Bar; +import org.apache.fory.xlang.PyCrossLanguageTest.Foo; import org.testng.annotations.Test; public class MetaSharedXlangTest extends ForyTestBase { diff --git a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java index 47fef66f8f..6d22bbae36 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import static org.testng.Assert.assertEquals; @@ -56,6 +56,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Language; @@ -82,8 +84,8 @@ /** Tests in this class need fory python installed. */ @Test -public class CrossLanguageTest extends ForyTestBase { - private static final Logger LOG = LoggerFactory.getLogger(CrossLanguageTest.class); +public class PyCrossLanguageTest extends ForyTestBase { + private static final Logger LOG = LoggerFactory.getLogger(PyCrossLanguageTest.class); private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language"; private static final String PYTHON_EXECUTABLE = "python"; diff --git a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java similarity index 68% rename from java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java index 9afcccbac0..8953c8acc9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; @@ -88,56 +89,55 @@ public void testCrossLanguageSerializer() throws Exception { } @Override - @Test - public void testList() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testMap() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testItem() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: simple struct tests covered in CrossLanguageTest"); } @Override - @Test - public void testColor() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: enum tests covered in CrossLanguageTest"); } @Override - @Test - public void testStructWithList() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: struct with list covered in CrossLanguageTest"); } @Override - @Test - public void testStructWithMap() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: struct with map covered in CrossLanguageTest"); } - @Override @Test public void testBufferVar() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testInteger() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } // ============================================================================ // Explicitly re-declare inherited test methods to enable running individual - // tests via Maven: mvn test -Dtest=org.apache.fory.PythonXlangTest#testXxx + // tests via Maven: mvn test -Dtest=org.apache.fory.xlang.PythonXlangTest#testXxx // // Maven Surefire cannot find inherited test methods when using the #methodName // syntax for test selection. By overriding and forwarding to the parent class, @@ -151,74 +151,74 @@ public void testStringSerializer() throws Exception { } @Override - @Test - public void testSimpleStruct() throws IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws IOException { + super.testSimpleStruct(enableCodegen); } @Override - @Test - public void testSimpleNamedStruct() throws IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws IOException { + super.testSimpleNamedStruct(enableCodegen); } @Override - @Test - public void testSkipIdCustom() throws IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws IOException { + super.testSkipIdCustom(enableCodegen); } @Override - @Test - public void testSkipNameCustom() throws IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws IOException { + super.testSkipNameCustom(enableCodegen); } @Override - @Test - public void testConsistentNamed() throws IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws IOException { + super.testConsistentNamed(enableCodegen); } @Override - @Test - public void testStructVersionCheck() throws IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws IOException { + super.testStructVersionCheck(enableCodegen); } @Override - @Test - public void testPolymorphicList() throws IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws IOException { + super.testPolymorphicList(enableCodegen); } @Override - @Test - public void testPolymorphicMap() throws IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws IOException { + super.testPolymorphicMap(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNull() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws IOException { // Python properly supports Optional and sends actual null values, // unlike Rust which sends default values. Override with Python-specific expectations. String caseName = "test_nullable_field_compatible_null"; @@ -226,7 +226,7 @@ public void testNullableFieldCompatibleNull() throws IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -285,9 +285,19 @@ public void testNullableFieldCompatibleNull() throws IOException { } @Override - @Test - public void testUnionXlang() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws IOException { // Skip: Python doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Python Union xlang support not implemented"); } + + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws IOException { + super.testRefSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws IOException { + super.testRefCompatible(enableCodegen); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java new file mode 100644 index 0000000000..43e91072df --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.xlang; + +import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.testng.SkipException; +import org.testng.annotations.Test; + +/** Executes cross-language tests against the Rust implementation. */ +@Test +public class RustXlangTest extends XlangTestBase { + private static final String RUST_EXECUTABLE = "cargo"; + private static final String RUST_MODULE = "test_cross_language"; + + private static final List RUST_BASE_COMMAND = + Arrays.asList( + RUST_EXECUTABLE, + "test", + "--test", + RUST_MODULE, + "", + "--", + "--nocapture", + "--ignored", + "--exact"); + + private static final int RUST_TESTCASE_INDEX = 4; + + @Override + protected void ensurePeerReady() { + String enabled = System.getenv("FORY_RUST_JAVA_CI"); + if (!"1".equals(enabled)) { + throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); + } + boolean rustInstalled = true; + try { + Process process = new ProcessBuilder("rustc", "--version").start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + rustInstalled = false; + } + } catch (IOException | InterruptedException e) { + rustInstalled = false; + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + } + if (!rustInstalled) { + throw new SkipException("Skipping RustXlangTest: rust not installed"); + } + } + + @Override + protected CommandContext buildCommandContext(String caseName, Path dataFile) { + List command = new ArrayList<>(RUST_BASE_COMMAND); + command.set(RUST_TESTCASE_INDEX, caseName); + ImmutableMap env = + envBuilder(dataFile) + .put("RUSTFLAGS", "-Awarnings") + .put("RUST_BACKTRACE", "1") + .put("ENABLE_FORY_DEBUG_OUTPUT", "1") + .put("FORY_PANIC_ON_ERROR", "1") + .build(); + return new CommandContext(command, env, new File("../../rust")); + } + + // ============================================================================ + // Test methods - duplicated from XlangTestBase for Maven Surefire discovery + // ============================================================================ + + @Test + public void testBuffer() throws java.io.IOException { + super.testBuffer(); + } + + @Test + public void testBufferVar() throws java.io.IOException { + super.testBufferVar(); + } + + @Test + public void testMurmurHash3() throws java.io.IOException { + super.testMurmurHash3(); + } + + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); + } + + @Test + public void testCrossLanguageSerializer() throws Exception { + super.testCrossLanguageSerializer(); + } + + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testEnumSchemaEvolutionCompatible(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); + } + + @Override + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); + } + + @Override + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNull(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { + super.testUnionXlang(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testRefSchemaConsistent(enableCodegen); + } + + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); + } +} diff --git a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java similarity index 86% rename from java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index f0fcf9a9e7..0eded0f2b6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; @@ -33,6 +33,8 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.ForyTestBase; import org.apache.fory.annotation.ForyField; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; @@ -421,12 +423,14 @@ public void testStringSerializer() throws Exception { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(false) .build(); _testStringSerializer(fory, caseName); Fory foryCompress = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(false) .withStringCompressed(true) .withWriteNumUtf16BytesForUtf8Encoding(false) .build(); @@ -454,6 +458,7 @@ public void testCrossLanguageSerializer() throws Exception { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(false) .build(); fory.register(Color.class, 101); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -542,14 +547,14 @@ static class SimpleStruct { int last; // Changed from Integer to int to match Rust } - @Test - public void testSimpleStruct() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { String caseName = "test_simple_struct"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, 101); fory.register(Item.class, 102); @@ -580,14 +585,14 @@ public void testSimpleStruct() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), obj); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { String caseName = "test_named_simple_struct"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, "demo", "color"); fory.register(Item.class, "demo", "item"); @@ -618,14 +623,14 @@ public void testSimpleNamedStruct() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), obj); } - @Test - public void testList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -653,14 +658,14 @@ public void testList() throws java.io.IOException { assertEqualsNullTolerant(fory.deserialize(buffer2), itemList2); } - @Test - public void testMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -698,13 +703,13 @@ static class Item1 { Integer f6; } - @Test - public void testInteger() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { String caseName = "test_integer"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) - .withCodegen(false) + .withCodegen(enableCodegen) .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); fory.register(Item1.class, 101); @@ -752,14 +757,14 @@ public void testInteger() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), 0); } - @Test - public void testItem() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { String caseName = "test_item"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); @@ -790,14 +795,14 @@ public void testItem() throws java.io.IOException { Assert.assertEquals(readItem3.name, ""); } - @Test - public void testColor() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { String caseName = "test_color"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, 101); @@ -830,14 +835,14 @@ static class StructWithUnion2 { org.apache.fory.type.union.Union2 union; } - @Test - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { String caseName = "test_union_xlang"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithUnion2.class, 301); @@ -871,14 +876,14 @@ static class StructWithList { List items; } - @Test - public void testStructWithList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_with_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithList.class, 201); @@ -908,14 +913,14 @@ static class StructWithMap { Map data; } - @Test - public void testStructWithMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_with_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithMap.class, 202); @@ -1010,8 +1015,7 @@ private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOE MyWrapper wrapper = new MyWrapper(); wrapper.color = Color.White; MyStruct myStruct = new MyStruct(42); - MyExt myExt = new MyExt(43); - wrapper.myExt = myExt; + wrapper.myExt = new MyExt(43); wrapper.myStruct = myStruct; byte[] serialize = fory1.serialize(wrapper); ExecutionContext ctx = prepareExecution(caseName, serialize); @@ -1021,14 +1025,14 @@ private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOE Assert.assertEquals(newWrapper, new EmptyWrapper()); } - @Test - public void testSkipIdCustom() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { String caseName = "test_skip_id_custom"; Fory fory1 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory1.register(Color.class, 101); fory1.register(MyStruct.class, 102); @@ -1039,7 +1043,7 @@ public void testSkipIdCustom() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory2.register(MyExt.class, 103); fory2.registerSerializer(MyExt.class, MyExtSerializer.class); @@ -1047,14 +1051,14 @@ public void testSkipIdCustom() throws java.io.IOException { _testSkipCustom(fory1, fory2, caseName); } - @Test - public void testSkipNameCustom() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { String caseName = "test_skip_name_custom"; Fory fory1 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory1.register(Color.class, "color"); fory1.register(MyStruct.class, "my_struct"); @@ -1065,7 +1069,7 @@ public void testSkipNameCustom() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory2.register(MyExt.class, "my_ext"); fory2.registerSerializer(MyExt.class, MyExtSerializer.class); @@ -1073,14 +1077,14 @@ public void testSkipNameCustom() throws java.io.IOException { _testSkipCustom(fory1, fory2, caseName); } - @Test - public void testConsistentNamed() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { String caseName = "test_consistent_named"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .withClassVersionCheck(true) .build(); fory.register(Color.class, "color"); @@ -1127,14 +1131,14 @@ static class VersionCheckStruct { double f3; } - @Test - public void testStructVersionCheck() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_version_check"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .withClassVersionCheck(true) .build(); fory.register(VersionCheckStruct.class, 201); @@ -1209,14 +1213,14 @@ static class AnimalMapHolder { Map animal_map; } - @Test - public void testPolymorphicList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_polymorphic_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); // Register concrete types, not the interface fory.register(Dog.class, 302); @@ -1268,14 +1272,14 @@ public void testPolymorphicList() throws java.io.IOException { Assert.assertEquals(((Cat) readHolder.animals.get(1)).lives, 7); } - @Test - public void testPolymorphicMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_polymorphic_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Dog.class, 302); fory.register(Cat.class, 303); @@ -1366,13 +1370,14 @@ static class TwoStringFieldStruct { String f2; } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_string_field_schema"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) .build(); fory.register(OneStringFieldStruct.class, 200); @@ -1390,13 +1395,14 @@ public void testOneStringFieldSchemaConsistent() throws java.io.IOException { Assert.assertEquals(result.f1, "hello"); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_string_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(OneStringFieldStruct.class, 200); @@ -1414,13 +1420,14 @@ public void testOneStringFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f1, "hello"); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_two_string_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TwoStringFieldStruct.class, 201); @@ -1440,14 +1447,15 @@ public void testTwoStringFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f2, "second"); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_schema_evolution_compatible"; // Fory for TwoStringFieldStruct Fory fory2 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory2.register(TwoStringFieldStruct.class, 200); @@ -1456,6 +1464,7 @@ public void testSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); foryEmpty.register(EmptyStruct.class, 200); @@ -1463,6 +1472,7 @@ public void testSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory1.register(OneStringFieldStruct.class, 200); @@ -1522,13 +1532,14 @@ static class TwoEnumFieldStruct { TestEnum f2; } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_enum_field_schema"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(OneEnumFieldStruct.class, 211); @@ -1547,13 +1558,14 @@ public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { Assert.assertEquals(result.f1, TestEnum.VALUE_B); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_enum_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(OneEnumFieldStruct.class, 211); @@ -1572,13 +1584,14 @@ public void testOneEnumFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f1, TestEnum.VALUE_A); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_two_enum_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(TwoEnumFieldStruct.class, 212); @@ -1599,14 +1612,15 @@ public void testTwoEnumFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f2, TestEnum.VALUE_C); } - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_enum_schema_evolution_compatible"; // Fory for TwoEnumFieldStruct Fory fory2 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory2.register(TestEnum.class, 210); fory2.register(TwoEnumFieldStruct.class, 211); @@ -1616,6 +1630,7 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); foryEmpty.register(TestEnum.class, 210); foryEmpty.register(EmptyStruct.class, 211); @@ -1624,6 +1639,7 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory1.register(TestEnum.class, 210); fory1.register(OneEnumFieldStruct.class, 211); @@ -1732,14 +1748,15 @@ static class NullableComprehensiveSchemaConsistent { Map nullableMap; } - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_not_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(NullableComprehensiveSchemaConsistent.class, 401); @@ -1790,14 +1807,15 @@ public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOExceptio Assert.assertEquals(result, obj); } - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(NullableComprehensiveSchemaConsistent.class, 401); @@ -1915,14 +1933,14 @@ static class NullableComprehensiveCompatible { Map nullableMap2; } - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_compatible_not_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -1981,14 +1999,14 @@ public void testNullableFieldCompatibleNotNull() throws java.io.IOException { Assert.assertEquals(result, obj); } - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_compatible_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -2080,43 +2098,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Assert.assertEquals(result, expected); } - // Keep the old simple structs for backward compatibility with existing tests - @Data - static class NullableFieldStruct { - int intField; - long longField; - float floatField; - double doubleField; - boolean boolField; - String stringField; - - @ForyField(nullable = true) - String nullableString1; - - @ForyField(nullable = true) - String nullableString2; - } - - @Data - static class NullableFieldStructCompatible { - int intField; - long longField; - float floatField; - double doubleField; - boolean boolField; - String stringField; - - @ForyField(nullable = true) - String nullableString1; - - @ForyField(nullable = true) - String nullableString2; - - @ForyField(nullable = true) - String nullableString3; - } - - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "rawtypes"}) private void assertStringEquals(Object actual, Object expected, boolean useToString) { if (useToString) { if (expected instanceof Map) { @@ -2156,7 +2138,6 @@ private void assertStringEquals(Object actual, Object expected, boolean useToStr * null strings become empty strings "". This method first tries direct comparison, then * normalizes null to empty string and compares again. Prints normalized values on mismatch. */ - @SuppressWarnings("unchecked") protected void assertEqualsNullTolerant(Object actual, Object expected) { // First try direct comparison if (Objects.equals(actual, expected)) { @@ -2184,8 +2165,166 @@ protected void assertEqualsNullTolerant(Object actual, Object expected) { } } + // ============================================================================ + // Reference Tracking Tests - Test struct field reference sharing + // ============================================================================ + + /** + * Inner struct for reference tracking tests in SCHEMA_CONSISTENT mode (compatible=false). A + * simple struct with id and name fields. + */ + @Data + static class RefInnerSchemaConsistent { + int id; + String name; + } + + /** + * Outer struct for reference tracking tests in SCHEMA_CONSISTENT mode. Contains two fields that + * can point to the same RefInnerSchemaConsistent instance. Both fields have ref tracking enabled. + */ + @Data + static class RefOuterSchemaConsistent { + @ForyField(ref = true, nullable = true, morphic = ForyField.Morphic.FINAL) + RefInnerSchemaConsistent inner1; + + @ForyField(ref = true, nullable = true, morphic = ForyField.Morphic.FINAL) + RefInnerSchemaConsistent inner2; + } + + /** + * Test reference tracking in SCHEMA_CONSISTENT mode (compatible=false). Creates an outer struct + * with two fields pointing to the same inner struct instance. Verifies that after + * serialization/deserialization across languages, both fields still reference the same object. + */ + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + String caseName = "test_ref_schema_consistent"; + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withRefTracking(true) + .withCodegen(enableCodegen) + .build(); + fory.register(RefInnerSchemaConsistent.class, 501); + fory.register(RefOuterSchemaConsistent.class, 502); + + // Create inner struct + RefInnerSchemaConsistent inner = new RefInnerSchemaConsistent(); + inner.id = 42; + inner.name = "shared_inner"; + + // Create outer struct with both fields pointing to the same inner struct + RefOuterSchemaConsistent outer = new RefOuterSchemaConsistent(); + outer.inner1 = inner; + outer.inner2 = inner; // Same reference as inner1 + + // Verify Java serialization preserves reference identity + byte[] javaBytes = fory.serialize(outer); + RefOuterSchemaConsistent javaResult = (RefOuterSchemaConsistent) fory.deserialize(javaBytes); + Assert.assertSame( + javaResult.inner1, javaResult.inner2, "Java: inner1 and inner2 should be same object"); + Assert.assertEquals(javaResult.inner1.id, 42); + Assert.assertEquals(javaResult.inner1.name, "shared_inner"); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, outer); + + ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); + runPeer(ctx); + + MemoryBuffer buffer2 = readBuffer(ctx.dataFile()); + RefOuterSchemaConsistent result = (RefOuterSchemaConsistent) fory.deserialize(buffer2); + + // Verify reference identity is preserved after cross-language round-trip + Assert.assertSame( + result.inner1, + result.inner2, + "After xlang round-trip: inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1.id, 42); + Assert.assertEquals(result.inner1.name, "shared_inner"); + } + + /** + * Inner struct for reference tracking tests in COMPATIBLE mode (compatible=true). A simple struct + * with id and name fields. + */ + @Data + static class RefInnerCompatible { + int id; + String name; + } + + /** + * Outer struct for reference tracking tests in COMPATIBLE mode. Contains two fields that can + * point to the same RefInnerCompatible instance. Both fields have ref tracking enabled. + */ + @Data + static class RefOuterCompatible { + @ForyField(ref = true, nullable = true) + RefInnerCompatible inner1; + + @ForyField(ref = true, nullable = true) + RefInnerCompatible inner2; + } + + /** + * Test reference tracking in COMPATIBLE mode (compatible=true). Creates an outer struct with two + * fields pointing to the same inner struct instance. Verifies that after + * serialization/deserialization across languages, both fields still reference the same object. + */ + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + String caseName = "test_ref_compatible"; + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withRefTracking(true) + .withCodegen(enableCodegen) + .withMetaCompressor(new NoOpMetaCompressor()) + .build(); + fory.register(RefInnerCompatible.class, 503); + fory.register(RefOuterCompatible.class, 504); + + // Create inner struct + RefInnerCompatible inner = new RefInnerCompatible(); + inner.id = 99; + inner.name = "compatible_shared"; + + // Create outer struct with both fields pointing to the same inner struct + RefOuterCompatible outer = new RefOuterCompatible(); + outer.inner1 = inner; + outer.inner2 = inner; // Same reference as inner1 + + // Verify Java serialization preserves reference identity + byte[] javaBytes = fory.serialize(outer); + RefOuterCompatible javaResult = (RefOuterCompatible) fory.deserialize(javaBytes); + Assert.assertSame( + javaResult.inner1, javaResult.inner2, "Java: inner1 and inner2 should be same object"); + Assert.assertEquals(javaResult.inner1.id, 99); + Assert.assertEquals(javaResult.inner1.name, "compatible_shared"); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, outer); + + ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); + runPeer(ctx); + + MemoryBuffer buffer2 = readBuffer(ctx.dataFile()); + RefOuterCompatible result = (RefOuterCompatible) fory.deserialize(buffer2); + + // Verify reference identity is preserved after cross-language round-trip + Assert.assertSame( + result.inner1, + result.inner2, + "After xlang round-trip: inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1.id, 99); + Assert.assertEquals(result.inner1.name, "compatible_shared"); + } + /** Normalize null values to empty strings in collections and maps recursively. */ - @SuppressWarnings("unchecked") private Object normalizeNulls(Object obj) { if (obj == null) { return ""; diff --git a/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java b/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java index 7970de4f24..7548fd4375 100644 --- a/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java +++ b/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java @@ -19,10 +19,8 @@ package org.apache.fory.test; -import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -64,23 +62,13 @@ private static ProcessBuilder buildProcess( private static boolean executeCommand( ProcessBuilder processBuilder, List command, int waitTimeoutSeconds) { try { + processBuilder.inheritIO(); Process process = processBuilder.start(); - // Capture output to log - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - BufferedReader errorReader = - new BufferedReader(new InputStreamReader(process.getErrorStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - while ((line = errorReader.readLine()) != null) { - System.err.println(line); - } boolean finished = process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); if (finished) { return process.exitValue() == 0; } else { - process.destroy(); // ensure the process is terminated + process.destroy(); return false; } } catch (Exception e) { diff --git a/python/pyfory/_serializer.py b/python/pyfory/_serializer.py index e1a8231985..33bbe6e003 100644 --- a/python/pyfory/_serializer.py +++ b/python/pyfory/_serializer.py @@ -39,7 +39,7 @@ class Serializer(ABC): def __init__(self, fory, type_: type): self.fory = fory self.type_: type = type_ - self.need_to_write_ref = not is_primitive_type(type_) + self.need_to_write_ref = fory.ref_tracking and not is_primitive_type(type_) def write(self, buffer, value): raise NotImplementedError diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index e800f59300..2e9769fbea 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -638,7 +638,7 @@ cdef int32_t NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF =KEY_HAS_NULL | VALUE_DECL_TY # Value is null, key type is declared type, and ref tracking for key is disabled. cdef int32_t NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE # Value is null, key type is declared type, and ref tracking for key is enabled. -cdef int32_t NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_VALUE_REF +cdef int32_t NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF @cython.final diff --git a/python/pyfory/collection.py b/python/pyfory/collection.py index 451d05b4d1..f8a026ddc7 100644 --- a/python/pyfory/collection.py +++ b/python/pyfory/collection.py @@ -342,7 +342,7 @@ def get_next_element(buffer, ref_resolver, type_resolver, is_py): # Value is null, key type is declared type, and ref tracking for key is disabled. NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE # Value is null, key type is declared type, and ref tracking for key is enabled. -NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_VALUE_REF +NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF class MapSerializer(Serializer): diff --git a/python/pyfory/meta/typedef_encoder.py b/python/pyfory/meta/typedef_encoder.py index fd4d7d3423..ae0e56bb65 100644 --- a/python/pyfory/meta/typedef_encoder.py +++ b/python/pyfory/meta/typedef_encoder.py @@ -97,8 +97,8 @@ def encode_typedef(type_resolver, cls): # Write fields info write_fields_info(type_resolver, buffer, field_infos) - # Get the encoded binary - binary = buffer.to_bytes() + # Get the encoded binary (only the written portion, not the full buffer) + binary = buffer.to_bytes(0, buffer.writer_index) # Compress if beneficial compressed_binary = type_resolver.get_meta_compressor().compress(binary) diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index a687b64a70..592de476ea 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -1694,7 +1694,7 @@ cdef class Serializer: def __init__(self, fory, type_: Union[type, TypeVar]): self.fory = fory self.type_ = type_ - self.need_to_write_ref = not is_primitive_type(type_) + self.need_to_write_ref = fory.ref_tracking and not is_primitive_type(type_) cpdef write(self, Buffer buffer, value): raise NotImplementedError(f"write method not implemented in {type(self)}") diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py index 53e09e0b67..d9c0ad0aaa 100644 --- a/python/pyfory/struct.py +++ b/python/pyfory/struct.py @@ -1053,6 +1053,11 @@ def visit_dict(self, field_name, key_type, value_type, types_path=None): def visit_customized(self, field_name, type_, types_path=None): if issubclass(type_, enum.Enum): return self.fory.type_resolver.get_serializer(type_) + # For custom types (dataclasses, etc.), try to get or create serializer + # This enables field-level serializer resolution for types like inner structs + typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False) + if typeinfo is not None: + return typeinfo.serializer return None def visit_other(self, field_name, type_, types_path=None): diff --git a/python/pyfory/tests/xlang_test_main.py b/python/pyfory/tests/xlang_test_main.py index 4ca5913d93..f96f477c77 100644 --- a/python/pyfory/tests/xlang_test_main.py +++ b/python/pyfory/tests/xlang_test_main.py @@ -219,6 +219,43 @@ class NullableComprehensiveSchemaConsistent: nullable_map: Optional[Dict[str, str]] = None +# ============================================================================ +# Reference Tracking Test Types +# ============================================================================ + + +@dataclass +class RefInnerSchemaConsistent: + """Inner struct for reference tracking test (SCHEMA_CONSISTENT mode).""" + + id: pyfory.int32 = 0 + name: str = "" + + +@dataclass +class RefOuterSchemaConsistent: + """Outer struct with two fields pointing to the same inner object (SCHEMA_CONSISTENT mode).""" + + inner1: Optional[RefInnerSchemaConsistent] = pyfory.field(default=None, ref=True, nullable=True) + inner2: Optional[RefInnerSchemaConsistent] = pyfory.field(default=None, ref=True, nullable=True) + + +@dataclass +class RefInnerCompatible: + """Inner struct for reference tracking test (COMPATIBLE mode).""" + + id: pyfory.int32 = 0 + name: str = "" + + +@dataclass +class RefOuterCompatible: + """Outer struct with two fields pointing to the same inner object (COMPATIBLE mode).""" + + inner1: Optional[RefInnerCompatible] = pyfory.field(default=None, ref=True, nullable=True) + inner2: Optional[RefInnerCompatible] = pyfory.field(default=None, ref=True, nullable=True) + + @dataclass class NullableComprehensiveCompatible: """ @@ -1104,6 +1141,87 @@ def test_nullable_field_compatible_null(): f.write(new_bytes) +# ============================================================================ +# Reference Tracking Tests +# ============================================================================ + + +def test_ref_schema_consistent(): + """ + Test cross-language reference tracking in SCHEMA_CONSISTENT mode (compatible=false). + + This test verifies that when Java serializes an object where two fields point to + the same instance, Python can properly deserialize it and both fields will reference + the same object. When re-serializing, the reference relationship should be preserved. + """ + data_file = get_data_file() + with open(data_file, "rb") as f: + data_bytes = f.read() + + fory = pyfory.Fory(xlang=True, compatible=False, ref=True) + fory.register_type(RefInnerSchemaConsistent, type_id=501) + fory.register_type(RefOuterSchemaConsistent, type_id=502) + + outer = fory.deserialize(data_bytes) + debug_print(f"Deserialized: {outer}") + + # Both inner1 and inner2 should have values + assert outer.inner1 is not None, "inner1 should not be None" + assert outer.inner2 is not None, "inner2 should not be None" + + # Both should have the same values (they reference the same object in Java) + assert outer.inner1.id == 42, f"inner1.id should be 42, got {outer.inner1.id}" + assert outer.inner1.name == "shared_inner", f"inner1.name should be 'shared_inner', got {outer.inner1.name}" + assert outer.inner1 == outer.inner2, "inner1 and inner2 should be equal (same reference)" + + # In Python, after deserialization with reference tracking, inner1 and inner2 + # should point to the same object (identity check) + assert outer.inner1 is outer.inner2, "inner1 and inner2 should be the same object (reference identity)" + + # Re-serialize and write back + new_bytes = fory.serialize(outer) + with open(data_file, "wb") as f: + f.write(new_bytes) + + +def test_ref_compatible(): + """ + Test cross-language reference tracking in COMPATIBLE mode (compatible=true). + + This test verifies reference tracking works correctly with schema evolution support. + The inner object is shared between two fields, and this relationship should be + preserved through serialization/deserialization. + """ + data_file = get_data_file() + with open(data_file, "rb") as f: + data_bytes = f.read() + + fory = pyfory.Fory(xlang=True, compatible=True, ref=True) + fory.register_type(RefInnerCompatible, type_id=503) + fory.register_type(RefOuterCompatible, type_id=504) + + outer = fory.deserialize(data_bytes) + debug_print(f"Deserialized: {outer}") + + # Both inner1 and inner2 should have values + assert outer.inner1 is not None, "inner1 should not be None" + assert outer.inner2 is not None, "inner2 should not be None" + + # Both should have the same values (they reference the same object in Java) + assert outer.inner1.id == 99, f"inner1.id should be 99, got {outer.inner1.id}" + assert outer.inner1.name == "compatible_shared", f"inner1.name should be 'compatible_shared', got {outer.inner1.name}" + assert outer.inner1 == outer.inner2, "inner1 and inner2 should be equal (same reference)" + + # In Python, after deserialization with reference tracking, inner1 and inner2 + # should point to the same object (identity check) + assert outer.inner1 is outer.inner2, "inner1 and inner2 should be the same object (reference identity)" + + # Re-serialize and write back + new_bytes = fory.serialize(outer) + with open(data_file, "wb") as f: + f.write(new_bytes) + + if __name__ == "__main__": """ This file is executed by PythonXlangTest.java and other cross-language tests. diff --git a/rust/fory-core/src/config.rs b/rust/fory-core/src/config.rs index 696ab8e198..991db1d724 100644 --- a/rust/fory-core/src/config.rs +++ b/rust/fory-core/src/config.rs @@ -34,6 +34,10 @@ pub struct Config { pub max_dyn_depth: u32, /// Whether class version checking is enabled. pub check_struct_version: bool, + /// Whether reference tracking is enabled. + /// When enabled, shared references and circular references are tracked + /// and preserved during serialization/deserialization. + pub track_ref: bool, } impl Default for Config { @@ -45,6 +49,7 @@ impl Default for Config { compress_string: false, max_dyn_depth: 5, check_struct_version: false, + track_ref: false, } } } @@ -90,4 +95,10 @@ impl Config { pub fn is_check_struct_version(&self) -> bool { self.check_struct_version } + + /// Check if reference tracking is enabled. + #[inline(always)] + pub fn is_track_ref(&self) -> bool { + self.track_ref + } } diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 3b941885d3..165de4bfc4 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -260,6 +260,34 @@ impl Fory { self } + /// Enables or disables reference tracking for shared and circular references. + /// + /// # Arguments + /// + /// * `track_ref` - If `true`, enables reference tracking which allows + /// preserving shared object references and circular references during + /// serialization/deserialization. + /// + /// # Returns + /// + /// Returns `self` for method chaining. + /// + /// # Default + /// + /// The default value is `false`. + /// + /// # Examples + /// + /// ```rust + /// use fory_core::Fory; + /// + /// let fory = Fory::default().track_ref(true); + /// ``` + pub fn track_ref(mut self, track_ref: bool) -> Self { + self.config.track_ref = track_ref; + self + } + /// Sets the maximum depth for nested dynamic object serialization. /// /// # Arguments @@ -590,8 +618,10 @@ impl Fory { if context.is_compatible() { context.writer.write_i32(-1); }; - // Use RefMode::Tracking for shared ref types (Rc, Arc, RcWeak, ArcWeak) - let ref_mode = if T::fory_is_shared_ref() { + // Use RefMode based on config: + // - If track_ref is enabled, use RefMode::Tracking for the root object + // - Otherwise, use RefMode::NullOnly which writes NOT_NULL_VALUE_FLAG + let ref_mode = if self.config.track_ref { RefMode::Tracking } else { RefMode::NullOnly @@ -983,8 +1013,10 @@ impl Fory { bytes_to_skip = context.load_type_meta(meta_offset as usize)?; } } - // Use RefMode::Tracking for shared ref types (Rc, Arc, RcWeak, ArcWeak) - let ref_mode = if T::fory_is_shared_ref() { + // Use RefMode based on config: + // - If track_ref is enabled, use RefMode::Tracking for the root object + // - Otherwise, use RefMode::NullOnly + let ref_mode = if self.config.track_ref { RefMode::Tracking } else { RefMode::NullOnly diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 615c7ac703..701fb75c06 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -120,6 +120,7 @@ pub struct WriteContext<'a> { compress_string: bool, xlang: bool, check_struct_version: bool, + track_ref: bool, // Context-specific fields default_writer: Option>, @@ -139,6 +140,7 @@ impl<'a> WriteContext<'a> { compress_string: config.compress_string, xlang: config.xlang, check_struct_version: config.check_struct_version, + track_ref: config.track_ref, default_writer: None, writer: Writer::from_buffer(Self::get_leak_buffer()), meta_resolver: MetaWriterResolver::default(), @@ -205,6 +207,12 @@ impl<'a> WriteContext<'a> { self.check_struct_version } + /// Check if reference tracking is enabled + #[inline(always)] + pub fn is_track_ref(&self) -> bool { + self.track_ref + } + #[inline(always)] pub fn empty(&mut self) -> bool { self.meta_resolver.empty() diff --git a/rust/fory-core/src/resolver/ref_resolver.rs b/rust/fory-core/src/resolver/ref_resolver.rs index 4c2c804719..3367c82d27 100644 --- a/rust/fory-core/src/resolver/ref_resolver.rs +++ b/rust/fory-core/src/resolver/ref_resolver.rs @@ -131,6 +131,21 @@ impl RefWriter { } } + /// Reserve a reference ID slot without storing anything. + /// + /// This is used for xlang compatibility where ALL objects (including struct values, + /// not just Rc/Arc) participate in reference tracking. + /// + /// # Returns + /// + /// The reserved reference ID + #[inline(always)] + pub fn reserve_ref_id(&mut self) -> u32 { + let ref_id = self.next_ref_id; + self.next_ref_id += 1; + ref_id + } + /// Clear all stored references. /// /// This is useful for reusing the RefWriter for multiple serialization operations. diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index d8685bba99..7aecffa452 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -38,48 +38,18 @@ impl Serializer for Arc match ref_mode { RefMode::None => { // No ref flag - write inner directly - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::NullOnly => { // Only null check, no ref tracking context.writer.write_i8(RefFlag::NotNullValue as i8); - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::Tracking => { // Full ref tracking with RefWriter @@ -90,40 +60,30 @@ impl Serializer for Arc // Already written as ref - done return Ok(()); } - // First occurrence - write inner - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + // First occurrence - write type info and data + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } } } - fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { - Err(Error::not_allowed( - "Arc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Arc where T is a shared ref type is not allowed for serialization.", + )); + } + T::fory_write_data_generic(&**self, context, has_generics) } - fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { - Err(Error::not_allowed( - "Arc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { @@ -149,8 +109,14 @@ impl Serializer for Arc read_arc(context, ref_mode, false, Some(typeinfo)) } - fn fory_read_data(_: &mut ReadContext) -> Result { - Err(Error::not_allowed("Arc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) + fn fory_read_data(context: &mut ReadContext) -> Result { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Arc where T is a shared ref type is not allowed for deserialization.", + )); + } + let inner = T::fory_read_data(context)?; + Ok(Arc::new(inner)) } fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { @@ -233,21 +199,10 @@ fn read_arc_inner( read_type_info: bool, typeinfo: Option>, ) -> Result { + // Read type info if needed, then read data directly + // No recursive ref handling needed since Arc only wraps allowed types if let Some(typeinfo) = typeinfo { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read_with_type_info(context, inner_ref_mode, typeinfo); - } - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read(context, inner_ref_mode, read_type_info); + return T::fory_read_with_type_info(context, RefMode::None, typeinfo); } if read_type_info { T::fory_read_type_info(context)?; diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index ab90d845d8..4bfd6373f8 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -37,48 +37,18 @@ impl Serializer for Rc { match ref_mode { RefMode::None => { // No ref flag - write inner directly - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::NullOnly => { // Only null check, no ref tracking context.writer.write_i8(RefFlag::NotNullValue as i8); - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::Tracking => { // Full ref tracking with RefWriter @@ -89,40 +59,30 @@ impl Serializer for Rc { // Already written as ref - done return Ok(()); } - // First occurrence - write inner - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + // First occurrence - write type info and data + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } } } - fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { - Err(Error::not_allowed( - "Rc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Rc where T is a shared ref type is not allowed for serialization.", + )); + } + T::fory_write_data_generic(&**self, context, has_generics) } - fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { - Err(Error::not_allowed( - "Rc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { @@ -148,8 +108,14 @@ impl Serializer for Rc { read_rc(context, ref_mode, false, Some(typeinfo)) } - fn fory_read_data(_: &mut ReadContext) -> Result { - Err(Error::not_allowed("Rc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) + fn fory_read_data(context: &mut ReadContext) -> Result { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Rc where T is a shared ref type is not allowed for deserialization.", + )); + } + let inner = T::fory_read_data(context)?; + Ok(Rc::new(inner)) } fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { @@ -232,21 +198,10 @@ fn read_rc_inner( read_type_info: bool, typeinfo: Option>, ) -> Result { + // Read type info if needed, then read data directly + // No recursive ref handling needed since Rc only wraps allowed types if let Some(typeinfo) = typeinfo { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read_with_type_info(context, inner_ref_mode, typeinfo); - } - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read(context, inner_ref_mode, read_type_info); + return T::fory_read_with_type_info(context, RefMode::None, typeinfo); } if read_type_info { T::fory_read_type_info(context)?; diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 7d6a2a574a..31ae7d9f5f 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -95,8 +95,17 @@ pub fn write( ref_mode: RefMode, write_type_info: bool, ) -> Result<(), Error> { - if ref_mode != RefMode::None { - context.writer.write_i8(RefFlag::NotNullValue as i8); + match ref_mode { + RefMode::None => {} + RefMode::NullOnly => { + context.writer.write_i8(RefFlag::NotNullValue as i8); + } + RefMode::Tracking => { + // For ref tracking mode, write RefValue flag and reserve a ref_id + // so this struct participates in reference tracking. + context.writer.write_i8(RefFlag::RefValue as i8); + context.ref_writer.reserve_ref_id(); + } } if write_type_info { T::fory_write_type_info(context)?; diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 1cc91f40a0..c4bd3f4ee7 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -319,6 +319,12 @@ impl Serializer for RcWeak { write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { + // Weak pointers require track_ref to be enabled on the Fory instance + if !context.is_track_ref() { + return Err(Error::invalid_ref( + "RcWeak requires track_ref to be enabled. Use Fory::default().track_ref(true)", + )); + } // Weak MUST use ref tracking - otherwise read value will be lost if ref_mode != RefMode::Tracking { return Err(Error::invalid_ref( @@ -479,6 +485,12 @@ impl Serializer for ArcWeak write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { + // Weak pointers require track_ref to be enabled on the Fory instance + if !context.is_track_ref() { + return Err(Error::invalid_ref( + "ArcWeak requires track_ref to be enabled. Use Fory::default().track_ref(true)", + )); + } // Weak MUST use ref tracking - otherwise read value will be lost if ref_mode != RefMode::Tracking { return Err(Error::invalid_ref( diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index 86c2e3eb90..ec2c28b598 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -65,10 +65,10 @@ pub enum RefMode { } impl RefMode { - /// Create RefMode from nullable and ref_tracking flags. + /// Create RefMode from nullable and track_ref flags. #[inline] - pub const fn from_flags(nullable: bool, ref_tracking: bool) -> Self { - match (nullable, ref_tracking) { + pub const fn from_flags(nullable: bool, track_ref: bool) -> Self { + match (nullable, track_ref) { (false, false) => RefMode::None, (true, false) => RefMode::NullOnly, (_, true) => RefMode::Tracking, @@ -405,9 +405,12 @@ pub const fn is_internal_type(type_id: u32) -> bool { ) } -/// Keep as const fn for compile time evaluation or constant folding +/// Keep as const fn for compile time evaluation or constant folding. +/// Returns true if this type needs type info written in compatible mode. +/// Only user-defined types (struct, ext, unknown) need type info. +/// Internal types (primitives, strings, collections, enums) don't need type info. #[inline(always)] -pub(crate) const fn need_to_write_type_for_field(type_id: TypeId) -> bool { +pub const fn need_to_write_type_for_field(type_id: TypeId) -> bool { matches!( type_id, TypeId::STRUCT diff --git a/rust/fory-derive/src/object/field_meta.rs b/rust/fory-derive/src/object/field_meta.rs index 1a252f5ab6..ff1a68793d 100644 --- a/rust/fory-derive/src/object/field_meta.rs +++ b/rust/fory-derive/src/object/field_meta.rs @@ -203,6 +203,11 @@ fn extract_option_inner_type(ty: &Type) -> Option { None } +/// Returns true if the outer type is Option, regardless of inner type +pub fn is_option_type(ty: &Type) -> bool { + extract_outer_type_name(ty) == "Option" +} + /// Classify a field type to determine default nullable/ref behavior pub fn classify_field_type(ty: &Type) -> FieldTypeClass { let type_name = extract_outer_type_name(ty); diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index bc1f48123e..582f82e8a0 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -20,7 +20,7 @@ use quote::quote; use std::sync::atomic::{AtomicU32, Ordering}; use syn::Field; -use super::field_meta::{classify_field_type, parse_field_meta}; +use super::field_meta::{classify_field_type, is_option_type, parse_field_meta}; use super::util::{ classify_trait_object_field, generic_tree_to_tokens, get_filtered_source_fields_iter, get_sort_fields_ts, parse_generic_tree, StructField, @@ -82,7 +82,11 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { // Parse field metadata for nullable/ref tracking and field ID let meta = parse_field_meta(field).unwrap_or_default(); let type_class = classify_field_type(ty); - let nullable = meta.effective_nullable(type_class); + // For nullable, check both the classified type AND whether outer type is Option + // This handles Option> correctly - classify_field_type returns Rc for ref_tracking, + // but we also need to detect that the outer wrapper is Option for nullable. + let is_outer_option = is_option_type(ty); + let nullable = meta.effective_nullable(type_class) || is_outer_option; let ref_tracking = meta.effective_ref_tracking(type_class); // Only use explicit field ID when user sets #[fory(id = N)] // Otherwise use -1 to indicate field name encoding should be used diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 1bf09fbc56..2e7aa12f50 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -22,9 +22,8 @@ use syn::Field; use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, - get_primitive_reader_method, get_struct_name, is_debug_enabled, - is_direct_primitive_numeric_type, is_primitive_type, is_skip_field, - should_skip_type_info_for_field, FieldRefMode, StructField, + get_primitive_reader_method, get_struct_name, is_debug_enabled, is_direct_primitive_type, + is_primitive_type, is_skip_field, should_skip_type_info_for_field, FieldRefMode, StructField, }; use crate::util::SourceField; @@ -213,7 +212,7 @@ pub fn gen_read_field(field: &Field, private_ident: &Ident, field_name: &str) -> } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info quote! { let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, true)?; } @@ -221,41 +220,52 @@ pub fn gen_read_field(field: &Field, private_ident: &Ident, field_name: &str) -> _ => { let skip_type_info = should_skip_type_info_for_field(ty); - // Check if this is a direct primitive numeric type that can use direct reader calls - if is_direct_primitive_numeric_type(ty) { + // Check if this is a direct primitive type that can use direct reader calls + // Only apply when ref_mode is None (no ref tracking needed) + if ref_mode == FieldRefMode::None && is_direct_primitive_type(ty) { let type_name = extract_type_name(ty); - let reader_method = get_primitive_reader_method(&type_name); - let reader_ident = syn::Ident::new(reader_method, proc_macro2::Span::call_site()); - quote! { - let #private_ident = context.reader.#reader_ident()?; - } - } else if skip_type_info { - // Known types (primitives, strings, collections) - skip type info at compile time - if ref_mode == FieldRefMode::None { + if type_name == "String" { + // String: call fory_read_data directly quote! { let #private_ident = <#ty as fory_core::Serializer>::fory_read_data(context)?; } } else { + // Numeric primitives: use direct buffer methods + let reader_method = get_primitive_reader_method(&type_name); + let reader_ident = + syn::Ident::new(reader_method, proc_macro2::Span::call_site()); quote! { - let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, false)?; + let #private_ident = context.reader.#reader_ident()?; } } - } else { - // Custom types (struct/enum/ext) - need runtime check for enums + } else if skip_type_info { + // Known types (primitives, strings, collections) - skip type info at compile time if ref_mode == FieldRefMode::None { quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - if need_type_info { - <#ty as fory_core::Serializer>::fory_read_type_info(context)?; - } let #private_ident = <#ty as fory_core::Serializer>::fory_read_data(context)?; } } else { quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, need_type_info)?; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, false)?; } } + } else { + // Custom types (struct/enum/ext) - always need mode-dependent type info logic + // Determine read_type_info based on mode: + // - compatible=true: use need_to_write_type_for_field (struct types need type info) + // - compatible=false: use fory_is_polymorphic + // This applies regardless of ref_mode because Java always writes type info + // for struct-type fields in compatible mode, even for non-nullable fields. + quote! { + let read_type_info = if context.is_compatible() { + fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ) + } else { + <#ty as fory_core::Serializer>::fory_is_polymorphic() + }; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, read_type_info)?; + } } } }; @@ -435,14 +445,21 @@ pub(crate) fn gen_read_compatible_match_arm_body( } StructField::VecBox(_) => { // Vec> uses standard Vec deserialization with polymorphic elements - // Check nullable flag from remote field info to determine if ref flag was written + // Check nullable and ref_tracking flags from remote field info quote! { let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -450,14 +467,21 @@ pub(crate) fn gen_read_compatible_match_arm_body( } StructField::HashMapBox(_, _) => { // HashMap> uses standard HashMap deserialization with polymorphic values - // Check nullable flag from remote field info to determine if ref flag was written + // Check nullable and ref_tracking flags from remote field info quote! { let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -494,9 +518,23 @@ pub(crate) fn gen_read_compatible_match_arm_body( } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info + // Use remote field's ref_tracking flag for ref_mode quote! { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, #ref_mode, true)?); + let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( + _field.field_type.type_id, + _field.field_type.nullable, + ); + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // Forward types are polymorphic, always read type info + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, true)?); } } StructField::None => { @@ -509,8 +547,16 @@ pub(crate) fn gen_read_compatible_match_arm_body( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -521,8 +567,16 @@ pub(crate) fn gen_read_compatible_match_arm_body( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = <#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?; + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = <#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?; } else { #var_name = <#ty as fory_core::Serializer>::fory_read_data(context)?; } @@ -530,26 +584,42 @@ pub(crate) fn gen_read_compatible_match_arm_body( } } else if dec_by_option { quote! { - let read_type_info = fory_core::serializer::util::field_need_read_type_info(_field.field_type.type_id); let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - let ref_mode = if read_ref_flag { fory_core::RefMode::NullOnly } else { fory_core::RefMode::None }; - // Always use fory_read which handles compatible mode correctly - // for nested struct types with different registered IDs + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // For ref-tracked struct types, Java writes type info after RefValue flag + let read_type_info = fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ); #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, read_type_info)?); } } else { quote! { - let read_type_info = fory_core::serializer::util::field_need_read_type_info(_field.field_type.type_id); let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - let ref_mode = if read_ref_flag { fory_core::RefMode::NullOnly } else { fory_core::RefMode::None }; - // Always use fory_read which handles compatible mode correctly - // for nested struct types with different registered IDs + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // For ref-tracked struct types, Java writes type info after RefValue flag + let read_type_info = fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ); #var_name = <#ty as fory_core::Serializer>::fory_read(context, ref_mode, read_type_info)?; } } @@ -593,6 +663,13 @@ pub fn gen_read(_struct_ident: &Ident) -> TokenStream { fory_core::RefFlag::NotNullValue as i8 }; if ref_flag == (fory_core::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::RefFlag::RefValue as i8) { + // For RefValueFlag with Tracking mode, reserve a ref_id to participate in ref tracking. + // This is needed for xlang compatibility where all objects (not just Rc/Arc) + // participate in reference tracking when ref tracking is enabled. + // Only reserve for Tracking mode, not NullOnly mode. + if ref_flag == (fory_core::RefFlag::RefValue as i8) && ref_mode == fory_core::RefMode::Tracking { + context.ref_reader.reserve_ref_id(); + } if context.is_compatible() { let type_info = if read_type_info { context.read_any_typeinfo()? diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 3b1ed38ebd..be25df9b19 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -208,6 +208,9 @@ fn is_forward_field_internal(ty: &Type, struct_name: &str) -> bool { } // Check smart pointers: Rc / Arc + // Only return true if: + // 1. Inner type is Rc (polymorphic) + // 2. Inner type references the containing struct (forward reference) if seg.ident == "Rc" || seg.ident == "Arc" { if let PathArguments::AngleBracketed(args) = &seg.arguments { if let Some(GenericArgument::Type(inner_ty)) = args.args.first() { @@ -226,9 +229,10 @@ fn is_forward_field_internal(ty: &Type, struct_name: &str) -> bool { return false; } } - // Inner type is not a trait object → return true + // Inner type is not a trait object - recursively check + // if it references the containing struct _ => { - return true; + return is_forward_field_internal(inner_ty, struct_name); } } } @@ -590,6 +594,18 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { (!PRIMITIVE_TYPE_NAMES.contains(&node.name.as_str()), node) }; + // If Rc or Arc, unwrap to inner type - these are reference wrappers + // that don't add type info to the field type (handled by ref_tracking flag) + let base_node = if base_node.name == "Rc" || base_node.name == "Arc" { + if let Some(inner) = base_node.generics.first() { + inner + } else { + base_node + } + } else { + base_node + }; + // `Vec>` rule stays as is if let Some(ts) = try_vec_of_option_primitive(base_node) { return ts; @@ -709,13 +725,18 @@ static PRIMITIVE_IO_METHODS: &[(&str, &str, &str)] = &[ ("u128", "write_u128", "read_u128"), ]; -/// Check if a type is a direct primitive numeric type (not wrapped in Option, Vec, etc.) -pub(super) fn is_direct_primitive_numeric_type(ty: &Type) -> bool { +/// Check if a type is a direct primitive type (numeric or String, not wrapped in Option, Vec, etc.) +pub(super) fn is_direct_primitive_type(ty: &Type) -> bool { if let Type::Path(type_path) = ty { if let Some(seg) = type_path.path.segments.last() { // Check if it's a simple type path without generics if matches!(seg.arguments, PathArguments::None) { let type_name = seg.ident.to_string(); + // Check for String type + if type_name == "String" { + return true; + } + // Check for numeric primitive types return PRIMITIVE_IO_METHODS .iter() .any(|(name, _, _)| *name == type_name.as_str()); diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 0ebac74fdc..3960860103 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -19,8 +19,7 @@ use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, get_field_accessor, get_field_name, get_filtered_source_fields_iter, get_primitive_writer_method, get_struct_name, - get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_numeric_type, - should_skip_type_info_for_field, FieldRefMode, StructField, + get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_type, FieldRefMode, StructField, }; use crate::util::SourceField; use fory_core::types::TypeId; @@ -242,30 +241,39 @@ fn gen_write_field_impl( } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info quote! { <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, true, false)?; } } _ => { - let skip_type_info = should_skip_type_info_for_field(ty); let type_id = get_type_id_by_type_ast(ty); - // Check if this is a direct primitive numeric type that can use direct writer calls - if is_direct_primitive_numeric_type(ty) { + // Check if this is a direct primitive type that can use direct writer calls + // Only apply when ref_mode is None (no ref tracking needed) + if ref_mode == FieldRefMode::None && is_direct_primitive_type(ty) { let type_name = extract_type_name(ty); - let writer_method = get_primitive_writer_method(&type_name); - let writer_ident = syn::Ident::new(writer_method, proc_macro2::Span::call_site()); - // For primitives: - // - use_self=true: #value_ts is `self.field`, which is T (copy happens automatically) - // - use_self=false: #value_ts is `field` from pattern match on &self, which is &T - let value_expr = if use_self { - quote! { #value_ts } + if type_name == "String" { + // String: call fory_write_data directly + quote! { + <#ty as fory_core::Serializer>::fory_write_data(&#value_ts, context)?; + } } else { - quote! { *#value_ts } - }; - quote! { - context.writer.#writer_ident(#value_expr); + // Numeric primitives: use direct buffer methods + let writer_method = get_primitive_writer_method(&type_name); + let writer_ident = + syn::Ident::new(writer_method, proc_macro2::Span::call_site()); + // For primitives: + // - use_self=true: #value_ts is `self.field`, which is T (copy happens automatically) + // - use_self=false: #value_ts is `field` from pattern match on &self, which is &T + let value_expr = if use_self { + quote! { #value_ts } + } else { + quote! { *#value_ts } + }; + quote! { + context.writer.#writer_ident(#value_expr); + } } } else if type_id == TypeId::LIST as u32 || type_id == TypeId::SET as u32 @@ -282,24 +290,21 @@ fn gen_write_field_impl( } } } else { - // Known types (primitives, strings, collections) - skip type info at compile time - // For custom types that we can't determine at compile time (like enums), - // we need to check at runtime whether to skip type info - if skip_type_info { - if ref_mode == FieldRefMode::None { - quote! { - <#ty as fory_core::Serializer>::fory_write_data(&#value_ts, context)?; - } + // Custom types (struct/enum/ext) - always need mode-dependent type info logic + // Determine write_type_info based on mode: + // - compatible=true: use need_to_write_type_for_field (struct types need type info) + // - compatible=false: use fory_is_polymorphic + // This applies regardless of ref_mode because Java always writes type info + // for struct-type fields in compatible mode, even for non-nullable fields. + quote! { + let write_type_info = if context.is_compatible() { + fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ) } else { - quote! { - <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, false, false)?; - } - } - } else { - quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, need_type_info, false)?; - } + <#ty as fory_core::Serializer>::fory_is_polymorphic() + }; + <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, write_type_info, false)?; } } } diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index 41f641ff19..15a4ccde65 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -251,7 +251,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default(); +//! let mut fory = Fory::default().track_ref(true); //! fory.register::(2000); //! //! let parent = Rc::new(RefCell::new(Node { @@ -293,7 +293,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default(); +//! let mut fory = Fory::default().track_ref(true); //! fory.register::(6000); //! //! let parent = Arc::new(Mutex::new(Node { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index b341c281b5..f62a894dce 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -52,6 +52,7 @@ struct Item { } #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct SimpleStruct { // field_order != sorted_order f1: HashMap, @@ -527,6 +528,7 @@ fn test_integer() { // - Java Integer fields (with nullable=false) -> Rust i32 (no ref flag) // All fields use i32 because Java xlang mode defaults to nullable=false for all non-primitives #[derive(ForyObject, Debug, PartialEq)] + #[fory(debug)] struct Item2 { f1: i32, f2: i32, @@ -620,6 +622,7 @@ impl ForyDefault for MyExt { } } #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct MyWrapper { color: Color, my_struct: MyStruct, @@ -1264,6 +1267,7 @@ fn test_enum_schema_evolution_compatible_reverse() { /// - Nullable fields (first half - boxed numeric types): Integer, Long, Float /// - Nullable fields (second half - @ForyField): Double, Boolean, String, List, Set, Map #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct NullableComprehensiveSchemaConsistent { // Base non-nullable primitive fields byte_field: i8, @@ -1310,6 +1314,7 @@ struct NullableComprehensiveSchemaConsistent { /// /// This tests that compatible mode properly handles schema differences across languages. #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct NullableComprehensiveCompatible { // Group 1: Nullable in Rust, Non-nullable in Java // Primitive fields @@ -1638,3 +1643,139 @@ fn test_union_xlang() { fory.serialize_to(&mut buf, &struct2).unwrap(); fs::write(&data_file_path, buf).unwrap(); } + +// ============================================================================ +// Reference Tracking Tests - Cross-language shared reference tests +// ============================================================================ + +use std::rc::Rc; + +/// Inner struct for reference tracking test (SCHEMA_CONSISTENT mode) +/// Matches Java RefInnerSchemaConsistent with type ID 501 +#[derive(ForyObject, Debug, PartialEq, Clone)] +struct RefInnerSchemaConsistent { + id: i32, + name: String, +} + +/// Outer struct for reference tracking test (SCHEMA_CONSISTENT mode) +/// Contains two fields that both point to the same inner object. +/// Matches Java RefOuterSchemaConsistent with type ID 502 +/// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking +#[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] +struct RefOuterSchemaConsistent { + inner1: Option>, + inner2: Option>, +} + +/// Inner struct for reference tracking test (COMPATIBLE mode) +/// Matches Java RefInnerCompatible with type ID 503 +#[derive(ForyObject, Debug, PartialEq, Clone)] +#[fory(debug)] +struct RefInnerCompatible { + id: i32, + name: String, +} + +/// Outer struct for reference tracking test (COMPATIBLE mode) +/// Contains two fields that both point to the same inner object. +/// Matches Java RefOuterCompatible with type ID 504 +/// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking +#[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] +struct RefOuterCompatible { + inner1: Option>, + inner2: Option>, +} + +/// Test cross-language reference tracking in SCHEMA_CONSISTENT mode (compatible=false). +/// +/// This test verifies that when Java serializes an object where two fields point to +/// the same instance, Rust can properly deserialize it and both fields will contain +/// equal values. When re-serializing, the reference relationship should be preserved. +#[test] +#[ignore] +fn test_ref_schema_consistent() { + let data_file_path = get_data_file(); + let bytes = fs::read(&data_file_path).unwrap(); + + let mut fory = Fory::default() + .compatible(false) + .xlang(true) + .track_ref(true); + fory.register::(501).unwrap(); + fory.register::(502).unwrap(); + + let outer: RefOuterSchemaConsistent = fory.deserialize(&bytes).unwrap(); + + // Both inner1 and inner2 should have values + assert!(outer.inner1.is_some(), "inner1 should not be None"); + assert!(outer.inner2.is_some(), "inner2 should not be None"); + + // Both should have the same values (they reference the same object in Java) + let inner1 = outer.inner1.as_ref().unwrap(); + let inner2 = outer.inner2.as_ref().unwrap(); + assert_eq!(inner1.id, 42); + assert_eq!(inner1.name, "shared_inner"); + // Compare the values (Rc contents) + assert_eq!( + inner1.as_ref(), + inner2.as_ref(), + "inner1 and inner2 should have equal values" + ); + + // With Rc, after deserialization with ref tracking, both fields should point to the same Rc + assert!( + Rc::ptr_eq(inner1, inner2), + "inner1 and inner2 should be the same Rc (reference identity)" + ); + + // Re-serialize and write back + let new_bytes = fory.serialize(&outer).unwrap(); + fs::write(&data_file_path, new_bytes).unwrap(); +} + +/// Test cross-language reference tracking in COMPATIBLE mode (compatible=true). +/// +/// This test verifies reference tracking works correctly with schema evolution support. +/// The inner object is shared between two fields, and this relationship should be +/// preserved through serialization/deserialization. +#[test] +#[ignore] +fn test_ref_compatible() { + let data_file_path = get_data_file(); + let bytes = fs::read(&data_file_path).unwrap(); + + let mut fory = Fory::default().compatible(true).xlang(true).track_ref(true); + fory.register::(503).unwrap(); + fory.register::(504).unwrap(); + + let outer: RefOuterCompatible = fory.deserialize(&bytes).unwrap(); + + // Both inner1 and inner2 should have values + assert!(outer.inner1.is_some(), "inner1 should not be None"); + assert!(outer.inner2.is_some(), "inner2 should not be None"); + + // Both should have the same values (they reference the same object in Java) + let inner1 = outer.inner1.as_ref().unwrap(); + let inner2 = outer.inner2.as_ref().unwrap(); + assert_eq!(inner1.id, 99); + assert_eq!(inner1.name, "compatible_shared"); + // Compare the values (Rc contents) + assert_eq!( + inner1.as_ref(), + inner2.as_ref(), + "inner1 and inner2 should have equal values" + ); + + // With Rc, after deserialization with ref tracking, both fields should point to the same Rc + assert!( + Rc::ptr_eq(inner1, inner2), + "inner1 and inner2 should be the same Rc (reference identity)" + ); + + // Re-serialize and write back + let new_bytes = fory.serialize(&outer).unwrap(); + fs::write(&data_file_path, new_bytes).unwrap(); +} diff --git a/rust/tests/tests/test_rc_arc.rs b/rust/tests/tests/test_rc_arc.rs index 135d14f2bd..55762ce280 100644 --- a/rust/tests/tests/test_rc_arc.rs +++ b/rust/tests/tests/test_rc_arc.rs @@ -18,10 +18,17 @@ //! Tests for Rc and Arc serialization support in Fory use fory_core::fory::Fory; +use fory_derive::ForyObject; use std::collections::HashMap; use std::rc::Rc; use std::sync::Arc; +/// A simple struct for testing nested Rc/Arc serialization +#[derive(ForyObject, Debug, Clone, PartialEq, Default)] +struct NestedData { + value: String, +} + #[test] fn test_rc_string_serialization() { let fory = Fory::default(); @@ -166,16 +173,19 @@ fn test_mixed_rc_arc_serialization() { #[test] fn test_nested_rc_arc() { - let fory = Fory::default(); + let mut fory = Fory::default(); + fory.register::(100).unwrap(); - // Test Rc containing Arc - let inner_data = Arc::new(String::from("nested")); + // Test Rc containing Arc with allowed struct type + let inner_data = Arc::new(NestedData { + value: String::from("nested"), + }); let outer_data = Rc::new(inner_data.clone()); let serialized = fory.serialize(&outer_data).unwrap(); - let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); + let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); - assert_eq!(**outer_data, **deserialized); + assert_eq!(outer_data.value, deserialized.value); } #[test] diff --git a/rust/tests/tests/test_weak.rs b/rust/tests/tests/test_weak.rs index 1a5490480a..ce3f53ea19 100644 --- a/rust/tests/tests/test_weak.rs +++ b/rust/tests/tests/test_weak.rs @@ -25,7 +25,7 @@ use std::sync::Mutex; #[test] fn test_rc_weak_null_serialization() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let weak: RcWeak = RcWeak::new(); @@ -37,7 +37,7 @@ fn test_rc_weak_null_serialization() { #[test] fn test_arc_weak_null_serialization() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let weak: ArcWeak = ArcWeak::new(); @@ -49,7 +49,7 @@ fn test_arc_weak_null_serialization() { #[test] fn test_rc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let weak = { let rc = Rc::new(42i32); @@ -69,7 +69,7 @@ fn test_rc_weak_dead_pointer_serializes_as_null() { #[test] fn test_arc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let weak = { let arc = Arc::new(String::from("test")); @@ -89,7 +89,7 @@ fn test_arc_weak_dead_pointer_serializes_as_null() { #[test] fn test_rc_weak_in_vec_circular_reference() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let data1 = Rc::new(42i32); let data2 = Rc::new(100i32); @@ -107,7 +107,7 @@ fn test_rc_weak_in_vec_circular_reference() { #[test] fn test_arc_weak_in_vec_circular_reference() { - let fory = Fory::default(); + let fory = Fory::default().track_ref(true); let data1 = Arc::new(String::from("hello")); let data2 = Arc::new(String::from("world")); @@ -133,7 +133,7 @@ fn test_rc_weak_field_in_struct() { weak_ref: RcWeak, } - let mut fory = Fory::default(); + let mut fory = Fory::default().track_ref(true); fory.register::(1000).unwrap(); let data = Rc::new(42i32); @@ -160,7 +160,7 @@ struct Node { #[test] fn test_node_circular_reference_with_parent_children() { // Register the Node type with Fory - let mut fory = Fory::default(); + let mut fory = Fory::default().track_ref(true); fory.register::(2000).unwrap(); // Create parent @@ -218,7 +218,7 @@ fn test_arc_mutex_circular_reference() { children: Vec>>, } - let mut fory = Fory::default(); + let mut fory = Fory::default().track_ref(true); fory.register::(6000).unwrap(); let parent = Arc::new(Mutex::new(Node {