feat(java/python/rust/go/c++): xlang fields reference and typeinfo alignment

.github/workflows/ci.yml

@@ -326,7 +326,7 @@ jobs:
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
           cd fory-core
-          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.RustXlangTest
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.RustXlangTest
 
   cpp:
     name: C++ CI
@@ -412,7 +412,7 @@ jobs:
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
           cd fory-core
-          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CPPXlangTest
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.CPPXlangTest
 
   cpp_examples:
     name: C++ Examples
@@ -533,8 +533,10 @@ jobs:
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
           cd fory-core
-          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.PythonXlangTest
-          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CrossLanguageTest
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PythonXlangTest
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PyCrossLanguageTest
+          cd ../fory-format
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.CrossLanguageTest
 
   go:
     name: Golang CI
@@ -599,7 +601,7 @@ jobs:
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
           cd fory-core
-          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.GoXlangTest
+          mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest
 
   lint:
     name: Code Style Check

AGENTS.md

@@ -85,7 +85,7 @@ Run C++ xlang tests:
 cd java
 mvn -T16 install -DskipTests
 cd fory-core
-FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.CPPXlangTest
+FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.CPPXlangTest
 ```
 
 ### Python Development
@@ -125,9 +125,9 @@ cd java
 mvn -T16 install -DskipTests
 cd fory-core
 # disable fory cython for faster debugging
-FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest
+FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest
 # enable fory cython
-FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest
+FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest
 ```
 
 ### Golang Development
@@ -159,7 +159,7 @@ Run Go xlang tests:
 cd java
 mvn -T16 install -DskipTests
 cd fory-core
-FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.GoXlangTest
+FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.GoXlangTest
 ```
 
 ### Rust Development
@@ -215,7 +215,7 @@ Run Rust xlang tests:
 cd java
 mvn -T16 install -DskipTests
 cd fory-core
-FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.RustXlangTest
+FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.RustXlangTest
 ```
 
 ### JavaScript/TypeScript Development

cpp/fory/meta/field.h

@@ -50,6 +50,12 @@ struct not_null {};
 /// tracking).
 struct ref {};
 
+/// Tag to mark a polymorphic shared_ptr/unique_ptr field as monomorphic.
+/// Use this when the field type has virtual methods but you know the actual
+/// runtime type will always be exactly T (not a derived type).
+/// This avoids dynamic type dispatch overhead during serialization.
+struct monomorphic {};
+
 namespace detail {
 
 // ============================================================================
@@ -96,10 +102,12 @@ inline constexpr bool has_option_v = (std::is_same_v<Tag, Options> || ...);
 // ============================================================================
 
 /// Compile-time field tag metadata entry
-template <int16_t Id, bool Nullable, bool Ref> struct FieldTagEntry {
+template <int16_t Id, bool Nullable, bool Ref, bool Monomorphic = false>
+struct FieldTagEntry {
   static constexpr int16_t id = Id;
   static constexpr bool is_nullable = Nullable;
   static constexpr bool track_ref = Ref;
+  static constexpr bool is_monomorphic = Monomorphic;
 };
 
 /// Default: no field tags defined for type T
@@ -163,6 +171,11 @@ template <typename T, int16_t Id, typename... Options> class field {
                 "fory::ref is only valid for shared_ptr "
                 "(reference tracking requires shared ownership).");
 
+  // Validate: monomorphic only for smart pointers
+  static_assert(!detail::has_option_v<monomorphic, Options...> ||
+                    detail::is_smart_ptr_v<T>,
+                "fory::monomorphic is only valid for shared_ptr/unique_ptr.");
+
   // Validate: no options for optional (inherently nullable)
   static_assert(!detail::is_optional_v<T> || sizeof...(Options) == 0,
                 "std::optional<T> is inherently nullable. No options allowed.");
@@ -189,6 +202,13 @@ template <typename T, int16_t Id, typename... Options> class field {
   static constexpr bool track_ref =
       detail::is_shared_ptr_v<T> && detail::has_option_v<ref, Options...>;
 
+  /// Monomorphic serialization is enabled if:
+  /// - It's std::shared_ptr or std::unique_ptr with fory::monomorphic option
+  /// - When true, the serializer will not use dynamic type dispatch
+  static constexpr bool is_monomorphic =
+      detail::is_smart_ptr_v<T> &&
+      detail::has_option_v<monomorphic, Options...>;
+
   T value{};
 
   // Default constructor
@@ -309,6 +329,19 @@ struct field_track_ref<field<T, Id, Options...>> {
 template <typename T>
 inline constexpr bool field_track_ref_v = field_track_ref<T>::value;
 
+/// Get is_monomorphic from field type
+template <typename T> struct field_is_monomorphic {
+  static constexpr bool value = false;
+};
+
+template <typename T, int16_t Id, typename... Options>
+struct field_is_monomorphic<field<T, Id, Options...>> {
+  static constexpr bool value = field<T, Id, Options...>::is_monomorphic;
+};
+
+template <typename T>
+inline constexpr bool field_is_monomorphic_v = field_is_monomorphic<T>::value;
+
 // ============================================================================
 // FORY_FIELD_TAGS Macro Support
 // ============================================================================
@@ -317,7 +350,7 @@ namespace detail {
 
 // Helper to parse field tag entry from macro arguments
 // Supports: (field, id), (field, id, nullable), (field, id, ref),
-//           (field, id, nullable, ref)
+//           (field, id, nullable, ref), (field, id, monomorphic), etc.
 template <typename FieldType, int16_t Id, typename... Options>
 struct ParseFieldTagEntry {
   static constexpr bool is_nullable =
@@ -327,6 +360,9 @@ struct ParseFieldTagEntry {
   static constexpr bool track_ref =
       is_shared_ptr_v<FieldType> && has_option_v<ref, Options...>;
 
+  static constexpr bool is_monomorphic =
+      is_smart_ptr_v<FieldType> && has_option_v<monomorphic, Options...>;
+
   // Compile-time validation
   static_assert(!has_option_v<nullable, Options...> ||
                     is_smart_ptr_v<FieldType>,
@@ -335,14 +371,19 @@ struct ParseFieldTagEntry {
   static_assert(!has_option_v<ref, Options...> || is_shared_ptr_v<FieldType>,
                 "fory::ref is only valid for shared_ptr");
 
-  using type = FieldTagEntry<Id, is_nullable, track_ref>;
+  static_assert(!has_option_v<monomorphic, Options...> ||
+                    is_smart_ptr_v<FieldType>,
+                "fory::monomorphic is only valid for shared_ptr/unique_ptr");
+
+  using type = FieldTagEntry<Id, is_nullable, track_ref, is_monomorphic>;
 };
 
 /// Get field tag entry by index from ForyFieldTagsImpl
 template <typename T, size_t Index, typename = void> struct GetFieldTagEntry {
   static constexpr int16_t id = -1;
   static constexpr bool is_nullable = false;
   static constexpr bool track_ref = false;
+  static constexpr bool is_monomorphic = false;
 };
 
 template <typename T, size_t Index>
@@ -355,6 +396,7 @@ struct GetFieldTagEntry<
   static constexpr int16_t id = Entry::id;
   static constexpr bool is_nullable = Entry::is_nullable;
   static constexpr bool track_ref = Entry::track_ref;
+  static constexpr bool is_monomorphic = Entry::is_monomorphic;
 };
 
 } // namespace detail
@@ -377,12 +419,14 @@ struct GetFieldTagEntry<
 #define FORY_FT_GET_OPT1_IMPL(f, i, o1, ...) o1
 #define FORY_FT_GET_OPT2(tuple) FORY_FT_GET_OPT2_IMPL tuple
 #define FORY_FT_GET_OPT2_IMPL(f, i, o1, o2, ...) o2
+#define FORY_FT_GET_OPT3(tuple) FORY_FT_GET_OPT3_IMPL tuple
+#define FORY_FT_GET_OPT3_IMPL(f, i, o1, o2, o3, ...) o3
 
-// Detect number of elements in tuple: 2, 3, or 4
+// Detect number of elements in tuple: 2, 3, 4, or 5
 #define FORY_FT_TUPLE_SIZE(tuple) FORY_FT_TUPLE_SIZE_IMPL tuple
 #define FORY_FT_TUPLE_SIZE_IMPL(...)                                           \
-  FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 4, 3, 2, 1, 0)
-#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, N, ...) N
+  FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 5, 4, 3, 2, 1, 0)
+#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, _5, N, ...) N
 
 // Create FieldTagEntry based on tuple size using indirect call pattern
 // This pattern ensures the concatenated macro name is properly rescanned
@@ -408,6 +452,12 @@ struct GetFieldTagEntry<
       decltype(std::declval<Type>().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple),  \
       ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple)>::type
 
+#define FORY_FT_MAKE_ENTRY_5(Type, tuple)                                      \
+  typename ::fory::detail::ParseFieldTagEntry<                                 \
+      decltype(std::declval<Type>().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple),  \
+      ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple),        \
+      ::fory::FORY_FT_GET_OPT3(tuple)>::type
+
 // Main macro: FORY_FIELD_TAGS(Type, (field1, id1), (field2, id2, nullable),...)
 // Note: Uses fory::detail:: instead of ::fory::detail:: for GCC compatibility
 #define FORY_FIELD_TAGS(Type, ...)                                             \

cpp/fory/serialization/collection_serializer.h

@@ -269,8 +269,12 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx,
   if (is_same_type) {
     bitmap |= COLL_IS_SAME_TYPE;
   }
+  // Only set TRACKING_REF if element is shared ref AND global ref tracking is
+  // enabled
   if constexpr (elem_is_shared_ref) {
-    bitmap |= COLL_TRACKING_REF;
+    if (ctx.track_ref()) {
+      bitmap |= COLL_TRACKING_REF;
+    }
   }
 
   // Write header
@@ -287,52 +291,54 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx,
     }
   }
 
+  // Determine if we're actually tracking refs for this collection
+  const bool tracking_refs = (bitmap & COLL_TRACKING_REF) != 0;
+
   // Write elements
   if (is_same_type) {
     // All elements have same type - type info written once above
-    if (!has_null) {
-      if constexpr (elem_is_shared_ref) {
-        // Write with ref flag, without type
-        for (const auto &elem : coll) {
-          Serializer<T>::write(elem, ctx, RefMode::NullOnly, false,
-                               has_generics);
-        }
-      } else {
-        // Write data directly
-        for (const auto &elem : coll) {
-          if constexpr (is_nullable_v<T>) {
-            using Inner = nullable_element_t<T>;
-            Serializer<Inner>::write_data(deref_nullable(elem), ctx);
+    if (tracking_refs) {
+      // Track refs - write ref flag per element per xlang spec
+      for (const auto &elem : coll) {
+        Serializer<T>::write(elem, ctx, RefMode::Tracking, false, has_generics);
+      }
+    } else if (!has_null) {
+      // No nulls, no ref tracking - write data directly without null flag
+      for (const auto &elem : coll) {
+        if constexpr (is_nullable_v<T>) {
+          using Inner = nullable_element_t<T>;
+          Serializer<Inner>::write_data(deref_nullable(elem), ctx);
+        } else if constexpr (elem_is_shared_ref) {
+          // For shared_ptr, use write_data which handles polymorphic types
+          Serializer<T>::write_data(elem, ctx);
+        } else {
+          if constexpr (is_generic_type_v<T>) {
+            Serializer<T>::write_data_generic(elem, ctx, has_generics);
           } else {
-            if constexpr (is_generic_type_v<T>) {
-              Serializer<T>::write_data_generic(elem, ctx, has_generics);
-            } else {
-              Serializer<T>::write_data(elem, ctx);
-            }
+            Serializer<T>::write_data(elem, ctx);
           }
         }
       }
     } else {
-      // Has null elements - write with ref flag for null tracking
+      // Has null elements - write with null flag
       for (const auto &elem : coll) {
         Serializer<T>::write(elem, ctx, RefMode::NullOnly, false, has_generics);
       }
     }
   } else {
     // Heterogeneous types - write type info per element
-    if (!has_null) {
-      if constexpr (elem_is_shared_ref) {
-        for (const auto &elem : coll) {
-          Serializer<T>::write(elem, ctx, RefMode::NullOnly, true,
-                               has_generics);
-        }
-      } else {
-        for (const auto &elem : coll) {
-          Serializer<T>::write(elem, ctx, RefMode::None, true, has_generics);
-        }
+    if (tracking_refs) {
+      // Track refs - write ref flag + type info per element
+      for (const auto &elem : coll) {
+        Serializer<T>::write(elem, ctx, RefMode::Tracking, true, has_generics);
+      }
+    } else if (!has_null) {
+      // No nulls - write without null flag (RefMode::None)
+      for (const auto &elem : coll) {
+        Serializer<T>::write(elem, ctx, RefMode::None, true, has_generics);
       }
     } else {
-      // Has null elements
+      // Has null elements - write with null flag (RefMode::NullOnly)
       for (const auto &elem : coll) {
         Serializer<T>::write(elem, ctx, RefMode::NullOnly, true, has_generics);
       }
@@ -416,7 +422,8 @@ inline Container read_collection_data_slow(ReadContext &ctx, uint32_t length) {
           return result;
         }
         if constexpr (elem_is_polymorphic) {
-          auto elem = Serializer<T>::read_with_type_info(ctx, RefMode::NullOnly,
+          // Use RefMode::Tracking to read ref flag per element
+          auto elem = Serializer<T>::read_with_type_info(ctx, RefMode::Tracking,
                                                          *elem_type_info);
           collection_insert(result, std::move(elem));
         } else {
@@ -1609,8 +1616,12 @@ struct Serializer<std::forward_list<T, Alloc>> {
       if (is_same_type) {
         bitmap |= COLL_IS_SAME_TYPE;
       }
+      // Only set TRACKING_REF if element is shared ref AND global ref tracking
+      // is enabled
       if constexpr (elem_is_shared_ref) {
-        bitmap |= COLL_TRACKING_REF;
+        if (ctx.track_ref()) {
+          bitmap |= COLL_TRACKING_REF;
+        }
       }
 
       // Write header

cpp/fory/serialization/fory.h

@@ -623,8 +623,11 @@ class Fory : public BaseFory {
       buffer.WriteInt32(-1); // Placeholder for meta offset (fixed 4 bytes)
     }
 
-    // Top-level serialization: YES ref flags, yes type info
-    Serializer<T>::write(obj, *write_ctx_, RefMode::NullOnly, true);
+    // Top-level serialization: use Tracking if ref tracking is enabled,
+    // otherwise NullOnly for nullable handling
+    const RefMode top_level_ref_mode =
+        write_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly;
+    Serializer<T>::write(obj, *write_ctx_, top_level_ref_mode, true);
     // Check for errors at serialization boundary
     if (FORY_PREDICT_FALSE(write_ctx_->has_error())) {
       return Unexpected(write_ctx_->take_error());
@@ -654,8 +657,11 @@ class Fory : public BaseFory {
       }
     }
 
-    // Top-level deserialization: YES ref flags, yes type info
-    T result = Serializer<T>::read(*read_ctx_, RefMode::NullOnly, true);
+    // Top-level deserialization: use Tracking if ref tracking is enabled,
+    // otherwise NullOnly for nullable handling
+    const RefMode top_level_ref_mode =
+        read_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly;
+    T result = Serializer<T>::read(*read_ctx_, top_level_ref_mode, true);
     // Check for errors at deserialization boundary
     if (FORY_PREDICT_FALSE(read_ctx_->has_error())) {
       return Unexpected(read_ctx_->take_error());