From 9f639505ef426d2d1f69184b0121bbbbb82dbb59 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 1 Jan 2026 17:50:57 +0800 Subject: [PATCH 01/35] add xlang ref tests and fix java morphic for xlang --- go/fory/ref_resolver.go | 7 + go/fory/struct.go | 11 ++ go/fory/tests/xlang/xlang_test_main.go | 160 ++++++++++++++++++ .../src/main/java/org/apache/fory/Fory.java | 27 +++ .../org/apache/fory/annotation/ForyField.java | 33 ++++ .../fory/serializer/ObjectSerializer.java | 7 + .../fory/serializer/SerializationBinding.java | 49 +++++- .../java/org/apache/fory/type/Descriptor.java | 12 ++ .../java/org/apache/fory/GoXlangTest.java | 19 +++ .../java/org/apache/fory/XlangTestBase.java | 159 +++++++++++++++++ 10 files changed, 480 insertions(+), 4 deletions(-) diff --git a/go/fory/ref_resolver.go b/go/fory/ref_resolver.go index 19ed5705f4..ba5f108a2b 100644 --- a/go/fory/ref_resolver.go +++ b/go/fory/ref_resolver.go @@ -203,16 +203,23 @@ func (r *RefResolver) PreserveRefId() (int32, error) { func (r *RefResolver) TryPreserveRefId(buffer *ByteBuffer) (int32, error) { var ctxErr Error + pos := buffer.ReaderIndex() headFlag := buffer.ReadInt8(&ctxErr) if ctxErr.HasError() { return 0, ctxErr } + if DebugOutputEnabled() { + fmt.Printf("[Go][fory-debug] TryPreserveRefId at position %d: headFlag=%d\n", pos, headFlag) + } if headFlag == RefFlag { // read ref id and get object from ref resolver refId := buffer.ReadVaruint32(&ctxErr) if ctxErr.HasError() { return 0, ctxErr } + if DebugOutputEnabled() { + fmt.Printf("[Go][fory-debug] TryPreserveRefId: REF_FLAG, refId=%d\n", refId) + } r.readObject = r.GetReadObject(int32(refId)) } else { r.readObject = reflect.Value{} diff --git a/go/fory/struct.go b/go/fory/struct.go index e7e655fc7e..13bada0f9a 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -642,7 +642,12 @@ func (s *structSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value // In compatible mode with meta share, struct hash is not written if !ctx.Compatible() { err := ctx.Err() + pos := buf.ReaderIndex() structHash := buf.ReadInt32(err) + if DebugOutputEnabled() { + fmt.Printf("[Go][fory-debug] Reading struct hash for %s at position %d: read=%d, expected=%d\n", + s.type_.String(), pos, structHash, s.structHash) + } if structHash != s.structHash { ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) return @@ -1769,6 +1774,12 @@ func (s *structSerializer) computeHash() int32 { typeId = UNKNOWN } } + // For user-defined types (struct, ext types), use UNKNOWN in fingerprint + // This matches Java's behavior where user-defined types return UNKNOWN + // to ensure consistent fingerprint computation across languages + if isUserDefinedType(int16(typeId)) { + typeId = UNKNOWN + } // For fixed-size arrays with primitive elements, use primitive array type IDs if field.Type.Kind() == reflect.Array { elemKind := field.Type.Elem().Kind() diff --git a/go/fory/tests/xlang/xlang_test_main.go b/go/fory/tests/xlang/xlang_test_main.go index 5d36efeadb..b6d195bc1e 100644 --- a/go/fory/tests/xlang/xlang_test_main.go +++ b/go/fory/tests/xlang/xlang_test_main.go @@ -1862,6 +1862,162 @@ func testNullableFieldCompatibleNull() { writeFile(dataFile, serialized) } +// ============================================================================ +// Reference Tracking Test Types +// ============================================================================ + +// RefInnerSchemaConsistent - Inner struct for ref tracking tests in SCHEMA_CONSISTENT mode +// Matches Java's RefInnerSchemaConsistent (type id 501) +type RefInnerSchemaConsistent struct { + Id int32 + Name string +} + +// RefOuterSchemaConsistent - Outer struct for ref tracking tests in SCHEMA_CONSISTENT mode +// Both fields point to the same RefInnerSchemaConsistent instance +// Matches Java's RefOuterSchemaConsistent (type id 502) +type RefOuterSchemaConsistent struct { + Inner1 *RefInnerSchemaConsistent `fory:"ref,nullable"` + Inner2 *RefInnerSchemaConsistent `fory:"ref,nullable"` +} + +// RefInnerCompatible - Inner struct for ref tracking tests in COMPATIBLE mode +// Matches Java's RefInnerCompatible (type id 503) +type RefInnerCompatible struct { + Id int32 + Name string +} + +// RefOuterCompatible - Outer struct for ref tracking tests in COMPATIBLE mode +// Both fields point to the same RefInnerCompatible instance +// Matches Java's RefOuterCompatible (type id 504) +type RefOuterCompatible struct { + Inner1 *RefInnerCompatible `fory:"ref,nullable"` + Inner2 *RefInnerCompatible `fory:"ref,nullable"` +} + +func getRefOuterSchemaConsistent(obj interface{}) RefOuterSchemaConsistent { + switch v := obj.(type) { + case RefOuterSchemaConsistent: + return v + case *RefOuterSchemaConsistent: + return *v + default: + panic(fmt.Sprintf("expected RefOuterSchemaConsistent, got %T", obj)) + } +} + +func getRefOuterCompatible(obj interface{}) RefOuterCompatible { + switch v := obj.(type) { + case RefOuterCompatible: + return v + case *RefOuterCompatible: + return *v + default: + panic(fmt.Sprintf("expected RefOuterCompatible, got %T", obj)) + } +} + +// ============================================================================ +// Reference Tracking Tests +// ============================================================================ + +func testRefSchemaConsistent() { + dataFile := getDataFile() + data := readFile(dataFile) + + f := fory.New(fory.WithXlang(true), fory.WithCompatible(false), fory.WithRefTracking(true)) + f.Register(RefInnerSchemaConsistent{}, 501) + f.Register(RefOuterSchemaConsistent{}, 502) + + buf := fory.NewByteBuffer(data) + var obj interface{} + err := f.DeserializeWithCallbackBuffers(buf, &obj, nil) + if err != nil { + panic(fmt.Sprintf("Failed to deserialize: %v", err)) + } + + result := getRefOuterSchemaConsistent(obj) + + // Verify both fields have the expected values + if result.Inner1 == nil { + panic("Inner1 is nil") + } + if result.Inner2 == nil { + panic("Inner2 is nil") + } + assertEqual(int32(42), result.Inner1.Id, "Inner1.Id") + assertEqual("shared_inner", result.Inner1.Name, "Inner1.Name") + assertEqual(int32(42), result.Inner2.Id, "Inner2.Id") + assertEqual("shared_inner", result.Inner2.Name, "Inner2.Name") + + // Verify reference identity is preserved + if result.Inner1 != result.Inner2 { + panic("Reference tracking failed: Inner1 and Inner2 should point to the same object") + } + fmt.Println("Reference identity verified: Inner1 == Inner2") + + // Re-serialize with same shared reference + outer := &RefOuterSchemaConsistent{ + Inner1: result.Inner1, + Inner2: result.Inner1, // Use same reference + } + serialized, err := f.Serialize(outer) + if err != nil { + panic(fmt.Sprintf("Failed to serialize: %v", err)) + } + + writeFile(dataFile, serialized) +} + +func testRefCompatible() { + dataFile := getDataFile() + data := readFile(dataFile) + + f := fory.New(fory.WithXlang(true), fory.WithCompatible(true), fory.WithRefTracking(true)) + f.Register(RefInnerCompatible{}, 503) + f.Register(RefOuterCompatible{}, 504) + + buf := fory.NewByteBuffer(data) + var obj interface{} + err := f.DeserializeWithCallbackBuffers(buf, &obj, nil) + if err != nil { + panic(fmt.Sprintf("Failed to deserialize: %v", err)) + } + + result := getRefOuterCompatible(obj) + + // Verify both fields have the expected values + if result.Inner1 == nil { + panic("Inner1 is nil") + } + if result.Inner2 == nil { + panic("Inner2 is nil") + } + assertEqual(int32(99), result.Inner1.Id, "Inner1.Id") + assertEqual("compatible_shared", result.Inner1.Name, "Inner1.Name") + assertEqual(int32(99), result.Inner2.Id, "Inner2.Id") + assertEqual("compatible_shared", result.Inner2.Name, "Inner2.Name") + + // Verify reference identity is preserved + if result.Inner1 != result.Inner2 { + panic("Reference tracking failed: Inner1 and Inner2 should point to the same object") + } + fmt.Println("Reference identity verified: Inner1 == Inner2") + + // Re-serialize with same shared reference + outer := &RefOuterCompatible{ + Inner1: result.Inner1, + Inner2: result.Inner1, // Use same reference + } + serialized, err := f.Serialize(outer) + if err != nil { + panic(fmt.Sprintf("Failed to serialize: %v", err)) + } + + writeFile(dataFile, serialized) +} + // ============================================================================ // Main // ============================================================================ @@ -1957,6 +2113,10 @@ func main() { testNullableFieldCompatibleNotNull() case "test_nullable_field_compatible_null": testNullableFieldCompatibleNull() + case "test_ref_schema_consistent": + testRefSchemaConsistent() + case "test_ref_compatible": + testRefCompatible() default: panic(fmt.Sprintf("Unknown test case: %s", *caseName)) } diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 186d1c342e..de8cf0cfea 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -543,15 +543,42 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { } public void xwriteRef(MemoryBuffer buffer, Object obj) { + int posBeforeRef = buffer.writerIndex(); if (!refResolver.writeRefOrNull(buffer, obj)) { + int posAfterRef = buffer.writerIndex(); ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); + int posAfterTypeInfo = buffer.writerIndex(); + if (config.isForyDebugOutputEnabled()) { + LOG.info( + "[Java][fory-debug] xwriteRef(root) for {} at pos {}: refFlag pos {}, typeInfo pos {}, calling xwriteData", + obj.getClass().getSimpleName(), + posBeforeRef, + posAfterRef, + posAfterTypeInfo); + } xwriteData(buffer, classInfo, obj); } } public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { + if (config.isForyDebugOutputEnabled()) { + LOG.info( + "[Java][fory-debug] xwriteRef(with-serializer) for {} at pos {}, needToWriteRef={}", + obj != null ? obj.getClass().getSimpleName() : "null", + buffer.writerIndex(), + serializer.needToWriteRef()); + } if (serializer.needToWriteRef()) { + int posBeforeRef = buffer.writerIndex(); if (!refResolver.writeRefOrNull(buffer, obj)) { + int posAfterRef = buffer.writerIndex(); + if (config.isForyDebugOutputEnabled()) { + LOG.info( + "[Java][fory-debug] xwriteRef(with-serializer) for {} at pos {}: refFlag written, pos now {}, calling xwrite", + obj.getClass().getSimpleName(), + posBeforeRef, + posAfterRef); + } depth++; serializer.xwrite(buffer, obj); depth--; diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java index 05ff0d7e5e..d27206dabb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java @@ -28,6 +28,25 @@ @Target({ElementType.FIELD, ElementType.METHOD}) public @interface ForyField { + /** Controls polymorphism behavior for struct fields in cross-language serialization. */ + enum Morphic { + /** + * Auto-detect based on declared type (default): + * + *
    + *
  • Interface/abstract class: treated as POLYMORPHIC (type info written) + *
  • Concrete class: treated as FINAL (no type info written) + *
+ */ + AUTO, + + /** Treat as final/sealed - no type info written, uses declared type's serializer directly. */ + FINAL, + + /** Treat as polymorphic - type info written to support subtypes at runtime. */ + POLYMORPHIC + } + /** * Field tag ID for schema evolution mode. * @@ -55,4 +74,18 @@ * defaults) */ boolean ref() default false; + + /** + * Controls polymorphism behavior for this field in cross-language serialization. + * + *
    + *
  • {@link Morphic#AUTO} (default): Interface/abstract types are polymorphic, concrete types + * are final + *
  • {@link Morphic#FINAL}: No type info written, uses declared type's serializer + *
  • {@link Morphic#POLYMORPHIC}: Type info written to support runtime subtypes + *
+ * + *

Default: AUTO (concrete struct types are final, interface/abstract are polymorphic) + */ + Morphic morphic() default Morphic.AUTO; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 43cd434ef1..179ed65674 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -149,6 +149,13 @@ public void write(MemoryBuffer buffer, T value) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; if (fory.checkClassVersion()) { + if (fory.getConfig().isForyDebugOutputEnabled()) { + LOG.info( + "[Java][fory-debug] Writing struct hash for {} at position {}: hash={}", + type.getSimpleName(), + buffer.writerIndex(), + classVersionHash); + } buffer.writeInt32(classVersionHash); } // write order: primitive,boxed,final,other,collection,map diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 3fb1c84179..5e6a609565 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -23,6 +23,8 @@ import static org.apache.fory.serializer.AbstractObjectSerializer.GenericTypeField; import org.apache.fory.Fory; +import org.apache.fory.logging.Logger; +import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassInfoHolder; @@ -30,6 +32,7 @@ import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; +import org.apache.fory.util.Utils; // This polymorphic interface has cost, do not expose it as a public class // If it's used in other packages in fory, duplicate it in those packages. @@ -318,6 +321,7 @@ public void writeContainerFieldValue( } static final class XlangSerializationBinding extends SerializationBinding { + private static final Logger LOG = LoggerFactory.getLogger(XlangSerializationBinding.class); private final XtypeResolver xtypeResolver; XlangSerializationBinding(Fory fory) { @@ -337,12 +341,27 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - fory.xwriteRef(buffer, obj); + // In COMPATIBLE mode (meta share enabled): write type info for schema evolution + // In SCHEMA_CONSISTENT mode: don't write type info, use serializer directly + if (fory.getConfig().isMetaShareEnabled()) { + fory.xwriteRef(buffer, obj); + } else { + // SCHEMA_CONSISTENT mode: resolve serializer and write without type info + ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder); + fory.xwriteRef(buffer, obj, classInfo.getSerializer()); + } } @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.xwriteRef(buffer, obj); + // In COMPATIBLE mode (meta share enabled): write type info for schema evolution + // In SCHEMA_CONSISTENT mode: don't write type info, use serializer directly + if (fory.getConfig().isMetaShareEnabled()) { + fory.xwriteRef(buffer, obj); + } else { + // SCHEMA_CONSISTENT mode: use provided classInfo's serializer + fory.xwriteRef(buffer, obj, classInfo.getSerializer()); + } } @Override @@ -358,13 +377,35 @@ public Object readRef(MemoryBuffer buffer, GenericTypeField field) { fory.getGenerics().popGenericType(); return o; } else { - return fory.xreadRef(buffer); + // In COMPATIBLE mode (meta share enabled): read type info for schema evolution + // In SCHEMA_CONSISTENT mode: don't read type info, use serializer directly + if (fory.getConfig().isMetaShareEnabled()) { + return fory.xreadRef(buffer); + } else { + // SCHEMA_CONSISTENT mode: resolve serializer and read without type info + ClassInfo classInfo = field.classInfoHolder.classInfo; + if (classInfo.getSerializer() == null) { + classInfo = xtypeResolver.getClassInfo(classInfo.getCls(), field.classInfoHolder); + } + return fory.xreadRef(buffer, classInfo.getSerializer()); + } } } @Override public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - return fory.xreadRef(buffer); + // In COMPATIBLE mode (meta share enabled): read type info for schema evolution + // In SCHEMA_CONSISTENT mode: don't read type info, use serializer directly + if (fory.getConfig().isMetaShareEnabled()) { + return fory.xreadRef(buffer); + } else { + // SCHEMA_CONSISTENT mode: resolve serializer and read without type info + ClassInfo classInfo = classInfoHolder.classInfo; + if (classInfo.getSerializer() == null) { + classInfo = xtypeResolver.getClassInfo(classInfo.getCls(), classInfoHolder); + } + return fory.xreadRef(buffer, classInfo.getSerializer()); + } } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java index bd9c6aa8cb..158c02968f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/Descriptor.java @@ -261,6 +261,18 @@ public ForyField getForyField() { return foryField; } + /** + * Returns the morphic setting for this field. + * + * @return the morphic setting from @ForyField annotation, or AUTO if not specified + */ + public ForyField.Morphic getMorphic() { + if (foryField != null) { + return foryField.morphic(); + } + return ForyField.Morphic.AUTO; + } + /** Try not use {@link TypeRef#getRawType()} since it's expensive. */ public Class getRawType() { Class type = this.type; diff --git a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java index 25b14831ec..976fd622f3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java @@ -437,4 +437,23 @@ public void testUnionXlang() throws java.io.IOException { // Skip: Go doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Go Union xlang support not implemented"); } + + @Override + @Test + public void testRefSchemaConsistent() throws java.io.IOException { + // Run the test to debug hash mismatch + super.testRefSchemaConsistent(); + } + + @Override + @Test + public void testRefCompatible() throws java.io.IOException { + // Skip: Go struct field reference tracking with nested struct types + // has compatibility issues with Java in COMPATIBLE mode. + // The issue is related to type info indexing when deserializing nested struct fields + // with ref tracking enabled. + // TODO: Investigate TypeInfo indexing for nested struct types with ref tracking. + throw new SkipException( + "Skipping: Go TypeInfo indexing issue for nested structs with ref tracking in COMPATIBLE mode"); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java index f0fcf9a9e7..f12fbcd682 100644 --- a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java @@ -2184,6 +2184,165 @@ protected void assertEqualsNullTolerant(Object actual, Object expected) { } } + // ============================================================================ + // Reference Tracking Tests - Test struct field reference sharing + // ============================================================================ + + /** + * Inner struct for reference tracking tests in SCHEMA_CONSISTENT mode (compatible=false). A + * simple struct with id and name fields. + */ + @Data + static class RefInnerSchemaConsistent { + int id; + String name; + } + + /** + * Outer struct for reference tracking tests in SCHEMA_CONSISTENT mode. Contains two fields that + * can point to the same RefInnerSchemaConsistent instance. Both fields have ref tracking enabled. + */ + @Data + static class RefOuterSchemaConsistent { + @ForyField(ref = true, nullable = true) + RefInnerSchemaConsistent inner1; + + @ForyField(ref = true, nullable = true) + RefInnerSchemaConsistent inner2; + } + + /** + * Test reference tracking in SCHEMA_CONSISTENT mode (compatible=false). Creates an outer struct + * with two fields pointing to the same inner struct instance. Verifies that after + * serialization/deserialization across languages, both fields still reference the same object. + */ + @Test + public void testRefSchemaConsistent() throws java.io.IOException { + String caseName = "test_ref_schema_consistent"; + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withRefTracking(true) + .withCodegen(false) + .build(); + fory.register(RefInnerSchemaConsistent.class, 501); + fory.register(RefOuterSchemaConsistent.class, 502); + + // Create inner struct + RefInnerSchemaConsistent inner = new RefInnerSchemaConsistent(); + inner.id = 42; + inner.name = "shared_inner"; + + // Create outer struct with both fields pointing to the same inner struct + RefOuterSchemaConsistent outer = new RefOuterSchemaConsistent(); + outer.inner1 = inner; + outer.inner2 = inner; // Same reference as inner1 + + // Verify Java serialization preserves reference identity + byte[] javaBytes = fory.serialize(outer); + RefOuterSchemaConsistent javaResult = (RefOuterSchemaConsistent) fory.deserialize(javaBytes); + Assert.assertSame( + javaResult.inner1, javaResult.inner2, "Java: inner1 and inner2 should be same object"); + Assert.assertEquals(javaResult.inner1.id, 42); + Assert.assertEquals(javaResult.inner1.name, "shared_inner"); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, outer); + + ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); + runPeer(ctx); + + MemoryBuffer buffer2 = readBuffer(ctx.dataFile()); + RefOuterSchemaConsistent result = (RefOuterSchemaConsistent) fory.deserialize(buffer2); + + // Verify reference identity is preserved after cross-language round-trip + Assert.assertSame( + result.inner1, + result.inner2, + "After xlang round-trip: inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1.id, 42); + Assert.assertEquals(result.inner1.name, "shared_inner"); + } + + /** + * Inner struct for reference tracking tests in COMPATIBLE mode (compatible=true). A simple struct + * with id and name fields. + */ + @Data + static class RefInnerCompatible { + int id; + String name; + } + + /** + * Outer struct for reference tracking tests in COMPATIBLE mode. Contains two fields that can + * point to the same RefInnerCompatible instance. Both fields have ref tracking enabled. + */ + @Data + static class RefOuterCompatible { + @ForyField(ref = true, nullable = true) + RefInnerCompatible inner1; + + @ForyField(ref = true, nullable = true) + RefInnerCompatible inner2; + } + + /** + * Test reference tracking in COMPATIBLE mode (compatible=true). Creates an outer struct with two + * fields pointing to the same inner struct instance. Verifies that after + * serialization/deserialization across languages, both fields still reference the same object. + */ + @Test + public void testRefCompatible() throws java.io.IOException { + String caseName = "test_ref_compatible"; + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withRefTracking(true) + .withCodegen(false) + .withMetaCompressor(new NoOpMetaCompressor()) + .build(); + fory.register(RefInnerCompatible.class, 503); + fory.register(RefOuterCompatible.class, 504); + + // Create inner struct + RefInnerCompatible inner = new RefInnerCompatible(); + inner.id = 99; + inner.name = "compatible_shared"; + + // Create outer struct with both fields pointing to the same inner struct + RefOuterCompatible outer = new RefOuterCompatible(); + outer.inner1 = inner; + outer.inner2 = inner; // Same reference as inner1 + + // Verify Java serialization preserves reference identity + byte[] javaBytes = fory.serialize(outer); + RefOuterCompatible javaResult = (RefOuterCompatible) fory.deserialize(javaBytes); + Assert.assertSame( + javaResult.inner1, javaResult.inner2, "Java: inner1 and inner2 should be same object"); + Assert.assertEquals(javaResult.inner1.id, 99); + Assert.assertEquals(javaResult.inner1.name, "compatible_shared"); + + MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(256); + fory.serialize(buffer, outer); + + ExecutionContext ctx = prepareExecution(caseName, buffer.getBytes(0, buffer.writerIndex())); + runPeer(ctx); + + MemoryBuffer buffer2 = readBuffer(ctx.dataFile()); + RefOuterCompatible result = (RefOuterCompatible) fory.deserialize(buffer2); + + // Verify reference identity is preserved after cross-language round-trip + Assert.assertSame( + result.inner1, + result.inner2, + "After xlang round-trip: inner1 and inner2 should be same object"); + Assert.assertEquals(result.inner1.id, 99); + Assert.assertEquals(result.inner1.name, "compatible_shared"); + } + /** Normalize null values to empty strings in collections and maps recursively. */ @SuppressWarnings("unchecked") private Object normalizeNulls(Object obj) { From e34d9ee60c7d7cc88953af63c03e9a63e1705353 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 1 Jan 2026 22:10:28 +0800 Subject: [PATCH 02/35] fix go serde --- go/fory/struct.go | 14 ++++---------- go/fory/type_def.go | 8 ++++++-- .../src/test/java/org/apache/fory/GoXlangTest.java | 8 +------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/go/fory/struct.go b/go/fory/struct.go index 13bada0f9a..8b914e622a 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -1656,20 +1656,14 @@ func isStructFieldType(ft FieldType) bool { } typeId := ft.TypeId() // Check base type IDs that need type info (struct and ext, NOT enum) - switch typeId { + // Always check the internal type ID (low byte) to handle composite type IDs + // which may be negative when stored as int32 (e.g., -2288 = (short)128784) + internalTypeId := TypeId(typeId & 0xFF) + switch internalTypeId { case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, EXT, NAMED_EXT: return true } - // Check for composite type IDs (customId << 8 | baseType) - if typeId > 255 { - baseType := typeId & 0xff - switch TypeId(baseType) { - case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, - EXT, NAMED_EXT: - return true - } - } return false } diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 3090429693..24ab4c8e7c 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -592,8 +592,10 @@ func (b *BaseFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInf // This is called for top-level field types where flags are NOT embedded in the type ID func readFieldType(buffer *ByteBuffer, err *Error) (FieldType, error) { typeId := buffer.ReadVaruint32Small7(err) + // Use internal type ID (low byte) for switch, but store the full typeId + internalTypeId := TypeId(typeId & 0xFF) - switch typeId { + switch internalTypeId { case LIST, SET: // For nested types, flags ARE embedded in the type ID elementType, etErr := readFieldTypeWithFlags(buffer, err) @@ -626,8 +628,10 @@ func readFieldTypeWithFlags(buffer *ByteBuffer, err *Error) (FieldType, error) { // trackingRef := (rawValue & 0b1) != 0 // Not used currently // nullable := (rawValue & 0b10) != 0 // Not used currently typeId := rawValue >> 2 + // Use internal type ID (low byte) for switch, but store the full typeId + internalTypeId := TypeId(typeId & 0xFF) - switch typeId { + switch internalTypeId { case LIST, SET: elementType, etErr := readFieldTypeWithFlags(buffer, err) if etErr != nil { diff --git a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java index 976fd622f3..c5fb79c2ac 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java @@ -448,12 +448,6 @@ public void testRefSchemaConsistent() throws java.io.IOException { @Override @Test public void testRefCompatible() throws java.io.IOException { - // Skip: Go struct field reference tracking with nested struct types - // has compatibility issues with Java in COMPATIBLE mode. - // The issue is related to type info indexing when deserializing nested struct fields - // with ref tracking enabled. - // TODO: Investigate TypeInfo indexing for nested struct types with ref tracking. - throw new SkipException( - "Skipping: Go TypeInfo indexing issue for nested structs with ref tracking in COMPATIBLE mode"); + super.testRefCompatible(); } } From eb0658907ea38afb9e7cbe21ae3828d801f01282 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 1 Jan 2026 22:25:04 +0800 Subject: [PATCH 03/35] fix go nullable tests --- go/fory/struct.go | 31 ++++++++++--------- go/fory/type_def.go | 29 ++++++++++------- .../java/org/apache/fory/GoXlangTest.java | 23 +++++--------- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/go/fory/struct.go b/go/fory/struct.go index 8b914e622a..93cf4316fc 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -1083,23 +1083,26 @@ func (s *structSerializer) initFieldsFromTypeResolver(typeResolver *TypeResolver fieldTypeId = typeIdFromKind(fieldType) } // Calculate nullable flag for serialization (wire format): - // - In COMPATIBLE mode: reference types default to nullable=true to match Java's - // wire format where reference types have null flags - // - In SCHEMA_CONSISTENT mode: reference types default to nullable=false to match - // Java's default behavior where reference types are non-nullable unless annotated - // - Primitives (int32, bool, etc.) are always non-nullable - // - Can be overridden by explicit fory tag - // Note: computeHash uses its own nullable calculation for fingerprint matching + // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. + // Only pointer types are nullable by default. + // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, + // so they are nullable by default. + // Can be overridden by explicit fory tag `fory:"nullable"`. internalId := TypeId(fieldTypeId & 0xFF) isEnum := internalId == ENUM || internalId == NAMED_ENUM - // Default nullable based on type (reference types are nullable by default) - // Go's codegen always writes null flags for reference types, so reflect must match - // This is consistent with Go's existing behavior and codegen - nullableFlag := fieldType.Kind() == reflect.Ptr || - fieldType.Kind() == reflect.Slice || - fieldType.Kind() == reflect.Map || - fieldType.Kind() == reflect.Interface + // Determine nullable based on mode + var nullableFlag bool + if typeResolver.fory.config.IsXlang { + // xlang mode: only pointer types are nullable by default per xlang spec + nullableFlag = fieldType.Kind() == reflect.Ptr + } else { + // Native mode: Go's natural semantics - all nil-able types are nullable + nullableFlag = fieldType.Kind() == reflect.Ptr || + fieldType.Kind() == reflect.Slice || + fieldType.Kind() == reflect.Map || + fieldType.Kind() == reflect.Interface + } if foryTag.NullableSet { // Override nullable flag if explicitly set in fory tag nullableFlag = foryTag.Nullable diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 24ab4c8e7c..e15c3304e0 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -431,20 +431,27 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { if err != nil { return nil, fmt.Errorf("failed to build field type for field %s: %w", fieldName, err) } - // Determine nullable based on Go type capability: - // - Pointer types (*T): can hold nil → nullable=true by default - // - Slices, maps, interfaces: can hold nil → nullable=true by default - // - Primitive types (int32, bool, etc.): cannot be nil → nullable=false - // Can be overridden by explicit fory tag + // Determine nullable based on mode: + // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. + // Only pointer types are nullable by default. + // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, + // so they are nullable by default. + // Can be overridden by explicit fory tag `fory:"nullable"` typeId := ft.TypeId() internalId := TypeId(typeId & 0xFF) isEnumField := internalId == ENUM || internalId == NAMED_ENUM - // Default nullable based on whether Go type can be nil - // Pointer types, slices, maps, interfaces can hold nil → nullable=true by default - nullableFlag := field.Type.Kind() == reflect.Ptr || - field.Type.Kind() == reflect.Slice || - field.Type.Kind() == reflect.Map || - field.Type.Kind() == reflect.Interface + // Determine nullable based on mode + var nullableFlag bool + if fory.config.IsXlang { + // xlang mode: only pointer types are nullable by default per xlang spec + nullableFlag = field.Type.Kind() == reflect.Ptr + } else { + // Native mode: Go's natural semantics - all nil-able types are nullable + nullableFlag = field.Type.Kind() == reflect.Ptr || + field.Type.Kind() == reflect.Slice || + field.Type.Kind() == reflect.Map || + field.Type.Kind() == reflect.Interface + } // Override nullable flag if explicitly set in fory tag if foryTag.NullableSet { nullableFlag = foryTag.Nullable diff --git a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java index c5fb79c2ac..919b250838 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java @@ -304,21 +304,13 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { @Override @Test public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - // Go's codegen always writes null flags for slice/map/interface fields, - // which is incompatible with Java's SCHEMA_CONSISTENT mode that expects no null flags. - // TODO: Update Go code generator to respect nullable flag in SCHEMA_CONSISTENT mode. - throw new SkipException( - "Skipping: Go codegen always writes null flags, incompatible with SCHEMA_CONSISTENT mode"); + super.testNullableFieldSchemaConsistentNotNull(); } @Override @Test public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - // Go's codegen always writes null flags for slice/map/interface fields, - // which is incompatible with Java's SCHEMA_CONSISTENT mode that expects no null flags. - // TODO: Update Go code generator to respect nullable flag in SCHEMA_CONSISTENT mode. - throw new SkipException( - "Skipping: Go codegen always writes null flags, incompatible with SCHEMA_CONSISTENT mode"); + super.testNullableFieldSchemaConsistentNull(); } @Override @@ -420,13 +412,14 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { expected.nullableFloat1 = 0.0f; expected.nullableDouble1 = 0.0; expected.nullableBool1 = false; - // Nullable group 2 - Go's nullable reference fields: + // Nullable group 2 - Go's reference fields: // - string (not a pointer): defaults to "" (empty string) when nil in Go - // - slices/maps: can be nil, so Go sends null + // - slices/maps: Go struct doesn't have fory:"nullable" tag, so they're non-nullable + // and are read as empty collections, not nil expected.nullableString2 = ""; - expected.nullableList2 = null; - expected.nullableSet2 = null; - expected.nullableMap2 = null; + expected.nullableList2 = new ArrayList<>(); + expected.nullableSet2 = new HashSet<>(); + expected.nullableMap2 = new HashMap<>(); Assert.assertEquals(result, expected); } From 45e7dc3753d8d260efb62b0808441a15f52d1daf Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Thu, 1 Jan 2026 22:42:26 +0800 Subject: [PATCH 04/35] add rust/cpp/python tests --- cpp/fory/serialization/xlang_test_main.cc | 158 ++++++++++++++++++++++ python/pyfory/tests/xlang_test_main.py | 114 ++++++++++++++++ rust/tests/tests/test_cross_language.rs | 108 +++++++++++++++ 3 files changed, 380 insertions(+) diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index f63fdeaed9..c1248be6ae 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -464,6 +464,70 @@ FORY_STRUCT(NullableComprehensiveCompatible, byte_field, short_field, int_field, nullable_float1, nullable_double1, nullable_bool1, nullable_string2, nullable_list2, nullable_set2, nullable_map2); +// ============================================================================ +// Reference Tracking Test Types - Cross-language shared reference tests +// ============================================================================ + +// Inner struct for reference tracking test (SCHEMA_CONSISTENT mode) +// Matches Java RefInnerSchemaConsistent with type ID 501 +struct RefInnerSchemaConsistent { + int32_t id; + std::string name; + bool operator==(const RefInnerSchemaConsistent &other) const { + return id == other.id && name == other.name; + } +}; +FORY_STRUCT(RefInnerSchemaConsistent, id, name); + +// Outer struct for reference tracking test (SCHEMA_CONSISTENT mode) +// Contains two fields that both point to the same inner object. +// Matches Java RefOuterSchemaConsistent with type ID 502 +// Uses std::shared_ptr for reference tracking to share the same object +struct RefOuterSchemaConsistent { + std::shared_ptr inner1; + std::shared_ptr inner2; + bool operator==(const RefOuterSchemaConsistent &other) const { + bool inner1_eq = (inner1 == nullptr && other.inner1 == nullptr) || + (inner1 != nullptr && other.inner1 != nullptr && + *inner1 == *other.inner1); + bool inner2_eq = (inner2 == nullptr && other.inner2 == nullptr) || + (inner2 != nullptr && other.inner2 != nullptr && + *inner2 == *other.inner2); + return inner1_eq && inner2_eq; + } +}; +FORY_STRUCT(RefOuterSchemaConsistent, inner1, inner2); + +// Inner struct for reference tracking test (COMPATIBLE mode) +// Matches Java RefInnerCompatible with type ID 503 +struct RefInnerCompatible { + int32_t id; + std::string name; + bool operator==(const RefInnerCompatible &other) const { + return id == other.id && name == other.name; + } +}; +FORY_STRUCT(RefInnerCompatible, id, name); + +// Outer struct for reference tracking test (COMPATIBLE mode) +// Contains two fields that both point to the same inner object. +// Matches Java RefOuterCompatible with type ID 504 +// Uses std::shared_ptr for reference tracking to share the same object +struct RefOuterCompatible { + std::shared_ptr inner1; + std::shared_ptr inner2; + bool operator==(const RefOuterCompatible &other) const { + bool inner1_eq = (inner1 == nullptr && other.inner1 == nullptr) || + (inner1 != nullptr && other.inner1 != nullptr && + *inner1 == *other.inner1); + bool inner2_eq = (inner2 == nullptr && other.inner2 == nullptr) || + (inner2 != nullptr && other.inner2 != nullptr && + *inner2 == *other.inner2); + return inner1_eq && inner2_eq; + } +}; +FORY_STRUCT(RefOuterCompatible, inner1, inner2); + namespace fory { namespace serialization { @@ -639,6 +703,8 @@ void RunTestNullableFieldSchemaConsistentNotNull(const std::string &data_file); void RunTestNullableFieldSchemaConsistentNull(const std::string &data_file); void RunTestNullableFieldCompatibleNotNull(const std::string &data_file); void RunTestNullableFieldCompatibleNull(const std::string &data_file); +void RunTestRefSchemaConsistent(const std::string &data_file); +void RunTestRefCompatible(const std::string &data_file); } // namespace int main(int argc, char **argv) { @@ -730,6 +796,10 @@ int main(int argc, char **argv) { RunTestNullableFieldCompatibleNotNull(data_file); } else if (case_name == "test_nullable_field_compatible_null") { RunTestNullableFieldCompatibleNull(data_file); + } else if (case_name == "test_ref_schema_consistent") { + RunTestRefSchemaConsistent(data_file); + } else if (case_name == "test_ref_compatible") { + RunTestRefCompatible(data_file); } else { Fail("Unknown test case: " + case_name); } @@ -2152,4 +2222,92 @@ void RunTestNullableFieldCompatibleNull(const std::string &data_file) { WriteFile(data_file, out); } +// ============================================================================ +// Reference Tracking Tests - Cross-language shared reference tests +// ============================================================================ + +void RunTestRefSchemaConsistent(const std::string &data_file) { + auto bytes = ReadFile(data_file); + // SCHEMA_CONSISTENT mode: compatible=false, xlang=true + auto fory = BuildFory(false, true); + EnsureOk(fory.register_struct(501), + "register RefInnerSchemaConsistent"); + EnsureOk(fory.register_struct(502), + "register RefOuterSchemaConsistent"); + + Buffer buffer = MakeBuffer(bytes); + auto outer = ReadNext(fory, buffer); + + // Both inner1 and inner2 should have values + if (outer.inner1 == nullptr) { + Fail("RefOuterSchemaConsistent: inner1 should not be null"); + } + if (outer.inner2 == nullptr) { + Fail("RefOuterSchemaConsistent: inner2 should not be null"); + } + + // Both should have the same values (they reference the same object in Java) + if (outer.inner1->id != 42) { + Fail("RefOuterSchemaConsistent: inner1.id should be 42, got " + + std::to_string(outer.inner1->id)); + } + if (outer.inner1->name != "shared_inner") { + Fail("RefOuterSchemaConsistent: inner1.name should be 'shared_inner', got " + + outer.inner1->name); + } + if (*outer.inner1 != *outer.inner2) { + Fail("RefOuterSchemaConsistent: inner1 and inner2 should be equal (same " + "reference)"); + } + + // In C++, shared_ptr may or may not point to the same object after + // deserialization, depending on reference tracking implementation. + // The key test is that both have equal values. + + // Re-serialize and write back + std::vector out; + AppendSerialized(fory, outer, out); + WriteFile(data_file, out); +} + +void RunTestRefCompatible(const std::string &data_file) { + auto bytes = ReadFile(data_file); + // COMPATIBLE mode: compatible=true, xlang=true + auto fory = BuildFory(true, true); + EnsureOk(fory.register_struct(503), + "register RefInnerCompatible"); + EnsureOk(fory.register_struct(504), + "register RefOuterCompatible"); + + Buffer buffer = MakeBuffer(bytes); + auto outer = ReadNext(fory, buffer); + + // Both inner1 and inner2 should have values + if (outer.inner1 == nullptr) { + Fail("RefOuterCompatible: inner1 should not be null"); + } + if (outer.inner2 == nullptr) { + Fail("RefOuterCompatible: inner2 should not be null"); + } + + // Both should have the same values (they reference the same object in Java) + if (outer.inner1->id != 99) { + Fail("RefOuterCompatible: inner1.id should be 99, got " + + std::to_string(outer.inner1->id)); + } + if (outer.inner1->name != "compatible_shared") { + Fail("RefOuterCompatible: inner1.name should be 'compatible_shared', got " + + outer.inner1->name); + } + if (*outer.inner1 != *outer.inner2) { + Fail("RefOuterCompatible: inner1 and inner2 should be equal (same " + "reference)"); + } + + // Re-serialize and write back + std::vector out; + AppendSerialized(fory, outer, out); + WriteFile(data_file, out); +} + } // namespace diff --git a/python/pyfory/tests/xlang_test_main.py b/python/pyfory/tests/xlang_test_main.py index 4ca5913d93..c161b1c94c 100644 --- a/python/pyfory/tests/xlang_test_main.py +++ b/python/pyfory/tests/xlang_test_main.py @@ -219,6 +219,39 @@ class NullableComprehensiveSchemaConsistent: nullable_map: Optional[Dict[str, str]] = None +# ============================================================================ +# Reference Tracking Test Types +# ============================================================================ + + +@dataclass +class RefInnerSchemaConsistent: + """Inner struct for reference tracking test (SCHEMA_CONSISTENT mode).""" + id: pyfory.int32 = 0 + name: str = "" + + +@dataclass +class RefOuterSchemaConsistent: + """Outer struct with two fields pointing to the same inner object (SCHEMA_CONSISTENT mode).""" + inner1: Optional[RefInnerSchemaConsistent] = None + inner2: Optional[RefInnerSchemaConsistent] = None + + +@dataclass +class RefInnerCompatible: + """Inner struct for reference tracking test (COMPATIBLE mode).""" + id: pyfory.int32 = 0 + name: str = "" + + +@dataclass +class RefOuterCompatible: + """Outer struct with two fields pointing to the same inner object (COMPATIBLE mode).""" + inner1: Optional[RefInnerCompatible] = None + inner2: Optional[RefInnerCompatible] = None + + @dataclass class NullableComprehensiveCompatible: """ @@ -1104,6 +1137,87 @@ def test_nullable_field_compatible_null(): f.write(new_bytes) +# ============================================================================ +# Reference Tracking Tests +# ============================================================================ + + +def test_ref_schema_consistent(): + """ + Test cross-language reference tracking in SCHEMA_CONSISTENT mode (compatible=false). + + This test verifies that when Java serializes an object where two fields point to + the same instance, Python can properly deserialize it and both fields will reference + the same object. When re-serializing, the reference relationship should be preserved. + """ + data_file = get_data_file() + with open(data_file, "rb") as f: + data_bytes = f.read() + + fory = pyfory.Fory(xlang=True, compatible=False) + fory.register_type(RefInnerSchemaConsistent, type_id=501) + fory.register_type(RefOuterSchemaConsistent, type_id=502) + + outer = fory.deserialize(data_bytes) + debug_print(f"Deserialized: {outer}") + + # Both inner1 and inner2 should have values + assert outer.inner1 is not None, "inner1 should not be None" + assert outer.inner2 is not None, "inner2 should not be None" + + # Both should have the same values (they reference the same object in Java) + assert outer.inner1.id == 42, f"inner1.id should be 42, got {outer.inner1.id}" + assert outer.inner1.name == "shared_inner", f"inner1.name should be 'shared_inner', got {outer.inner1.name}" + assert outer.inner1 == outer.inner2, "inner1 and inner2 should be equal (same reference)" + + # In Python, after deserialization with reference tracking, inner1 and inner2 + # should point to the same object (identity check) + assert outer.inner1 is outer.inner2, "inner1 and inner2 should be the same object (reference identity)" + + # Re-serialize and write back + new_bytes = fory.serialize(outer) + with open(data_file, "wb") as f: + f.write(new_bytes) + + +def test_ref_compatible(): + """ + Test cross-language reference tracking in COMPATIBLE mode (compatible=true). + + This test verifies reference tracking works correctly with schema evolution support. + The inner object is shared between two fields, and this relationship should be + preserved through serialization/deserialization. + """ + data_file = get_data_file() + with open(data_file, "rb") as f: + data_bytes = f.read() + + fory = pyfory.Fory(xlang=True, compatible=True) + fory.register_type(RefInnerCompatible, type_id=503) + fory.register_type(RefOuterCompatible, type_id=504) + + outer = fory.deserialize(data_bytes) + debug_print(f"Deserialized: {outer}") + + # Both inner1 and inner2 should have values + assert outer.inner1 is not None, "inner1 should not be None" + assert outer.inner2 is not None, "inner2 should not be None" + + # Both should have the same values (they reference the same object in Java) + assert outer.inner1.id == 99, f"inner1.id should be 99, got {outer.inner1.id}" + assert outer.inner1.name == "compatible_shared", f"inner1.name should be 'compatible_shared', got {outer.inner1.name}" + assert outer.inner1 == outer.inner2, "inner1 and inner2 should be equal (same reference)" + + # In Python, after deserialization with reference tracking, inner1 and inner2 + # should point to the same object (identity check) + assert outer.inner1 is outer.inner2, "inner1 and inner2 should be the same object (reference identity)" + + # Re-serialize and write back + new_bytes = fory.serialize(outer) + with open(data_file, "wb") as f: + f.write(new_bytes) + + if __name__ == "__main__": """ This file is executed by PythonXlangTest.java and other cross-language tests. diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index b341c281b5..f0ff1738b8 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1638,3 +1638,111 @@ fn test_union_xlang() { fory.serialize_to(&mut buf, &struct2).unwrap(); fs::write(&data_file_path, buf).unwrap(); } + +// ============================================================================ +// Reference Tracking Tests - Cross-language shared reference tests +// ============================================================================ + +/// Inner struct for reference tracking test (SCHEMA_CONSISTENT mode) +/// Matches Java RefInnerSchemaConsistent with type ID 501 +#[derive(ForyObject, Debug, PartialEq, Clone)] +struct RefInnerSchemaConsistent { + id: i32, + name: String, +} + +/// Outer struct for reference tracking test (SCHEMA_CONSISTENT mode) +/// Contains two fields that both point to the same inner object. +/// Matches Java RefOuterSchemaConsistent with type ID 502 +#[derive(ForyObject, Debug, PartialEq)] +struct RefOuterSchemaConsistent { + #[fory(ref = true, nullable = true)] + inner1: Option, + #[fory(ref = true, nullable = true)] + inner2: Option, +} + +/// Inner struct for reference tracking test (COMPATIBLE mode) +/// Matches Java RefInnerCompatible with type ID 503 +#[derive(ForyObject, Debug, PartialEq, Clone)] +struct RefInnerCompatible { + id: i32, + name: String, +} + +/// Outer struct for reference tracking test (COMPATIBLE mode) +/// Contains two fields that both point to the same inner object. +/// Matches Java RefOuterCompatible with type ID 504 +#[derive(ForyObject, Debug, PartialEq)] +struct RefOuterCompatible { + #[fory(ref = true, nullable = true)] + inner1: Option, + #[fory(ref = true, nullable = true)] + inner2: Option, +} + +/// Test cross-language reference tracking in SCHEMA_CONSISTENT mode (compatible=false). +/// +/// This test verifies that when Java serializes an object where two fields point to +/// the same instance, Rust can properly deserialize it and both fields will contain +/// equal values. When re-serializing, the reference relationship should be preserved. +#[test] +#[ignore] +fn test_ref_schema_consistent() { + let data_file_path = get_data_file(); + let bytes = fs::read(&data_file_path).unwrap(); + + let mut fory = Fory::default().compatible(false).xlang(true); + fory.register::(501).unwrap(); + fory.register::(502).unwrap(); + + let outer: RefOuterSchemaConsistent = fory.deserialize(&bytes).unwrap(); + + // Both inner1 and inner2 should have values + assert!(outer.inner1.is_some(), "inner1 should not be None"); + assert!(outer.inner2.is_some(), "inner2 should not be None"); + + // Both should have the same values (they reference the same object in Java) + let inner1 = outer.inner1.as_ref().unwrap(); + let inner2 = outer.inner2.as_ref().unwrap(); + assert_eq!(inner1.id, 42); + assert_eq!(inner1.name, "shared_inner"); + assert_eq!(inner1, inner2, "inner1 and inner2 should be equal (same reference)"); + + // Re-serialize and write back + let new_bytes = fory.serialize(&outer).unwrap(); + fs::write(&data_file_path, new_bytes).unwrap(); +} + +/// Test cross-language reference tracking in COMPATIBLE mode (compatible=true). +/// +/// This test verifies reference tracking works correctly with schema evolution support. +/// The inner object is shared between two fields, and this relationship should be +/// preserved through serialization/deserialization. +#[test] +#[ignore] +fn test_ref_compatible() { + let data_file_path = get_data_file(); + let bytes = fs::read(&data_file_path).unwrap(); + + let mut fory = Fory::default().compatible(true).xlang(true); + fory.register::(503).unwrap(); + fory.register::(504).unwrap(); + + let outer: RefOuterCompatible = fory.deserialize(&bytes).unwrap(); + + // Both inner1 and inner2 should have values + assert!(outer.inner1.is_some(), "inner1 should not be None"); + assert!(outer.inner2.is_some(), "inner2 should not be None"); + + // Both should have the same values (they reference the same object in Java) + let inner1 = outer.inner1.as_ref().unwrap(); + let inner2 = outer.inner2.as_ref().unwrap(); + assert_eq!(inner1.id, 99); + assert_eq!(inner1.name, "compatible_shared"); + assert_eq!(inner1, inner2, "inner1 and inner2 should be equal (same reference)"); + + // Re-serialize and write back + let new_bytes = fory.serialize(&outer).unwrap(); + fs::write(&data_file_path, new_bytes).unwrap(); +} From 0a99e31bb847381c21d63657686e1b14d5c55ad8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 14:47:21 +0800 Subject: [PATCH 05/35] fix xlang ref tracking --- cpp/fory/serialization/fory.h | 14 +- cpp/fory/serialization/ref_resolver.h | 5 + .../serialization/smart_ptr_serializers.h | 28 ++- cpp/fory/serialization/struct_serializer.h | 17 +- cpp/fory/serialization/type_resolver.cc | 21 ++- cpp/fory/serialization/type_resolver.h | 34 ++++ cpp/fory/serialization/xlang_test_main.cc | 40 ++++- .../java/org/apache/fory/CPPXlangTest.java | 10 ++ .../java/org/apache/fory/PythonXlangTest.java | 10 ++ .../java/org/apache/fory/RustXlangTest.java | 10 ++ python/pyfory/struct.py | 5 + python/pyfory/tests/xlang_test_main.py | 16 +- rust/fory-core/src/config.rs | 11 ++ rust/fory-core/src/fory.rs | 40 ++++- rust/fory-core/src/resolver/ref_resolver.rs | 15 ++ rust/fory-core/src/serializer/arc.rs | 109 ++++-------- rust/fory-core/src/serializer/rc.rs | 109 ++++-------- rust/fory-core/src/serializer/struct_.rs | 13 +- rust/fory-core/src/types.rs | 7 +- rust/fory-derive/src/object/field_meta.rs | 5 + rust/fory-derive/src/object/misc.rs | 8 +- rust/fory-derive/src/object/read.rs | 159 +++++++++++++----- rust/fory-derive/src/object/util.rs | 29 +++- rust/fory-derive/src/object/write.rs | 73 ++++---- rust/tests/tests/test_cross_language.rs | 32 ++-- rust/tests/tests/test_rc_arc.rs | 20 ++- 26 files changed, 543 insertions(+), 297 deletions(-) diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h index 7a0f93e802..cfd288ea31 100644 --- a/cpp/fory/serialization/fory.h +++ b/cpp/fory/serialization/fory.h @@ -623,8 +623,11 @@ class Fory : public BaseFory { buffer.WriteInt32(-1); // Placeholder for meta offset (fixed 4 bytes) } - // Top-level serialization: YES ref flags, yes type info - Serializer::write(obj, *write_ctx_, RefMode::NullOnly, true); + // Top-level serialization: use Tracking if ref tracking is enabled, + // otherwise NullOnly for nullable handling + const RefMode top_level_ref_mode = + write_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; + Serializer::write(obj, *write_ctx_, top_level_ref_mode, true); // Check for errors at serialization boundary if (FORY_PREDICT_FALSE(write_ctx_->has_error())) { return Unexpected(write_ctx_->take_error()); @@ -654,8 +657,11 @@ class Fory : public BaseFory { } } - // Top-level deserialization: YES ref flags, yes type info - T result = Serializer::read(*read_ctx_, RefMode::NullOnly, true); + // Top-level deserialization: use Tracking if ref tracking is enabled, + // otherwise NullOnly for nullable handling + const RefMode top_level_ref_mode = + read_ctx_->track_ref() ? RefMode::Tracking : RefMode::NullOnly; + T result = Serializer::read(*read_ctx_, top_level_ref_mode, true); // Check for errors at deserialization boundary if (FORY_PREDICT_FALSE(read_ctx_->has_error())) { return Unexpected(read_ctx_->take_error()); diff --git a/cpp/fory/serialization/ref_resolver.h b/cpp/fory/serialization/ref_resolver.h index cbea3b6e4b..cde89a1165 100644 --- a/cpp/fory/serialization/ref_resolver.h +++ b/cpp/fory/serialization/ref_resolver.h @@ -78,6 +78,11 @@ class RefWriter { return false; } + /// Reserve a ref_id slot without storing a pointer. + /// Used for types (like structs) that Java tracks but C++ doesn't reference. + /// This keeps ref ID numbering in sync across languages. + uint32_t reserve_ref_id() { return next_id_++; } + /// Reset resolver for reuse in new serialization. /// Clears all tracked references. void reset() { diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index c0ae5ab5e7..019feb902c 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -323,14 +323,21 @@ template struct Serializer> { return; } + bool is_first_occurrence = false; if (ctx.track_ref()) { if (ctx.ref_writer().try_write_shared_ref(ctx, ptr)) { return; } + is_first_occurrence = true; } else { ctx.write_int8(NOT_NULL_VALUE_FLAG); } + // In compatible mode with ref tracking, first occurrence (RefValue) + // requires type info to be written after the ref flag. + const bool should_write_type = + ctx.is_compatible() && is_first_occurrence ? true : write_type; + // For polymorphic types, serialize the concrete type dynamically if constexpr (is_polymorphic) { // Get the concrete type_index from the actual object @@ -345,7 +352,7 @@ template struct Serializer> { const TypeInfo *type_info = type_info_res.value(); // Write type info if requested - if (write_type) { + if (should_write_type) { auto write_res = ctx.write_any_typeinfo( static_cast(TypeId::UNKNOWN), concrete_type_id); if (!write_res.ok()) { @@ -362,7 +369,7 @@ template struct Serializer> { // Non-polymorphic path Serializer::write( *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + should_write_type); } } @@ -481,7 +488,8 @@ template struct Serializer> { } uint32_t reserved_ref_id = 0; - if (flag == REF_VALUE_FLAG) { + const bool is_first_occurrence = flag == REF_VALUE_FLAG; + if (is_first_occurrence) { if (!tracking_refs) { ctx.set_error(Error::invalid_ref( "REF_VALUE flag encountered when reference tracking disabled")); @@ -490,9 +498,15 @@ template struct Serializer> { reserved_ref_id = ctx.ref_reader().reserve_ref_id(); } + // In compatible mode with ref tracking, first occurrence (RefValue) + // has type info written after the ref flag by Java/Go. + // So we must read type info for first occurrence. + const bool should_read_type = + ctx.is_compatible() && is_first_occurrence ? true : read_type; + // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!read_type) { + if (!should_read_type) { ctx.set_error(Error::type_error( "Cannot deserialize polymorphic std::shared_ptr " "without type info (read_type=false)")); @@ -520,7 +534,7 @@ template struct Serializer> { } T *obj_ptr = static_cast(raw_ptr); auto result = std::shared_ptr(obj_ptr); - if (flag == REF_VALUE_FLAG) { + if (is_first_occurrence) { ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); } return result; @@ -528,12 +542,12 @@ template struct Serializer> { // Non-polymorphic path T value = Serializer::read( ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - read_type); + should_read_type); if (ctx.has_error()) { return nullptr; } auto result = std::make_shared(std::move(value)); - if (flag == REF_VALUE_FLAG) { + if (is_first_occurrence) { ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); } return result; diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 3b6ccdcf57..786fc30069 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -1947,7 +1947,15 @@ struct Serializer>> { static void write(const T &obj, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - write_not_null_ref_flag(ctx, ref_mode); + // Handle ref flag based on mode + if (ref_mode == RefMode::Tracking && ctx.track_ref()) { + // In Tracking mode, write REF_VALUE_FLAG (0) and reserve a ref_id slot + // to keep ref IDs in sync with Java (which tracks all objects) + ctx.write_int8(REF_VALUE_FLAG); + ctx.ref_writer().reserve_ref_id(); + } else if (ref_mode != RefMode::None) { + ctx.write_int8(NOT_NULL_VALUE_FLAG); + } if (write_type) { // Direct lookup using compile-time type_index() - O(1) hash lookup @@ -2048,6 +2056,13 @@ struct Serializer>> { constexpr int8_t null_flag = static_cast(RefFlag::Null); if (ref_flag == not_null_value_flag || ref_flag == ref_value_flag) { + // When ref_flag is RefValue (0), Java assigned a ref_id to this object. + // We must reserve a matching ref_id slot so that nested refs line up. + // Structs can't actually be referenced (only shared_ptrs can), but we + // need the ref_id numbering to stay in sync with Java. + if (ctx.track_ref() && ref_flag == ref_value_flag) { + ctx.ref_reader().reserve_ref_id(); + } // In compatible mode: use meta sharing (matches Rust behavior) if (ctx.is_compatible()) { // In compatible mode: always use remote TypeMeta for schema evolution diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 3f85389c4f..319eadad26 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -133,8 +133,11 @@ Result, Error> FieldInfo::to_bytes() const { uint8_t header = (std::min(FIELD_NAME_SIZE_THRESHOLD, name_size - 1) << 2) & 0x3C; + if (field_type.ref_tracking) { + header |= 1; // bit 0 for ref tracking + } if (field_type.nullable) { - header |= 2; + header |= 2; // bit 1 for nullable } header |= (encoding_idx << 6); @@ -974,16 +977,19 @@ std::string TypeMeta::compute_struct_fingerprint( // Java's ObjectSerializer.getTypeId returns Types.UNKNOWN (0) for: // - abstract classes, interfaces, and enum types + // - user-defined struct types (in xlang mode) // This aligns the hash computation with Java's behavior. uint32_t effective_type_id = fi.field_type.type_id; if (effective_type_id == static_cast(TypeId::ENUM) || - effective_type_id == static_cast(TypeId::NAMED_ENUM)) { + effective_type_id == static_cast(TypeId::NAMED_ENUM) || + effective_type_id == static_cast(TypeId::STRUCT) || + effective_type_id == static_cast(TypeId::NAMED_STRUCT)) { effective_type_id = static_cast(TypeId::UNKNOWN); } fingerprint.append(std::to_string(effective_type_id)); fingerprint.push_back(','); - // ref flag: currently always 0 in C++ (no ref tracking support yet) - fingerprint.push_back('0'); + // Use field-level ref tracking flag from FORY_FIELD_TAGS or fory::field<> + fingerprint.push_back(fi.field_type.ref_tracking ? '1' : '0'); fingerprint.push_back(','); fingerprint.append(fi.field_type.nullable ? "1;" : "0;"); } @@ -1001,11 +1007,10 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { // Use the low 64 bits and then keep low 32 bits as i32. uint64_t low = static_cast(hash_out[0]); uint32_t version = static_cast(low & 0xFFFF'FFFFu); -#ifdef FORY_DEBUG + // DEBUG: Print fingerprint for debugging version mismatch std::cerr << "[xlang][debug] struct_version type_name=" << meta.type_name - << ", fingerprint=\"" << fingerprint << "\" version=" << version - << std::endl; -#endif + << ", fingerprint=\"" << fingerprint + << "\" version=" << static_cast(version) << std::endl; return static_cast(version); } diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 7fb6cc8a26..fe4b0a8d22 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -497,7 +497,41 @@ template struct FieldInfoBuilder { // Unwrap fory::field<> to get the underlying type for FieldTypeBuilder using UnwrappedFieldType = fory::unwrap_field_t; + // Get nullable and track_ref from field tags (FORY_FIELD_TAGS or + // fory::field<>) + constexpr bool is_nullable = []() { + if constexpr (is_fory_field_v) { + return ActualFieldType::is_nullable; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::is_nullable; + } else { + // Default: nullable if std::optional or std::shared_ptr + return is_optional_v || + is_shared_ptr_v; + } + }(); + + constexpr bool track_ref = []() { + if constexpr (is_fory_field_v) { + return ActualFieldType::track_ref; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::track_ref; + } else { + return false; + } + }(); + FieldType field_type = FieldTypeBuilder::build(false); + // Override nullable and ref_tracking from field-level metadata + field_type.nullable = is_nullable; + field_type.ref_tracking = track_ref; + field_type.ref_mode = make_ref_mode(is_nullable, track_ref); + // DEBUG: Print field info for debugging fingerprint mismatch + std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() + << " Index=" << Index << " field=" << field_name + << " has_tags=" << ::fory::detail::has_field_tags_v + << " is_nullable=" << is_nullable << " track_ref=" << track_ref + << std::endl; return FieldInfo(std::move(field_name), std::move(field_type)); } }; diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index c1248be6ae..220b3e2cb6 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -476,6 +476,9 @@ struct RefInnerSchemaConsistent { bool operator==(const RefInnerSchemaConsistent &other) const { return id == other.id && name == other.name; } + bool operator!=(const RefInnerSchemaConsistent &other) const { + return !(*this == other); + } }; FORY_STRUCT(RefInnerSchemaConsistent, id, name); @@ -497,6 +500,22 @@ struct RefOuterSchemaConsistent { } }; FORY_STRUCT(RefOuterSchemaConsistent, inner1, inner2); +FORY_FIELD_TAGS(RefOuterSchemaConsistent, (inner1, 0, nullable, ref), + (inner2, 1, nullable, ref)); +// Verify field tags are correctly parsed +static_assert(fory::detail::has_field_tags_v, + "RefOuterSchemaConsistent should have field tags"); +static_assert(fory::detail::GetFieldTagEntry::id == + 0, + "inner1 should have id=0"); +static_assert( + fory::detail::GetFieldTagEntry::is_nullable == + true, + "inner1 should be nullable"); +static_assert( + fory::detail::GetFieldTagEntry::track_ref == + true, + "inner1 should have track_ref=true"); // Inner struct for reference tracking test (COMPATIBLE mode) // Matches Java RefInnerCompatible with type ID 503 @@ -506,6 +525,9 @@ struct RefInnerCompatible { bool operator==(const RefInnerCompatible &other) const { return id == other.id && name == other.name; } + bool operator!=(const RefInnerCompatible &other) const { + return !(*this == other); + } }; FORY_STRUCT(RefInnerCompatible, id, name); @@ -527,6 +549,8 @@ struct RefOuterCompatible { } }; FORY_STRUCT(RefOuterCompatible, inner1, inner2); +FORY_FIELD_TAGS(RefOuterCompatible, (inner1, 0, nullable, ref), + (inner2, 1, nullable, ref)); namespace fory { namespace serialization { @@ -653,11 +677,12 @@ void AppendSerialized(Fory &fory, const T &value, std::vector &out) { } Fory BuildFory(bool compatible = true, bool xlang = true, - bool check_struct_version = false) { + bool check_struct_version = false, bool track_ref = false) { return Fory::builder() .compatible(compatible) .xlang(xlang) .check_struct_version(check_struct_version) + .track_ref(track_ref) .build(); } @@ -2228,8 +2253,8 @@ void RunTestNullableFieldCompatibleNull(const std::string &data_file) { void RunTestRefSchemaConsistent(const std::string &data_file) { auto bytes = ReadFile(data_file); - // SCHEMA_CONSISTENT mode: compatible=false, xlang=true - auto fory = BuildFory(false, true); + // SCHEMA_CONSISTENT mode: compatible=false, xlang=true, check_struct_version=true, track_ref=true + auto fory = BuildFory(false, true, true, true); EnsureOk(fory.register_struct(501), "register RefInnerSchemaConsistent"); EnsureOk(fory.register_struct(502), @@ -2252,8 +2277,9 @@ void RunTestRefSchemaConsistent(const std::string &data_file) { std::to_string(outer.inner1->id)); } if (outer.inner1->name != "shared_inner") { - Fail("RefOuterSchemaConsistent: inner1.name should be 'shared_inner', got " + - outer.inner1->name); + Fail( + "RefOuterSchemaConsistent: inner1.name should be 'shared_inner', got " + + outer.inner1->name); } if (*outer.inner1 != *outer.inner2) { Fail("RefOuterSchemaConsistent: inner1 and inner2 should be equal (same " @@ -2272,8 +2298,8 @@ void RunTestRefSchemaConsistent(const std::string &data_file) { void RunTestRefCompatible(const std::string &data_file) { auto bytes = ReadFile(data_file); - // COMPATIBLE mode: compatible=true, xlang=true - auto fory = BuildFory(true, true); + // COMPATIBLE mode: compatible=true, xlang=true, check_struct_version=false, track_ref=true + auto fory = BuildFory(true, true, false, true); EnsureOk(fory.register_struct(503), "register RefInnerCompatible"); EnsureOk(fory.register_struct(504), diff --git a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java index aed21ab137..7097e50cce 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java @@ -368,4 +368,14 @@ public void testUnionXlang() throws java.io.IOException { // Skip: C++ doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: C++ Union xlang support not implemented"); } + + @Test + public void testRefSchemaConsistent() throws java.io.IOException { + super.testRefSchemaConsistent(); + } + + @Test + public void testRefCompatible() throws java.io.IOException { + super.testRefCompatible(); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java index 9afcccbac0..d079c883ed 100644 --- a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java @@ -290,4 +290,14 @@ public void testUnionXlang() throws IOException { // Skip: Python doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Python Union xlang support not implemented"); } + + @Test + public void testRefSchemaConsistent() throws IOException { + super.testRefSchemaConsistent(); + } + + @Test + public void testRefCompatible() throws IOException { + super.testRefCompatible(); + } } diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java index c753607a49..60c00a0024 100644 --- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java @@ -257,4 +257,14 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { public void testUnionXlang() throws java.io.IOException { super.testUnionXlang(); } + + @Test + public void testRefSchemaConsistent() throws java.io.IOException { + super.testRefSchemaConsistent(); + } + + @Test + public void testRefCompatible() throws java.io.IOException { + super.testRefCompatible(); + } } diff --git a/python/pyfory/struct.py b/python/pyfory/struct.py index 53e09e0b67..d9c0ad0aaa 100644 --- a/python/pyfory/struct.py +++ b/python/pyfory/struct.py @@ -1053,6 +1053,11 @@ def visit_dict(self, field_name, key_type, value_type, types_path=None): def visit_customized(self, field_name, type_, types_path=None): if issubclass(type_, enum.Enum): return self.fory.type_resolver.get_serializer(type_) + # For custom types (dataclasses, etc.), try to get or create serializer + # This enables field-level serializer resolution for types like inner structs + typeinfo = self.fory.type_resolver.get_typeinfo(type_, create=False) + if typeinfo is not None: + return typeinfo.serializer return None def visit_other(self, field_name, type_, types_path=None): diff --git a/python/pyfory/tests/xlang_test_main.py b/python/pyfory/tests/xlang_test_main.py index c161b1c94c..f96f477c77 100644 --- a/python/pyfory/tests/xlang_test_main.py +++ b/python/pyfory/tests/xlang_test_main.py @@ -227,6 +227,7 @@ class NullableComprehensiveSchemaConsistent: @dataclass class RefInnerSchemaConsistent: """Inner struct for reference tracking test (SCHEMA_CONSISTENT mode).""" + id: pyfory.int32 = 0 name: str = "" @@ -234,13 +235,15 @@ class RefInnerSchemaConsistent: @dataclass class RefOuterSchemaConsistent: """Outer struct with two fields pointing to the same inner object (SCHEMA_CONSISTENT mode).""" - inner1: Optional[RefInnerSchemaConsistent] = None - inner2: Optional[RefInnerSchemaConsistent] = None + + inner1: Optional[RefInnerSchemaConsistent] = pyfory.field(default=None, ref=True, nullable=True) + inner2: Optional[RefInnerSchemaConsistent] = pyfory.field(default=None, ref=True, nullable=True) @dataclass class RefInnerCompatible: """Inner struct for reference tracking test (COMPATIBLE mode).""" + id: pyfory.int32 = 0 name: str = "" @@ -248,8 +251,9 @@ class RefInnerCompatible: @dataclass class RefOuterCompatible: """Outer struct with two fields pointing to the same inner object (COMPATIBLE mode).""" - inner1: Optional[RefInnerCompatible] = None - inner2: Optional[RefInnerCompatible] = None + + inner1: Optional[RefInnerCompatible] = pyfory.field(default=None, ref=True, nullable=True) + inner2: Optional[RefInnerCompatible] = pyfory.field(default=None, ref=True, nullable=True) @dataclass @@ -1154,7 +1158,7 @@ def test_ref_schema_consistent(): with open(data_file, "rb") as f: data_bytes = f.read() - fory = pyfory.Fory(xlang=True, compatible=False) + fory = pyfory.Fory(xlang=True, compatible=False, ref=True) fory.register_type(RefInnerSchemaConsistent, type_id=501) fory.register_type(RefOuterSchemaConsistent, type_id=502) @@ -1192,7 +1196,7 @@ def test_ref_compatible(): with open(data_file, "rb") as f: data_bytes = f.read() - fory = pyfory.Fory(xlang=True, compatible=True) + fory = pyfory.Fory(xlang=True, compatible=True, ref=True) fory.register_type(RefInnerCompatible, type_id=503) fory.register_type(RefOuterCompatible, type_id=504) diff --git a/rust/fory-core/src/config.rs b/rust/fory-core/src/config.rs index 696ab8e198..077a47c564 100644 --- a/rust/fory-core/src/config.rs +++ b/rust/fory-core/src/config.rs @@ -34,6 +34,10 @@ pub struct Config { pub max_dyn_depth: u32, /// Whether class version checking is enabled. pub check_struct_version: bool, + /// Whether reference tracking is enabled. + /// When enabled, shared references and circular references are tracked + /// and preserved during serialization/deserialization. + pub ref_tracking: bool, } impl Default for Config { @@ -45,6 +49,7 @@ impl Default for Config { compress_string: false, max_dyn_depth: 5, check_struct_version: false, + ref_tracking: false, } } } @@ -90,4 +95,10 @@ impl Config { pub fn is_check_struct_version(&self) -> bool { self.check_struct_version } + + /// Check if reference tracking is enabled. + #[inline(always)] + pub fn is_ref_tracking(&self) -> bool { + self.ref_tracking + } } diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index 3b941885d3..a9a3ce948a 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -260,6 +260,34 @@ impl Fory { self } + /// Enables or disables reference tracking for shared and circular references. + /// + /// # Arguments + /// + /// * `ref_tracking` - If `true`, enables reference tracking which allows + /// preserving shared object references and circular references during + /// serialization/deserialization. + /// + /// # Returns + /// + /// Returns `self` for method chaining. + /// + /// # Default + /// + /// The default value is `false`. + /// + /// # Examples + /// + /// ```rust + /// use fory_core::Fory; + /// + /// let fory = Fory::default().ref_tracking(true); + /// ``` + pub fn ref_tracking(mut self, ref_tracking: bool) -> Self { + self.config.ref_tracking = ref_tracking; + self + } + /// Sets the maximum depth for nested dynamic object serialization. /// /// # Arguments @@ -590,8 +618,10 @@ impl Fory { if context.is_compatible() { context.writer.write_i32(-1); }; - // Use RefMode::Tracking for shared ref types (Rc, Arc, RcWeak, ArcWeak) - let ref_mode = if T::fory_is_shared_ref() { + // Use RefMode based on config: + // - If ref_tracking is enabled, use RefMode::Tracking for the root object + // - Otherwise, use RefMode::NullOnly which writes NOT_NULL_VALUE_FLAG + let ref_mode = if self.config.ref_tracking { RefMode::Tracking } else { RefMode::NullOnly @@ -983,8 +1013,10 @@ impl Fory { bytes_to_skip = context.load_type_meta(meta_offset as usize)?; } } - // Use RefMode::Tracking for shared ref types (Rc, Arc, RcWeak, ArcWeak) - let ref_mode = if T::fory_is_shared_ref() { + // Use RefMode based on config: + // - If ref_tracking is enabled, use RefMode::Tracking for the root object + // - Otherwise, use RefMode::NullOnly + let ref_mode = if self.config.ref_tracking { RefMode::Tracking } else { RefMode::NullOnly diff --git a/rust/fory-core/src/resolver/ref_resolver.rs b/rust/fory-core/src/resolver/ref_resolver.rs index 4c2c804719..3367c82d27 100644 --- a/rust/fory-core/src/resolver/ref_resolver.rs +++ b/rust/fory-core/src/resolver/ref_resolver.rs @@ -131,6 +131,21 @@ impl RefWriter { } } + /// Reserve a reference ID slot without storing anything. + /// + /// This is used for xlang compatibility where ALL objects (including struct values, + /// not just Rc/Arc) participate in reference tracking. + /// + /// # Returns + /// + /// The reserved reference ID + #[inline(always)] + pub fn reserve_ref_id(&mut self) -> u32 { + let ref_id = self.next_ref_id; + self.next_ref_id += 1; + ref_id + } + /// Clear all stored references. /// /// This is useful for reusing the RefWriter for multiple serialization operations. diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index d8685bba99..a581e2cdd4 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -38,48 +38,18 @@ impl Serializer for Arc match ref_mode { RefMode::None => { // No ref flag - write inner directly - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::NullOnly => { // Only null check, no ref tracking context.writer.write_i8(RefFlag::NotNullValue as i8); - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::Tracking => { // Full ref tracking with RefWriter @@ -90,40 +60,26 @@ impl Serializer for Arc // Already written as ref - done return Ok(()); } - // First occurrence - write inner - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + // First occurrence - write type info and data + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } } } - fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { - Err(Error::not_allowed( - "Arc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data_generic(&self, context: &mut WriteContext, has_generics: bool) -> Result<(), Error> { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Arc where T is a shared ref type is not allowed for serialization.", + )); + } + T::fory_write_data_generic(&**self, context, has_generics) } - fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { - Err(Error::not_allowed( - "Arc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { @@ -149,8 +105,14 @@ impl Serializer for Arc read_arc(context, ref_mode, false, Some(typeinfo)) } - fn fory_read_data(_: &mut ReadContext) -> Result { - Err(Error::not_allowed("Arc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) + fn fory_read_data(context: &mut ReadContext) -> Result { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Arc where T is a shared ref type is not allowed for deserialization.", + )); + } + let inner = T::fory_read_data(context)?; + Ok(Arc::new(inner)) } fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { @@ -233,21 +195,10 @@ fn read_arc_inner( read_type_info: bool, typeinfo: Option>, ) -> Result { + // Read type info if needed, then read data directly + // No recursive ref handling needed since Arc only wraps allowed types if let Some(typeinfo) = typeinfo { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read_with_type_info(context, inner_ref_mode, typeinfo); - } - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read(context, inner_ref_mode, read_type_info); + return T::fory_read_with_type_info(context, RefMode::None, typeinfo); } if read_type_info { T::fory_read_type_info(context)?; diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index ab90d845d8..39832996f2 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -37,48 +37,18 @@ impl Serializer for Rc { match ref_mode { RefMode::None => { // No ref flag - write inner directly - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::NullOnly => { // Only null check, no ref tracking context.writer.write_i8(RefFlag::NotNullValue as i8); - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } RefMode::Tracking => { // Full ref tracking with RefWriter @@ -89,40 +59,26 @@ impl Serializer for Rc { // Already written as ref - done return Ok(()); } - // First occurrence - write inner - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - T::fory_write( - &**self, - context, - inner_ref_mode, - write_type_info, - has_generics, - ) - } else { - if write_type_info { - T::fory_write_type_info(context)?; - } - T::fory_write_data_generic(self, context, has_generics) + // First occurrence - write type info and data + if write_type_info { + T::fory_write_type_info(context)?; } + T::fory_write_data_generic(self, context, has_generics) } } } - fn fory_write_data_generic(&self, _: &mut WriteContext, _: bool) -> Result<(), Error> { - Err(Error::not_allowed( - "Rc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data_generic(&self, context: &mut WriteContext, has_generics: bool) -> Result<(), Error> { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Rc where T is a shared ref type is not allowed for serialization.", + )); + } + T::fory_write_data_generic(&**self, context, has_generics) } - fn fory_write_data(&self, _: &mut WriteContext) -> Result<(), Error> { - Err(Error::not_allowed( - "Rc should be written using `fory_write` to handle reference tracking properly", - )) + fn fory_write_data(&self, context: &mut WriteContext) -> Result<(), Error> { + self.fory_write_data_generic(context, false) } fn fory_write_type_info(context: &mut WriteContext) -> Result<(), Error> { @@ -148,8 +104,14 @@ impl Serializer for Rc { read_rc(context, ref_mode, false, Some(typeinfo)) } - fn fory_read_data(_: &mut ReadContext) -> Result { - Err(Error::not_allowed("Rc should be read using `fory_read/fory_read_with_type_info` to handle reference tracking properly")) + fn fory_read_data(context: &mut ReadContext) -> Result { + if T::fory_is_shared_ref() { + return Err(Error::not_allowed( + "Rc where T is a shared ref type is not allowed for deserialization.", + )); + } + let inner = T::fory_read_data(context)?; + Ok(Rc::new(inner)) } fn fory_read_type_info(context: &mut ReadContext) -> Result<(), Error> { @@ -232,21 +194,10 @@ fn read_rc_inner( read_type_info: bool, typeinfo: Option>, ) -> Result { + // Read type info if needed, then read data directly + // No recursive ref handling needed since Rc only wraps allowed types if let Some(typeinfo) = typeinfo { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read_with_type_info(context, inner_ref_mode, typeinfo); - } - if T::fory_is_shared_ref() || T::fory_is_polymorphic() { - let inner_ref_mode = if T::fory_is_shared_ref() { - RefMode::Tracking - } else { - RefMode::None - }; - return T::fory_read(context, inner_ref_mode, read_type_info); + return T::fory_read_with_type_info(context, RefMode::None, typeinfo); } if read_type_info { T::fory_read_type_info(context)?; diff --git a/rust/fory-core/src/serializer/struct_.rs b/rust/fory-core/src/serializer/struct_.rs index 7d6a2a574a..31ae7d9f5f 100644 --- a/rust/fory-core/src/serializer/struct_.rs +++ b/rust/fory-core/src/serializer/struct_.rs @@ -95,8 +95,17 @@ pub fn write( ref_mode: RefMode, write_type_info: bool, ) -> Result<(), Error> { - if ref_mode != RefMode::None { - context.writer.write_i8(RefFlag::NotNullValue as i8); + match ref_mode { + RefMode::None => {} + RefMode::NullOnly => { + context.writer.write_i8(RefFlag::NotNullValue as i8); + } + RefMode::Tracking => { + // For ref tracking mode, write RefValue flag and reserve a ref_id + // so this struct participates in reference tracking. + context.writer.write_i8(RefFlag::RefValue as i8); + context.ref_writer.reserve_ref_id(); + } } if write_type_info { T::fory_write_type_info(context)?; diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index 86c2e3eb90..eb2d72a1c7 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -405,9 +405,12 @@ pub const fn is_internal_type(type_id: u32) -> bool { ) } -/// Keep as const fn for compile time evaluation or constant folding +/// Keep as const fn for compile time evaluation or constant folding. +/// Returns true if this type needs type info written in compatible mode. +/// Only user-defined types (struct, ext, unknown) need type info. +/// Internal types (primitives, strings, collections, enums) don't need type info. #[inline(always)] -pub(crate) const fn need_to_write_type_for_field(type_id: TypeId) -> bool { +pub const fn need_to_write_type_for_field(type_id: TypeId) -> bool { matches!( type_id, TypeId::STRUCT diff --git a/rust/fory-derive/src/object/field_meta.rs b/rust/fory-derive/src/object/field_meta.rs index 1a252f5ab6..ff1a68793d 100644 --- a/rust/fory-derive/src/object/field_meta.rs +++ b/rust/fory-derive/src/object/field_meta.rs @@ -203,6 +203,11 @@ fn extract_option_inner_type(ty: &Type) -> Option { None } +/// Returns true if the outer type is Option, regardless of inner type +pub fn is_option_type(ty: &Type) -> bool { + extract_outer_type_name(ty) == "Option" +} + /// Classify a field type to determine default nullable/ref behavior pub fn classify_field_type(ty: &Type) -> FieldTypeClass { let type_name = extract_outer_type_name(ty); diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs index bc1f48123e..582f82e8a0 100644 --- a/rust/fory-derive/src/object/misc.rs +++ b/rust/fory-derive/src/object/misc.rs @@ -20,7 +20,7 @@ use quote::quote; use std::sync::atomic::{AtomicU32, Ordering}; use syn::Field; -use super::field_meta::{classify_field_type, parse_field_meta}; +use super::field_meta::{classify_field_type, is_option_type, parse_field_meta}; use super::util::{ classify_trait_object_field, generic_tree_to_tokens, get_filtered_source_fields_iter, get_sort_fields_ts, parse_generic_tree, StructField, @@ -82,7 +82,11 @@ pub fn gen_field_fields_info(source_fields: &[SourceField<'_>]) -> TokenStream { // Parse field metadata for nullable/ref tracking and field ID let meta = parse_field_meta(field).unwrap_or_default(); let type_class = classify_field_type(ty); - let nullable = meta.effective_nullable(type_class); + // For nullable, check both the classified type AND whether outer type is Option + // This handles Option> correctly - classify_field_type returns Rc for ref_tracking, + // but we also need to detect that the outer wrapper is Option for nullable. + let is_outer_option = is_option_type(ty); + let nullable = meta.effective_nullable(type_class) || is_outer_option; let ref_tracking = meta.effective_ref_tracking(type_class); // Only use explicit field ID when user sets #[fory(id = N)] // Otherwise use -1 to indicate field name encoding should be used diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index 1bf09fbc56..ed16babea8 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -23,7 +23,7 @@ use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, get_primitive_reader_method, get_struct_name, is_debug_enabled, - is_direct_primitive_numeric_type, is_primitive_type, is_skip_field, + is_direct_primitive_type, is_primitive_type, is_skip_field, should_skip_type_info_for_field, FieldRefMode, StructField, }; use crate::util::SourceField; @@ -213,7 +213,7 @@ pub fn gen_read_field(field: &Field, private_ident: &Ident, field_name: &str) -> } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info quote! { let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, true)?; } @@ -221,41 +221,51 @@ pub fn gen_read_field(field: &Field, private_ident: &Ident, field_name: &str) -> _ => { let skip_type_info = should_skip_type_info_for_field(ty); - // Check if this is a direct primitive numeric type that can use direct reader calls - if is_direct_primitive_numeric_type(ty) { + // Check if this is a direct primitive type that can use direct reader calls + // Only apply when ref_mode is None (no ref tracking needed) + if ref_mode == FieldRefMode::None && is_direct_primitive_type(ty) { let type_name = extract_type_name(ty); - let reader_method = get_primitive_reader_method(&type_name); - let reader_ident = syn::Ident::new(reader_method, proc_macro2::Span::call_site()); - quote! { - let #private_ident = context.reader.#reader_ident()?; - } - } else if skip_type_info { - // Known types (primitives, strings, collections) - skip type info at compile time - if ref_mode == FieldRefMode::None { + if type_name == "String" { + // String: call fory_read_data directly quote! { let #private_ident = <#ty as fory_core::Serializer>::fory_read_data(context)?; } } else { + // Numeric primitives: use direct buffer methods + let reader_method = get_primitive_reader_method(&type_name); + let reader_ident = syn::Ident::new(reader_method, proc_macro2::Span::call_site()); quote! { - let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, false)?; + let #private_ident = context.reader.#reader_ident()?; } } - } else { - // Custom types (struct/enum/ext) - need runtime check for enums + } else if skip_type_info { + // Known types (primitives, strings, collections) - skip type info at compile time if ref_mode == FieldRefMode::None { quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - if need_type_info { - <#ty as fory_core::Serializer>::fory_read_type_info(context)?; - } let #private_ident = <#ty as fory_core::Serializer>::fory_read_data(context)?; } } else { quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, need_type_info)?; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, false)?; } } + } else { + // Custom types (struct/enum/ext) - always need mode-dependent type info logic + // Determine read_type_info based on mode: + // - compatible=true: use need_to_write_type_for_field (struct types need type info) + // - compatible=false: use fory_is_polymorphic + // This applies regardless of ref_mode because Java always writes type info + // for struct-type fields in compatible mode, even for non-nullable fields. + quote! { + let read_type_info = if context.is_compatible() { + fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ) + } else { + <#ty as fory_core::Serializer>::fory_is_polymorphic() + }; + let #private_ident = <#ty as fory_core::Serializer>::fory_read(context, #ref_mode, read_type_info)?; + } } } }; @@ -435,14 +445,21 @@ pub(crate) fn gen_read_compatible_match_arm_body( } StructField::VecBox(_) => { // Vec> uses standard Vec deserialization with polymorphic elements - // Check nullable flag from remote field info to determine if ref flag was written + // Check nullable and ref_tracking flags from remote field info quote! { let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -450,14 +467,21 @@ pub(crate) fn gen_read_compatible_match_arm_body( } StructField::HashMapBox(_, _) => { // HashMap> uses standard HashMap deserialization with polymorphic values - // Check nullable flag from remote field info to determine if ref flag was written + // Check nullable and ref_tracking flags from remote field info quote! { let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -494,9 +518,23 @@ pub(crate) fn gen_read_compatible_match_arm_body( } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info + // Use remote field's ref_tracking flag for ref_mode quote! { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, #ref_mode, true)?); + let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( + _field.field_type.type_id, + _field.field_type.nullable, + ); + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // Forward types are polymorphic, always read type info + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, true)?); } } StructField::None => { @@ -509,8 +547,16 @@ pub(crate) fn gen_read_compatible_match_arm_body( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?); + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?); } else { #var_name = Some(<#ty as fory_core::Serializer>::fory_read_data(context)?); } @@ -521,8 +567,16 @@ pub(crate) fn gen_read_compatible_match_arm_body( _field.field_type.type_id, _field.field_type.nullable, ); - if read_ref_flag { - #var_name = <#ty as fory_core::Serializer>::fory_read(context, fory_core::RefMode::NullOnly, false)?; + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + if read_ref_flag || _field.field_type.ref_tracking { + #var_name = <#ty as fory_core::Serializer>::fory_read(context, ref_mode, false)?; } else { #var_name = <#ty as fory_core::Serializer>::fory_read_data(context)?; } @@ -530,26 +584,42 @@ pub(crate) fn gen_read_compatible_match_arm_body( } } else if dec_by_option { quote! { - let read_type_info = fory_core::serializer::util::field_need_read_type_info(_field.field_type.type_id); let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - let ref_mode = if read_ref_flag { fory_core::RefMode::NullOnly } else { fory_core::RefMode::None }; - // Always use fory_read which handles compatible mode correctly - // for nested struct types with different registered IDs + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // For ref-tracked struct types, Java writes type info after RefValue flag + let read_type_info = fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ); #var_name = Some(<#ty as fory_core::Serializer>::fory_read(context, ref_mode, read_type_info)?); } } else { quote! { - let read_type_info = fory_core::serializer::util::field_need_read_type_info(_field.field_type.type_id); let read_ref_flag = fory_core::serializer::util::field_need_write_ref_into( _field.field_type.type_id, _field.field_type.nullable, ); - let ref_mode = if read_ref_flag { fory_core::RefMode::NullOnly } else { fory_core::RefMode::None }; - // Always use fory_read which handles compatible mode correctly - // for nested struct types with different registered IDs + // Use RefMode::Tracking if remote field has ref_tracking enabled + let ref_mode = if _field.field_type.ref_tracking { + fory_core::RefMode::Tracking + } else if read_ref_flag { + fory_core::RefMode::NullOnly + } else { + fory_core::RefMode::None + }; + // For ref-tracked struct types, Java writes type info after RefValue flag + let read_type_info = fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ); #var_name = <#ty as fory_core::Serializer>::fory_read(context, ref_mode, read_type_info)?; } } @@ -593,6 +663,13 @@ pub fn gen_read(_struct_ident: &Ident) -> TokenStream { fory_core::RefFlag::NotNullValue as i8 }; if ref_flag == (fory_core::RefFlag::NotNullValue as i8) || ref_flag == (fory_core::RefFlag::RefValue as i8) { + // For RefValueFlag with Tracking mode, reserve a ref_id to participate in ref tracking. + // This is needed for xlang compatibility where all objects (not just Rc/Arc) + // participate in reference tracking when ref tracking is enabled. + // Only reserve for Tracking mode, not NullOnly mode. + if ref_flag == (fory_core::RefFlag::RefValue as i8) && ref_mode == fory_core::RefMode::Tracking { + context.ref_reader.reserve_ref_id(); + } if context.is_compatible() { let type_info = if read_type_info { context.read_any_typeinfo()? diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs index 3b1ed38ebd..be25df9b19 100644 --- a/rust/fory-derive/src/object/util.rs +++ b/rust/fory-derive/src/object/util.rs @@ -208,6 +208,9 @@ fn is_forward_field_internal(ty: &Type, struct_name: &str) -> bool { } // Check smart pointers: Rc / Arc + // Only return true if: + // 1. Inner type is Rc (polymorphic) + // 2. Inner type references the containing struct (forward reference) if seg.ident == "Rc" || seg.ident == "Arc" { if let PathArguments::AngleBracketed(args) = &seg.arguments { if let Some(GenericArgument::Type(inner_ty)) = args.args.first() { @@ -226,9 +229,10 @@ fn is_forward_field_internal(ty: &Type, struct_name: &str) -> bool { return false; } } - // Inner type is not a trait object → return true + // Inner type is not a trait object - recursively check + // if it references the containing struct _ => { - return true; + return is_forward_field_internal(inner_ty, struct_name); } } } @@ -590,6 +594,18 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode) -> TokenStream { (!PRIMITIVE_TYPE_NAMES.contains(&node.name.as_str()), node) }; + // If Rc or Arc, unwrap to inner type - these are reference wrappers + // that don't add type info to the field type (handled by ref_tracking flag) + let base_node = if base_node.name == "Rc" || base_node.name == "Arc" { + if let Some(inner) = base_node.generics.first() { + inner + } else { + base_node + } + } else { + base_node + }; + // `Vec>` rule stays as is if let Some(ts) = try_vec_of_option_primitive(base_node) { return ts; @@ -709,13 +725,18 @@ static PRIMITIVE_IO_METHODS: &[(&str, &str, &str)] = &[ ("u128", "write_u128", "read_u128"), ]; -/// Check if a type is a direct primitive numeric type (not wrapped in Option, Vec, etc.) -pub(super) fn is_direct_primitive_numeric_type(ty: &Type) -> bool { +/// Check if a type is a direct primitive type (numeric or String, not wrapped in Option, Vec, etc.) +pub(super) fn is_direct_primitive_type(ty: &Type) -> bool { if let Type::Path(type_path) = ty { if let Some(seg) = type_path.path.segments.last() { // Check if it's a simple type path without generics if matches!(seg.arguments, PathArguments::None) { let type_name = seg.ident.to_string(); + // Check for String type + if type_name == "String" { + return true; + } + // Check for numeric primitive types return PRIMITIVE_IO_METHODS .iter() .any(|(name, _, _)| *name == type_name.as_str()); diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 0ebac74fdc..1ab0b3c7d2 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -19,8 +19,8 @@ use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, get_field_accessor, get_field_name, get_filtered_source_fields_iter, get_primitive_writer_method, get_struct_name, - get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_numeric_type, - should_skip_type_info_for_field, FieldRefMode, StructField, + get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_type, FieldRefMode, + StructField, }; use crate::util::SourceField; use fory_core::types::TypeId; @@ -242,30 +242,38 @@ fn gen_write_field_impl( } } StructField::Forward => { - // Forward types - respect field meta for ref mode + // Forward types (trait objects, forward references) - polymorphic, always need type info quote! { <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, true, false)?; } } _ => { - let skip_type_info = should_skip_type_info_for_field(ty); let type_id = get_type_id_by_type_ast(ty); - // Check if this is a direct primitive numeric type that can use direct writer calls - if is_direct_primitive_numeric_type(ty) { + // Check if this is a direct primitive type that can use direct writer calls + // Only apply when ref_mode is None (no ref tracking needed) + if ref_mode == FieldRefMode::None && is_direct_primitive_type(ty) { let type_name = extract_type_name(ty); - let writer_method = get_primitive_writer_method(&type_name); - let writer_ident = syn::Ident::new(writer_method, proc_macro2::Span::call_site()); - // For primitives: - // - use_self=true: #value_ts is `self.field`, which is T (copy happens automatically) - // - use_self=false: #value_ts is `field` from pattern match on &self, which is &T - let value_expr = if use_self { - quote! { #value_ts } + if type_name == "String" { + // String: call fory_write_data directly + quote! { + <#ty as fory_core::Serializer>::fory_write_data(&#value_ts, context)?; + } } else { - quote! { *#value_ts } - }; - quote! { - context.writer.#writer_ident(#value_expr); + // Numeric primitives: use direct buffer methods + let writer_method = get_primitive_writer_method(&type_name); + let writer_ident = syn::Ident::new(writer_method, proc_macro2::Span::call_site()); + // For primitives: + // - use_self=true: #value_ts is `self.field`, which is T (copy happens automatically) + // - use_self=false: #value_ts is `field` from pattern match on &self, which is &T + let value_expr = if use_self { + quote! { #value_ts } + } else { + quote! { *#value_ts } + }; + quote! { + context.writer.#writer_ident(#value_expr); + } } } else if type_id == TypeId::LIST as u32 || type_id == TypeId::SET as u32 @@ -282,24 +290,21 @@ fn gen_write_field_impl( } } } else { - // Known types (primitives, strings, collections) - skip type info at compile time - // For custom types that we can't determine at compile time (like enums), - // we need to check at runtime whether to skip type info - if skip_type_info { - if ref_mode == FieldRefMode::None { - quote! { - <#ty as fory_core::Serializer>::fory_write_data(&#value_ts, context)?; - } + // Custom types (struct/enum/ext) - always need mode-dependent type info logic + // Determine write_type_info based on mode: + // - compatible=true: use need_to_write_type_for_field (struct types need type info) + // - compatible=false: use fory_is_polymorphic + // This applies regardless of ref_mode because Java always writes type info + // for struct-type fields in compatible mode, even for non-nullable fields. + quote! { + let write_type_info = if context.is_compatible() { + fory_core::types::need_to_write_type_for_field( + <#ty as fory_core::Serializer>::fory_static_type_id() + ) } else { - quote! { - <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, false, false)?; - } - } - } else { - quote! { - let need_type_info = fory_core::serializer::util::field_need_write_type_info(<#ty as fory_core::Serializer>::fory_static_type_id()); - <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, need_type_info, false)?; - } + <#ty as fory_core::Serializer>::fory_is_polymorphic() + }; + <#ty as fory_core::Serializer>::fory_write(&#value_ts, context, #ref_mode, write_type_info, false)?; } } } diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index f0ff1738b8..52139199fd 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1643,6 +1643,8 @@ fn test_union_xlang() { // Reference Tracking Tests - Cross-language shared reference tests // ============================================================================ +use std::rc::Rc; + /// Inner struct for reference tracking test (SCHEMA_CONSISTENT mode) /// Matches Java RefInnerSchemaConsistent with type ID 501 #[derive(ForyObject, Debug, PartialEq, Clone)] @@ -1654,12 +1656,11 @@ struct RefInnerSchemaConsistent { /// Outer struct for reference tracking test (SCHEMA_CONSISTENT mode) /// Contains two fields that both point to the same inner object. /// Matches Java RefOuterSchemaConsistent with type ID 502 +/// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking #[derive(ForyObject, Debug, PartialEq)] struct RefOuterSchemaConsistent { - #[fory(ref = true, nullable = true)] - inner1: Option, - #[fory(ref = true, nullable = true)] - inner2: Option, + inner1: Option>, + inner2: Option>, } /// Inner struct for reference tracking test (COMPATIBLE mode) @@ -1673,12 +1674,11 @@ struct RefInnerCompatible { /// Outer struct for reference tracking test (COMPATIBLE mode) /// Contains two fields that both point to the same inner object. /// Matches Java RefOuterCompatible with type ID 504 +/// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking #[derive(ForyObject, Debug, PartialEq)] struct RefOuterCompatible { - #[fory(ref = true, nullable = true)] - inner1: Option, - #[fory(ref = true, nullable = true)] - inner2: Option, + inner1: Option>, + inner2: Option>, } /// Test cross-language reference tracking in SCHEMA_CONSISTENT mode (compatible=false). @@ -1692,7 +1692,7 @@ fn test_ref_schema_consistent() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().compatible(false).xlang(true); + let mut fory = Fory::default().compatible(false).xlang(true).ref_tracking(true); fory.register::(501).unwrap(); fory.register::(502).unwrap(); @@ -1707,7 +1707,11 @@ fn test_ref_schema_consistent() { let inner2 = outer.inner2.as_ref().unwrap(); assert_eq!(inner1.id, 42); assert_eq!(inner1.name, "shared_inner"); - assert_eq!(inner1, inner2, "inner1 and inner2 should be equal (same reference)"); + // Compare the values (Rc contents) + assert_eq!(inner1.as_ref(), inner2.as_ref(), "inner1 and inner2 should have equal values"); + + // With Rc, after deserialization with ref tracking, both fields should point to the same Rc + assert!(Rc::ptr_eq(inner1, inner2), "inner1 and inner2 should be the same Rc (reference identity)"); // Re-serialize and write back let new_bytes = fory.serialize(&outer).unwrap(); @@ -1725,7 +1729,7 @@ fn test_ref_compatible() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().compatible(true).xlang(true); + let mut fory = Fory::default().compatible(true).xlang(true).ref_tracking(true); fory.register::(503).unwrap(); fory.register::(504).unwrap(); @@ -1740,7 +1744,11 @@ fn test_ref_compatible() { let inner2 = outer.inner2.as_ref().unwrap(); assert_eq!(inner1.id, 99); assert_eq!(inner1.name, "compatible_shared"); - assert_eq!(inner1, inner2, "inner1 and inner2 should be equal (same reference)"); + // Compare the values (Rc contents) + assert_eq!(inner1.as_ref(), inner2.as_ref(), "inner1 and inner2 should have equal values"); + + // With Rc, after deserialization with ref tracking, both fields should point to the same Rc + assert!(Rc::ptr_eq(inner1, inner2), "inner1 and inner2 should be the same Rc (reference identity)"); // Re-serialize and write back let new_bytes = fory.serialize(&outer).unwrap(); diff --git a/rust/tests/tests/test_rc_arc.rs b/rust/tests/tests/test_rc_arc.rs index 135d14f2bd..55762ce280 100644 --- a/rust/tests/tests/test_rc_arc.rs +++ b/rust/tests/tests/test_rc_arc.rs @@ -18,10 +18,17 @@ //! Tests for Rc and Arc serialization support in Fory use fory_core::fory::Fory; +use fory_derive::ForyObject; use std::collections::HashMap; use std::rc::Rc; use std::sync::Arc; +/// A simple struct for testing nested Rc/Arc serialization +#[derive(ForyObject, Debug, Clone, PartialEq, Default)] +struct NestedData { + value: String, +} + #[test] fn test_rc_string_serialization() { let fory = Fory::default(); @@ -166,16 +173,19 @@ fn test_mixed_rc_arc_serialization() { #[test] fn test_nested_rc_arc() { - let fory = Fory::default(); + let mut fory = Fory::default(); + fory.register::(100).unwrap(); - // Test Rc containing Arc - let inner_data = Arc::new(String::from("nested")); + // Test Rc containing Arc with allowed struct type + let inner_data = Arc::new(NestedData { + value: String::from("nested"), + }); let outer_data = Rc::new(inner_data.clone()); let serialized = fory.serialize(&outer_data).unwrap(); - let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); + let deserialized: Rc> = fory.deserialize(&serialized).unwrap(); - assert_eq!(**outer_data, **deserialized); + assert_eq!(outer_data.value, deserialized.value); } #[test] From aad87a1b416636d63b4f2177099c66954916f367 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 20:18:45 +0800 Subject: [PATCH 06/35] refactor xlang final checks --- .../src/main/java/org/apache/fory/Fory.java | 96 ++++++------------- .../fory/builder/BaseObjectCodecBuilder.java | 37 ++++--- .../fory/builder/MetaSharedCodecBuilder.java | 5 +- .../fory/builder/ObjectCodecBuilder.java | 11 +-- .../apache/fory/resolver/TypeResolver.java | 15 +++ .../apache/fory/resolver/XtypeResolver.java | 14 +-- .../serializer/AbstractObjectSerializer.java | 9 +- .../fory/serializer/MetaSharedSerializer.java | 5 +- .../fory/serializer/ObjectSerializer.java | 5 +- .../fory/serializer/SerializationBinding.java | 32 +------ .../apache/fory/type/DescriptorGrouper.java | 8 +- .../java/org/apache/fory/XlangTestBase.java | 4 +- .../fory/type/DescriptorGrouperTest.java | 4 +- .../java/org/apache/fory/test/TestUtils.java | 16 +--- rust/tests/tests/test_cross_language.rs | 8 ++ 15 files changed, 101 insertions(+), 168 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index de8cf0cfea..2037bea7de 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -484,44 +484,6 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { } } - /** Write object class and data without tracking ref. */ - public void writeNullable(MemoryBuffer buffer, Object obj) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj); - } - } - - public void writeNullable(MemoryBuffer buffer, Object obj, Serializer serializer) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - serializer.write(buffer, obj); - } - } - - /** Write object class and data without tracking ref. */ - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - - public void writeNullable(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - if (obj == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - writeNonRef(buffer, obj, classInfo); - } - } - /** * Serialize a not-null and non-reference object to buffer. * @@ -543,42 +505,30 @@ public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { } public void xwriteRef(MemoryBuffer buffer, Object obj) { - int posBeforeRef = buffer.writerIndex(); if (!refResolver.writeRefOrNull(buffer, obj)) { - int posAfterRef = buffer.writerIndex(); ClassInfo classInfo = xtypeResolver.writeClassInfo(buffer, obj); - int posAfterTypeInfo = buffer.writerIndex(); - if (config.isForyDebugOutputEnabled()) { - LOG.info( - "[Java][fory-debug] xwriteRef(root) for {} at pos {}: refFlag pos {}, typeInfo pos {}, calling xwriteData", - obj.getClass().getSimpleName(), - posBeforeRef, - posAfterRef, - posAfterTypeInfo); - } xwriteData(buffer, classInfo, obj); } } - public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { - if (config.isForyDebugOutputEnabled()) { - LOG.info( - "[Java][fory-debug] xwriteRef(with-serializer) for {} at pos {}, needToWriteRef={}", - obj != null ? obj.getClass().getSimpleName() : "null", - buffer.writerIndex(), - serializer.needToWriteRef()); + public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { + if (!refResolver.writeRefOrNull(buffer, obj)) { + ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder); + xtypeResolver.writeClassInfo(buffer, obj); + xwriteData(buffer, classInfo, obj); + } + } + + public void xwriteRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { + if (!refResolver.writeRefOrNull(buffer, obj)) { + xtypeResolver.writeClassInfo(buffer, obj); + xwriteData(buffer, classInfo, obj); } + } + + public void xwriteRef(MemoryBuffer buffer, T obj, Serializer serializer) { if (serializer.needToWriteRef()) { - int posBeforeRef = buffer.writerIndex(); if (!refResolver.writeRefOrNull(buffer, obj)) { - int posAfterRef = buffer.writerIndex(); - if (config.isForyDebugOutputEnabled()) { - LOG.info( - "[Java][fory-debug] xwriteRef(with-serializer) for {} at pos {}: refFlag written, pos now {}, calling xwrite", - obj.getClass().getSimpleName(), - posBeforeRef, - posAfterRef); - } depth++; serializer.xwrite(buffer, obj); depth--; @@ -618,14 +568,11 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { break; case Types.INT32: case Types.VAR_INT32: - // TODO(chaokunyang) support other encoding buffer.writeVarInt32((Integer) obj); break; case Types.INT64: case Types.VAR_INT64: - // TODO(chaokunyang) support other encoding case Types.SLI_INT64: - // TODO(chaokunyang) support varint encoding buffer.writeVarInt64((Long) obj); break; case Types.FLOAT32: @@ -1091,6 +1038,19 @@ public Object xreadRef(MemoryBuffer buffer) { } } + public Object xreadRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + RefResolver refResolver = this.refResolver; + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { + // ref value or not-null value + Object o = readDataInternal(buffer, xtypeResolver.readClassInfo(buffer, classInfoHolder)); + refResolver.setReadObject(nextReadRefId, o); + return o; + } else { + return refResolver.getReadObject(); + } + } + public Object xreadRef(MemoryBuffer buffer, Serializer serializer) { if (serializer.needToWriteRef()) { RefResolver refResolver = this.refResolver; diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 914a7ced48..794e7af9fb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -411,23 +411,23 @@ protected Expression serializeField( if (useRefTracking) { return new If( not(writeRefOrNull(buffer, fieldValue)), - serializeForNotNullForField(fieldValue, buffer, typeRef, null, false)); + serializeForNotNullForField(fieldValue, buffer, descriptor, null, false)); } else { // if typeToken is not final, ref tracking of subclass will be ignored too. if (typeRef.isPrimitive()) { - return serializeForNotNullForField(fieldValue, buffer, typeRef, null, false); + return serializeForNotNullForField(fieldValue, buffer, descriptor, null, false); } if (nullable) { Expression action = new ListExpression( new Invoke(buffer, "writeByte", Literal.ofByte(Fory.NOT_NULL_VALUE_FLAG)), - serializeForNotNullForField(fieldValue, buffer, typeRef, null, false)); + serializeForNotNullForField(fieldValue, buffer, descriptor, null, false)); return new If( eqNull(fieldValue), new Invoke(buffer, "writeByte", Literal.ofByte(Fory.NULL_FLAG)), action); } else { - return serializeForNotNullForField(fieldValue, buffer, typeRef, null, false); + return serializeForNotNullForField(fieldValue, buffer, descriptor, null, false); } } } @@ -435,9 +435,10 @@ protected Expression serializeField( private Expression serializeForNotNullForField( Expression inputObject, Expression buffer, - TypeRef typeRef, + Descriptor descriptor, Expression serializer, boolean generateNewMethod) { + TypeRef typeRef = descriptor.getTypeRef(); Class clz = getRawType(typeRef); if (isPrimitive(clz) || isBoxed(clz)) { return serializePrimitive(inputObject, buffer, clz); @@ -452,7 +453,7 @@ private Expression serializeForNotNullForField( } else if (useMapSerialization(typeRef)) { action = serializeForMap(buffer, inputObject, typeRef, serializer, generateNewMethod); } else { - action = serializeForNotNullObjectForField(inputObject, buffer, typeRef, serializer); + action = serializeForNotNullObjectForField(inputObject, buffer, descriptor, serializer); } return action; } @@ -483,12 +484,13 @@ private Expression serializePrimitive(Expression inputObject, Expression buffer, } private Expression serializeForNotNullObjectForField( - Expression inputObject, Expression buffer, TypeRef typeRef, Expression serializer) { + Expression inputObject, Expression buffer, Descriptor descriptor, Expression serializer) { + TypeRef typeRef = descriptor.getTypeRef(); Class clz = getRawType(typeRef); if (serializer != null) { return new Invoke(serializer, writeMethodName, buffer, inputObject); } - if (isMonomorphic(clz)) { + if (isMonomorphic(descriptor)) { // Use descriptor to get the appropriate serializer serializer = getSerializerForField(clz); return new Invoke(serializer, writeMethodName, buffer, inputObject); @@ -613,7 +615,13 @@ protected boolean useMapSerialization(Class type) { * the method can still return false. For example, we return false in meta share mode to write * class defs for the non-inner final types. */ - protected abstract boolean isMonomorphic(Class clz); + protected boolean isMonomorphic(Class clz) { + return typeResolver(r -> r.isMonomorphic(clz)); + } + + protected boolean isMonomorphic(Descriptor descriptor) { + return typeResolver(r -> r.isMonomorphic(descriptor)); + } protected boolean isMonomorphic(TypeRef typeRef) { return isMonomorphic(typeRef.getRawType()); @@ -1803,10 +1811,10 @@ protected Expression deserializeField( boolean typeNeedsRef = needWriteRef(typeRef); if (useRefTracking) { - return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, typeRef, null)); + return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (!nullable) { - Expression value = deserializeForNotNullForField(buffer, typeRef, null); + Expression value = deserializeForNotNullForField(buffer, descriptor, null); if (typeNeedsRef) { // When a field explicitly disables ref tracking (@ForyField(trackingRef=false)) @@ -1828,7 +1836,7 @@ protected Expression deserializeField( buffer, typeRef, callback, - () -> deserializeForNotNullForField(buffer, typeRef, null), + () -> deserializeForNotNullForField(buffer, descriptor, null), true, localFieldType); @@ -1842,7 +1850,8 @@ protected Expression deserializeField( } private Expression deserializeForNotNullForField( - Expression buffer, TypeRef typeRef, Expression serializer) { + Expression buffer, Descriptor descriptor, Expression serializer) { + TypeRef typeRef = descriptor.getTypeRef(); Class cls = getRawType(typeRef); if (isPrimitive(cls) || isBoxed(cls)) { return deserializePrimitive(buffer, cls); @@ -1859,7 +1868,7 @@ private Expression deserializeForNotNullForField( if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); } - if (isMonomorphic(cls)) { + if (isMonomorphic(descriptor)) { // Use descriptor to get the appropriate serializer serializer = getSerializerForField(cls); Class returnType = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index ada1a19fe9..e88e4b4807 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -102,12 +102,11 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) LOG.info("========== sorted descriptors for {} ==========", classDef.getClassName()); for (Descriptor d : sortedDescriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } objectCodecOptimizer = diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 6e999d54d4..4cf0fb79a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -111,12 +111,11 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { List sortedDescriptors = grouper.getSortedDescriptors(); for (Descriptor d : sortedDescriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } classVersionHash = @@ -155,12 +154,6 @@ protected void addCommonImports() { ctx.addImport(Generated.GeneratedObjectSerializer.class); } - /** Mark non-inner registered final types as non-final to write class def for those types. */ - @Override - protected boolean isMonomorphic(Class clz) { - return typeResolver(r -> r.isMonomorphic(clz)); - } - /** * Return an expression that serialize java bean of type {@link CodecBuilder#beanClass} to buffer. */ diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 24600e1695..67a8d133e3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -223,6 +223,21 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract boolean isRegisteredByName(Class cls); + public boolean isMonomorphic(Descriptor descriptor) { + ForyField foryField = descriptor.getForyField(); + if (foryField != null) { + switch (foryField.morphic()) { + case POLYMORPHIC: + return false; + case FINAL: + return true; + default: + return isMonomorphic(descriptor.getRawType()); + } + } + return isMonomorphic(descriptor.getRawType()); + } + public abstract boolean isMonomorphic(Class clz); public abstract ClassInfo getClassInfo(Class cls); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 08abaa2011..476e904590 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -920,19 +920,7 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - clz -> { - ClassInfo classInfo = getClassInfo(clz, false); - if (classInfo == null || clz.isEnum()) { - return false; - } - byte foryTypeId = (byte) (classInfo.xtypeId & 0xff); - if (foryTypeId == 0 - || foryTypeId == Types.UNKNOWN - || Types.isUserDefinedType(foryTypeId)) { - return false; - } - return foryTypeId != Types.LIST && foryTypeId != Types.SET && foryTypeId != Types.MAP; - }, + this::isMonomorphic, descriptors, descriptorsGroupedOrdered, descriptorUpdator, diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index b11e071ed7..6038d51f89 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -1091,15 +1091,14 @@ protected T newBean() { for (Descriptor d : boxed) { finalFields[cnt++] = new FinalTypeField(fory, d); } - // TODO(chaokunyang) Support Pojo generics besides Map/Collection subclass - // when it's supported in BaseObjectCodecBuilder. for (Descriptor d : finals) { finalFields[cnt++] = new FinalTypeField(fory, d); } boolean[] isFinal = new boolean[finalFields.length]; for (int i = 0; i < isFinal.length; i++) { - ClassInfo classInfo = finalFields[i].classInfo; - isFinal[i] = classInfo != null && fory.getClassResolver().isMonomorphic(classInfo.getCls()); + FinalTypeField finalField = finalFields[i]; + ClassInfo classInfo = finalField.classInfo; + isFinal[i] = classInfo != null && fory.getClassResolver().isMonomorphic(finalField.descriptor); } cnt = 0; GenericTypeField[] otherFields = new GenericTypeField[grouper.getOtherDescriptors().size()]; @@ -1121,6 +1120,7 @@ protected T newBean() { } public static class InternalFieldInfo { + protected final Descriptor descriptor; protected final TypeRef typeRef; protected final short classId; protected final String qualifiedFieldName; @@ -1132,6 +1132,7 @@ public static class InternalFieldInfo { protected final boolean isPrimitive; private InternalFieldInfo(Fory fory, Descriptor d, short classId) { + this.descriptor = d; this.typeRef = d.getTypeRef(); this.classId = classId; this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 8b8bfdf925..8a4c539517 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -109,12 +109,11 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { "========== MetaSharedSerializer sorted descriptors for {} ==========", type.getName()); for (Descriptor d : descriptorGrouper.getSortedDescriptors()) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } // d.getField() may be null if not exists in this class when meta share enabled. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 179ed65674..82a568c061 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -116,12 +116,11 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { LOG.info("========== ObjectSerializer sorted descriptors for {} ==========", cls.getName()); for (Descriptor d : descriptors) { LOG.info( - " {} -> {}, ref {}, nullable {}, morphic {}", + " {} -> {}, ref {}, nullable {}", d.getName(), d.getTypeName(), d.isTrackingRef(), - d.isNullable(), - d.isFinalField()); + d.isNullable()); } } if (isRecord) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 5e6a609565..2309e3a895 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -341,27 +341,12 @@ public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder) { - // In COMPATIBLE mode (meta share enabled): write type info for schema evolution - // In SCHEMA_CONSISTENT mode: don't write type info, use serializer directly - if (fory.getConfig().isMetaShareEnabled()) { - fory.xwriteRef(buffer, obj); - } else { - // SCHEMA_CONSISTENT mode: resolve serializer and write without type info - ClassInfo classInfo = xtypeResolver.getClassInfo(obj.getClass(), classInfoHolder); - fory.xwriteRef(buffer, obj, classInfo.getSerializer()); - } + fory.xwriteRef(buffer, obj, classInfoHolder); } @Override public void writeRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - // In COMPATIBLE mode (meta share enabled): write type info for schema evolution - // In SCHEMA_CONSISTENT mode: don't write type info, use serializer directly - if (fory.getConfig().isMetaShareEnabled()) { - fory.xwriteRef(buffer, obj); - } else { - // SCHEMA_CONSISTENT mode: use provided classInfo's serializer - fory.xwriteRef(buffer, obj, classInfo.getSerializer()); - } + fory.xwriteRef(buffer, obj, classInfo); } @Override @@ -394,18 +379,7 @@ public Object readRef(MemoryBuffer buffer, GenericTypeField field) { @Override public Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - // In COMPATIBLE mode (meta share enabled): read type info for schema evolution - // In SCHEMA_CONSISTENT mode: don't read type info, use serializer directly - if (fory.getConfig().isMetaShareEnabled()) { - return fory.xreadRef(buffer); - } else { - // SCHEMA_CONSISTENT mode: resolve serializer and read without type info - ClassInfo classInfo = classInfoHolder.classInfo; - if (classInfo.getSerializer() == null) { - classInfo = xtypeResolver.getClassInfo(classInfo.getCls(), classInfoHolder); - } - return fory.xreadRef(buffer, classInfo.getSerializer()); - } + return fory.xreadRef(buffer, classInfoHolder); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java index f7e86d91df..dbbeb06338 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java @@ -82,7 +82,7 @@ public static String getFieldSortKey(Descriptor descriptor) { return c; }; private final Collection descriptors; - private final Predicate> isMonomorphic; + private final Predicate isMonomorphic; private final Function descriptorUpdater; private final boolean descriptorsGroupedOrdered; private boolean sorted = false; @@ -181,7 +181,7 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole * @param comparator comparator for non-primitive fields. */ private DescriptorGrouper( - Predicate> isMonomorphic, + Predicate isMonomorphic, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdater, @@ -232,7 +232,7 @@ public DescriptorGrouper sort() { collectionDescriptors.add(descriptorUpdater.apply(descriptor)); } else if (TypeUtils.isMap(descriptor.getRawType())) { mapDescriptors.add(descriptorUpdater.apply(descriptor)); - } else if (isMonomorphic.test(descriptor.getRawType())) { + } else if (isMonomorphic.test(descriptor)) { finalDescriptors.add(descriptorUpdater.apply(descriptor)); } else { otherDescriptors.add(descriptorUpdater.apply(descriptor)); @@ -297,7 +297,7 @@ private static Descriptor createDescriptor(Descriptor d) { } public static DescriptorGrouper createDescriptorGrouper( - Predicate> isMonomorphic, + Predicate isMonomorphic, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdator, diff --git a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java index f12fbcd682..be2fa72ab5 100644 --- a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java @@ -2204,10 +2204,10 @@ static class RefInnerSchemaConsistent { */ @Data static class RefOuterSchemaConsistent { - @ForyField(ref = true, nullable = true) + @ForyField(ref = true, nullable = true, morphic = ForyField.Morphic.FINAL) RefInnerSchemaConsistent inner1; - @ForyField(ref = true, nullable = true) + @ForyField(ref = true, nullable = true, morphic = ForyField.Morphic.FINAL) RefInnerSchemaConsistent inner2; } diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index 58fc6555d2..487680d20f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -178,7 +178,7 @@ public void testGrouper() { new TypeRef>() {}, "c" + index++, -1, "TestClass", false)); DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, + d -> ReflectionUtils.isMonomorphic(d.getRawType()), descriptors, false, null, @@ -262,7 +262,7 @@ public void testGrouper() { public void testCompressedPrimitiveGrouper() { DescriptorGrouper grouper = DescriptorGrouper.createDescriptorGrouper( - ReflectionUtils::isMonomorphic, + d -> ReflectionUtils.isMonomorphic(d.getRawType()), createDescriptors(), false, null, diff --git a/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java b/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java index 7970de4f24..7548fd4375 100644 --- a/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java +++ b/java/fory-test-core/src/main/java/org/apache/fory/test/TestUtils.java @@ -19,10 +19,8 @@ package org.apache.fory.test; -import java.io.BufferedReader; import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -64,23 +62,13 @@ private static ProcessBuilder buildProcess( private static boolean executeCommand( ProcessBuilder processBuilder, List command, int waitTimeoutSeconds) { try { + processBuilder.inheritIO(); Process process = processBuilder.start(); - // Capture output to log - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - BufferedReader errorReader = - new BufferedReader(new InputStreamReader(process.getErrorStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - while ((line = errorReader.readLine()) != null) { - System.err.println(line); - } boolean finished = process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); if (finished) { return process.exitValue() == 0; } else { - process.destroy(); // ensure the process is terminated + process.destroy(); return false; } } catch (Exception e) { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 52139199fd..7b83e7d35a 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -52,6 +52,7 @@ struct Item { } #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct SimpleStruct { // field_order != sorted_order f1: HashMap, @@ -527,6 +528,7 @@ fn test_integer() { // - Java Integer fields (with nullable=false) -> Rust i32 (no ref flag) // All fields use i32 because Java xlang mode defaults to nullable=false for all non-primitives #[derive(ForyObject, Debug, PartialEq)] + #[fory(debug)] struct Item2 { f1: i32, f2: i32, @@ -620,6 +622,7 @@ impl ForyDefault for MyExt { } } #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct MyWrapper { color: Color, my_struct: MyStruct, @@ -1264,6 +1267,7 @@ fn test_enum_schema_evolution_compatible_reverse() { /// - Nullable fields (first half - boxed numeric types): Integer, Long, Float /// - Nullable fields (second half - @ForyField): Double, Boolean, String, List, Set, Map #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct NullableComprehensiveSchemaConsistent { // Base non-nullable primitive fields byte_field: i8, @@ -1310,6 +1314,7 @@ struct NullableComprehensiveSchemaConsistent { /// /// This tests that compatible mode properly handles schema differences across languages. #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct NullableComprehensiveCompatible { // Group 1: Nullable in Rust, Non-nullable in Java // Primitive fields @@ -1658,6 +1663,7 @@ struct RefInnerSchemaConsistent { /// Matches Java RefOuterSchemaConsistent with type ID 502 /// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct RefOuterSchemaConsistent { inner1: Option>, inner2: Option>, @@ -1666,6 +1672,7 @@ struct RefOuterSchemaConsistent { /// Inner struct for reference tracking test (COMPATIBLE mode) /// Matches Java RefInnerCompatible with type ID 503 #[derive(ForyObject, Debug, PartialEq, Clone)] +#[fory(debug)] struct RefInnerCompatible { id: i32, name: String, @@ -1676,6 +1683,7 @@ struct RefInnerCompatible { /// Matches Java RefOuterCompatible with type ID 504 /// Uses Option> for nullable reference-tracked fields - Rc enables reference tracking #[derive(ForyObject, Debug, PartialEq)] +#[fory(debug)] struct RefOuterCompatible { inner1: Option>, inner2: Option>, From 318b0c881d053157442e8a7331f39b77605ae162 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 22:26:18 +0800 Subject: [PATCH 07/35] refactor java fields read/write --- .../src/main/java/org/apache/fory/Fory.java | 17 + .../org/apache/fory/annotation/ForyField.java | 5 +- .../fory/builder/BaseObjectCodecBuilder.java | 3 +- .../fory/builder/ObjectCodecBuilder.java | 2 +- .../fory/builder/ObjectCodecOptimizer.java | 4 +- .../org/apache/fory/meta/ClassDefEncoder.java | 2 +- .../apache/fory/resolver/ClassResolver.java | 6 +- .../apache/fory/resolver/TypeResolver.java | 2 + .../apache/fory/resolver/XtypeResolver.java | 14 +- .../serializer/AbstractObjectSerializer.java | 675 +++++++----------- .../apache/fory/serializer/FieldGroups.java | 238 ++++++ .../serializer/MetaSharedLayerSerializer.java | 161 ++--- .../fory/serializer/MetaSharedSerializer.java | 81 +-- .../NonexistentClassSerializers.java | 62 +- .../fory/serializer/ObjectSerializer.java | 168 +---- .../fory/serializer/SerializationBinding.java | 82 ++- .../collection/ChildContainerSerializers.java | 11 +- .../apache/fory/type/DescriptorGrouper.java | 28 +- .../fory-core/native-image.properties | 5 +- .../fory/type/DescriptorGrouperTest.java | 2 +- 20 files changed, 767 insertions(+), 801 deletions(-) create mode 100644 java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index 2037bea7de..2126a51731 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -496,6 +496,12 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { writeData(buffer, classInfo, obj); } + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + depth++; + serializer.write(buffer, obj); + depth--; + } + public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { classResolver.writeClassInfo(buffer, classInfo); Serializer serializer = classInfo.getSerializer(); @@ -555,6 +561,13 @@ public void xwriteNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { xwriteData(buffer, classInfo, obj); } + public void xwriteNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + depth++; + serializer.xwrite(buffer, obj); + depth--; + ; + } + public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) { switch (classInfo.getXtypeId()) { case Types.BOOL: @@ -957,6 +970,10 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { return readDataInternal(buffer, classResolver.readClassInfo(buffer, classInfoHolder)); } + public Object readNonRef(MemoryBuffer buffer, ClassInfo classInfo) { + return readDataInternal(buffer, classInfo); + } + /** Read object class and data without tracking ref. */ public Object readNullable(MemoryBuffer buffer) { byte headFlag = buffer.readByte(); diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java index d27206dabb..11c0976439 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java @@ -34,8 +34,9 @@ enum Morphic { * Auto-detect based on declared type (default): * *

    - *
  • Interface/abstract class: treated as POLYMORPHIC (type info written) - *
  • Concrete class: treated as FINAL (no type info written) + *
  • Xlang mode: only interface/abstract class are treated as POLYMORPHIC, concrete classes + * are treated as FINAL (no type info written) + *
  • Java native mode: all classes without {@code final} modifier are treated as POLYMORPHIC *
*/ AUTO, diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 794e7af9fb..90b92a6635 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1811,7 +1811,8 @@ protected Expression deserializeField( boolean typeNeedsRef = needWriteRef(typeRef); if (useRefTracking) { - return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); + return readRef( + buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (!nullable) { Expression value = deserializeForNotNullForField(buffer, descriptor, null); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 4cf0fb79a3..13b317f2fa 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -604,7 +604,7 @@ private Expression checkClassVersion(Expression buffer) { "checkClassVersion", PRIMITIVE_VOID_TYPE, false, - foryRef, + beanClassExpr(), inlineInvoke(buffer, readIntFunc(), PRIMITIVE_INT_TYPE), Objects.requireNonNull(classVersionHash)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java index 992370f759..01264fffa9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java @@ -117,9 +117,9 @@ private void buildGroups() { boxedReadWeight, boxedReadGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getFinalDescriptors()), 9, finalWriteGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 9, finalWriteGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getFinalDescriptors()), 5, finalReadGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 5, finalReadGroups), MutableTuple3.of( new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 4, otherReadGroups), MutableTuple3.of( diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java index b3f062a1cc..8b057c5e3f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDefEncoder.java @@ -74,7 +74,7 @@ static List buildFields(Fory fory, Class cls, boolean resolveParent) { .getBoxedDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); descriptorGrouper - .getFinalDescriptors() + .getBuildInDescriptors() .forEach(descriptor -> fields.add(descriptor.getField())); descriptorGrouper .getOtherDescriptors() diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 95f90637f0..5ffc874360 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -717,6 +717,10 @@ public boolean isMonomorphic(Class clz) { return ReflectionUtils.isMonomorphic(clz); } + public boolean isBuildIn(Descriptor descriptor) { + return isMonomorphic(descriptor); + } + /** Returns true if cls is fory inner registered class. */ boolean isInnerClass(Class cls) { Short classId = extRegistry.registeredClassIdMap.get(cls); @@ -1930,7 +1934,7 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - fory.getClassResolver()::isMonomorphic, + this::isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator, diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index 67a8d133e3..b7b1cb4d61 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -223,6 +223,8 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract boolean isRegisteredByName(Class cls); + public abstract boolean isBuildIn(Descriptor descriptor); + public boolean isMonomorphic(Descriptor descriptor) { ForyField foryField = descriptor.getForyField(); if (foryField != null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index 476e904590..f213c0c0b0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -459,6 +459,12 @@ public boolean isMonomorphic(Class clz) { return false; } + public boolean isBuildIn(Descriptor descriptor) { + Class rawType = descriptor.getRawType(); + byte xtypeId = getXtypeId(rawType); + return !Types.isUserDefinedType(xtypeId); + } + @Override public ClassInfo getClassInfo(Class cls) { ClassInfo classInfo = classInfoMap.get(cls); @@ -920,7 +926,7 @@ public DescriptorGrouper createDescriptorGrouper( boolean descriptorsGroupedOrdered, Function descriptorUpdator) { return DescriptorGrouper.createDescriptorGrouper( - this::isMonomorphic, + this::isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator, @@ -942,7 +948,7 @@ public DescriptorGrouper createDescriptorGrouper( private static final int UNKNOWN_TYPE_ID = Types.UNKNOWN; - private int getXtypeId(Class cls) { + private byte getXtypeId(Class cls) { if (isSet(cls)) { return Types.SET; } @@ -955,8 +961,8 @@ private int getXtypeId(Class cls) { if (isMap(cls)) { return Types.MAP; } - if (fory.getXtypeResolver().isRegistered(cls)) { - return fory.getXtypeResolver().getClassInfo(cls).getXtypeId(); + if (isRegistered(cls)) { + return (byte) (getClassInfo(cls).getXtypeId() & 0xff); } else { if (cls.isEnum()) { return Types.ENUM; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 6038d51f89..1d485c966c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -23,12 +23,9 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.stream.Collectors; import org.apache.fory.Fory; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; import org.apache.fory.reflect.FieldAccessor; @@ -37,19 +34,12 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.reflect.TypeRef; import org.apache.fory.resolver.ClassInfo; -import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; -import org.apache.fory.resolver.RefMode; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; -import org.apache.fory.serializer.converter.FieldConverter; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; -import org.apache.fory.type.FinalObjectTypeStub; -import org.apache.fory.type.GenericType; import org.apache.fory.type.Generics; -import org.apache.fory.type.TypeUtils; -import org.apache.fory.util.StringUtils; import org.apache.fory.util.record.RecordComponent; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -59,7 +49,7 @@ public abstract class AbstractObjectSerializer extends Serializer { protected final ClassResolver classResolver; protected final boolean isRecord; protected final ObjectCreator objectCreator; - private InternalFieldInfo[] fieldInfos; + private FieldGroups.SerializationFieldInfo[] fieldInfos; private RecordInfo copyRecordInfo; public AbstractObjectSerializer(Fory fory, Class type) { @@ -74,220 +64,93 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec this.objectCreator = objectCreator; } - /** - * Read final object field value. Note that primitive field value can't be read by this method, - * because primitive field doesn't write null flag. - */ - static Object readFinalObjectFieldValue( + static void writeOtherFieldValue( SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - FinalTypeField fieldInfo, - boolean isFinal, - MemoryBuffer buffer) { - Serializer serializer = fieldInfo.classInfo.getSerializer(); - binding.incReadDepth(); - Object fieldValue; - if (isFinal) { + MemoryBuffer buffer, + FieldGroups.SerializationFieldInfo fieldInfo, + Object fieldValue) { + if (fieldInfo.useDeclaredTypeInfo) { switch (fieldInfo.refMode) { case NONE: - fieldValue = binding.read(buffer, serializer); + binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); break; case NULL_ONLY: - fieldValue = binding.readNullable(buffer, serializer); + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + binding.writeNonRef(buffer, fieldValue, fieldInfo.serializer); + } break; case TRACKING: - // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still - // consistent with jit serializer. - fieldValue = binding.readRef(buffer, serializer); + binding.writeRef(buffer, fieldValue, fieldInfo.serializer); break; default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } } else { switch (fieldInfo.refMode) { case NONE: - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); + binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); break; case NULL_ONLY: - { - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - binding.decDepth(); - return null; - } - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); + binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); } break; case TRACKING: - { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { - typeResolver.readClassInfo(buffer, fieldInfo.classInfo); - fieldValue = serializer.read(buffer); - refResolver.setReadObject(nextReadRefId, fieldValue); - } else { - fieldValue = refResolver.getReadObject(); - } - } + binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); break; default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - } - binding.decDepth(); - return fieldValue; - } - - /** - * Read a non-container field value that is not a final type. Handles enum types, reference - * tracking, and nullable fields according to xlang serialization protocol. - * - * @param binding the serialization binding for read operations - * @param fieldInfo the field metadata including type info and nullability - * @param buffer the buffer to read from - * @return the deserialized field value, or null if the field is nullable and was null - */ - static Object readOtherFieldValue( - SerializationBinding binding, GenericTypeField fieldInfo, MemoryBuffer buffer) { - // Note: Enum has special handling for xlang compatibility - no type info for enum fields - if (fieldInfo.genericType.getCls().isEnum()) { - // Only read null flag when the field is nullable (for xlang compatibility) - if (fieldInfo.nullable && buffer.readByte() == Fory.NULL_FLAG) { - return null; + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } - return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); - } - Object fieldValue; - switch (fieldInfo.refMode) { - case NONE: - binding.preserveRefId(-1); - fieldValue = binding.readNonRef(buffer, fieldInfo); - break; - case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } - fieldValue = binding.readNonRef(buffer, fieldInfo); - } - break; - case TRACKING: - fieldValue = binding.readRef(buffer, fieldInfo); - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); } - return fieldValue; } - /** - * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, - * and pushes/pops generic type information for proper deserialization of parameterized types. - * - * @param binding the serialization binding for read operations - * @param generics the generics context for tracking parameterized types - * @param fieldInfo the field metadata including generic type info and nullability - * @param buffer the buffer to read from - * @return the deserialized container field value, or null if the field is nullable and was null - */ - static Object readContainerFieldValue( + static void writeContainerFieldValue( SerializationBinding binding, + RefResolver refResolver, + TypeResolver typeResolver, Generics generics, - GenericTypeField fieldInfo, - MemoryBuffer buffer) { - Object fieldValue; + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer, + Object fieldValue) { switch (fieldInfo.refMode) { case NONE: - binding.preserveRefId(-1); generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + binding.writeContainerFieldValue( + buffer, + fieldValue, + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); generics.popGenericType(); break; case NULL_ONLY: - { - binding.preserveRefId(-1); - byte headFlag = buffer.readByte(); - if (headFlag == Fory.NULL_FLAG) { - return null; - } + if (fieldValue == null) { + buffer.writeByte(Fory.NULL_FLAG); + } else { + buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + binding.writeContainerFieldValue( + buffer, + fieldValue, + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); generics.popGenericType(); } break; case TRACKING: - generics.pushGenericType(fieldInfo.genericType); - fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); - generics.popGenericType(); - break; - default: - throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); - } - return fieldValue; - } - - /** - * Write a primitive field value to buffer using the field accessor. - * - * @param fory the fory instance for compression settings - * @param buffer the buffer to write to - * @param targetObject the object containing the field - * @param fieldAccessor the accessor to get the field value - * @param classId the class ID of the primitive type - * @return true if classId is not a primitive type and needs further write handling - */ - static boolean writePrimitiveFieldValue( - Fory fory, - MemoryBuffer buffer, - Object targetObject, - FieldAccessor fieldAccessor, - short classId) { - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); - } - switch (classId) { - case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: - buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: - buffer.writeByte((Byte) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: - buffer.writeChar((Character) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: - buffer.writeInt16((Short) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_INT_CLASS_ID: - { - int fieldValue = (Integer) fieldAccessor.get(targetObject); - if (fory.compressInt()) { - buffer.writeVarInt32(fieldValue); - } else { - buffer.writeInt32(fieldValue); - } - return false; - } - case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: - buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_LONG_CLASS_ID: - { - long fieldValue = (long) fieldAccessor.get(targetObject); - fory.writeInt64(buffer, fieldValue); - return false; + if (!refResolver.writeRefOrNull(buffer, fieldValue)) { + ClassInfo classInfo = + typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); + generics.pushGenericType(fieldInfo.genericType); + binding.writeContainerFieldValue(buffer, fieldValue, classInfo); + generics.popGenericType(); } - case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: - buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); - return false; + break; default: - return true; + throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); } } @@ -527,6 +390,224 @@ static boolean writeBasicNullableObjectFieldValue( } } + /** + * Read final object field value. Note that primitive field value can't be read by this method, + * because primitive field doesn't write null flag. + */ + static Object readFinalObjectFieldValue( + SerializationBinding binding, + RefResolver refResolver, + TypeResolver typeResolver, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + Serializer serializer = fieldInfo.classInfo.getSerializer(); + binding.incReadDepth(); + Object fieldValue; + if (fieldInfo.useDeclaredTypeInfo) { + switch (fieldInfo.refMode) { + case NONE: + fieldValue = binding.read(buffer, serializer); + break; + case NULL_ONLY: + fieldValue = binding.readNullable(buffer, serializer); + break; + case TRACKING: + // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still + // consistent with jit serializer. + fieldValue = binding.readRef(buffer, serializer); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + } else { + switch (fieldInfo.refMode) { + case NONE: + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + break; + case NULL_ONLY: + { + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + binding.decDepth(); + return null; + } + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + } + break; + case TRACKING: + { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fory.NOT_NULL_VALUE_FLAG) { + typeResolver.readClassInfo(buffer, fieldInfo.classInfo); + fieldValue = serializer.read(buffer); + refResolver.setReadObject(nextReadRefId, fieldValue); + } else { + fieldValue = refResolver.getReadObject(); + } + } + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + } + binding.decDepth(); + return fieldValue; + } + + /** + * Read a non-container field value that is not a final type. Handles enum types, reference + * tracking, and nullable fields according to xlang serialization protocol. + * + * @param binding the serialization binding for read operations + * @param fieldInfo the field metadata including type info and nullability + * @param buffer the buffer to read from + * @return the deserialized field value, or null if the field is nullable and was null + */ + static Object readOtherFieldValue( + SerializationBinding binding, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + // Note: Enum has special handling for xlang compatibility - no type info for enum fields + if (fieldInfo.genericType.getCls().isEnum()) { + // Only read null flag when the field is nullable (for xlang compatibility) + if (fieldInfo.nullable && buffer.readByte() == Fory.NULL_FLAG) { + return null; + } + return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer); + } + Object fieldValue; + switch (fieldInfo.refMode) { + case NONE: + binding.preserveRefId(-1); + fieldValue = binding.readNonRef(buffer, fieldInfo); + break; + case NULL_ONLY: + { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; + } + fieldValue = binding.readNonRef(buffer, fieldInfo); + } + break; + case TRACKING: + fieldValue = binding.readRef(buffer, fieldInfo); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + return fieldValue; + } + + /** + * Read a container field value (Collection or Map). Handles reference tracking, nullable fields, + * and pushes/pops generic type information for proper deserialization of parameterized types. + * + * @param binding the serialization binding for read operations + * @param generics the generics context for tracking parameterized types + * @param fieldInfo the field metadata including generic type info and nullability + * @param buffer the buffer to read from + * @return the deserialized container field value, or null if the field is nullable and was null + */ + static Object readContainerFieldValue( + SerializationBinding binding, + Generics generics, + FieldGroups.SerializationFieldInfo fieldInfo, + MemoryBuffer buffer) { + Object fieldValue; + switch (fieldInfo.refMode) { + case NONE: + binding.preserveRefId(-1); + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); + break; + case NULL_ONLY: + { + binding.preserveRefId(-1); + byte headFlag = buffer.readByte(); + if (headFlag == Fory.NULL_FLAG) { + return null; + } + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValue(buffer, fieldInfo); + generics.popGenericType(); + } + break; + case TRACKING: + generics.pushGenericType(fieldInfo.genericType); + fieldValue = binding.readContainerFieldValueRef(buffer, fieldInfo); + generics.popGenericType(); + break; + default: + throw new IllegalStateException("Unknown refMode: " + fieldInfo.refMode); + } + return fieldValue; + } + + /** + * Write a primitive field value to buffer using the field accessor. + * + * @param fory the fory instance for compression settings + * @param buffer the buffer to write to + * @param targetObject the object containing the field + * @param fieldAccessor the accessor to get the field value + * @param classId the class ID of the primitive type + * @return true if classId is not a primitive type and needs further write handling + */ + static boolean writePrimitiveFieldValue( + Fory fory, + MemoryBuffer buffer, + Object targetObject, + FieldAccessor fieldAccessor, + short classId) { + long fieldOffset = fieldAccessor.getFieldOffset(); + if (fieldOffset != -1) { + return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); + } + switch (classId) { + case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: + buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: + buffer.writeByte((Byte) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: + buffer.writeChar((Character) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: + buffer.writeInt16((Short) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_INT_CLASS_ID: + { + int fieldValue = (Integer) fieldAccessor.get(targetObject); + if (fory.compressInt()) { + buffer.writeVarInt32(fieldValue); + } else { + buffer.writeInt32(fieldValue); + } + return false; + } + case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: + buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_LONG_CLASS_ID: + { + long fieldValue = (long) fieldAccessor.get(targetObject); + fory.writeInt64(buffer, fieldValue); + return false; + } + case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: + buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); + return false; + default: + return true; + } + } + /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. @@ -856,13 +937,13 @@ private T copyRecord(T originObj) { } private Object[] copyFields(T originObj) { - InternalFieldInfo[] fieldInfos = this.fieldInfos; + FieldGroups.SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); } Object[] fieldValues = new Object[fieldInfos.length]; for (int i = 0; i < fieldInfos.length; i++) { - InternalFieldInfo fieldInfo = fieldInfos[i]; + FieldGroups.SerializationFieldInfo fieldInfo = fieldInfos[i]; FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); if (fieldOffset != -1) { @@ -877,11 +958,11 @@ private Object[] copyFields(T originObj) { } private void copyFields(T originObj, T newObj) { - InternalFieldInfo[] fieldInfos = this.fieldInfos; + FieldGroups.SerializationFieldInfo[] fieldInfos = this.fieldInfos; if (fieldInfos == null) { fieldInfos = buildFieldsInfo(); } - for (InternalFieldInfo fieldInfo : fieldInfos) { + for (FieldGroups.SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); // record class won't go to this path; @@ -930,8 +1011,8 @@ private void copyFields(T originObj, T newObj) { } public static void copyFields( - Fory fory, InternalFieldInfo[] fieldInfos, Object originObj, Object newObj) { - for (InternalFieldInfo fieldInfo : fieldInfos) { + Fory fory, FieldGroups.SerializationFieldInfo[] fieldInfos, Object originObj, Object newObj) { + for (FieldGroups.SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; long fieldOffset = fieldAccessor.getFieldOffset(); // record class won't go to this path; @@ -1012,7 +1093,7 @@ private Object copyField(Object targetObject, long fieldOffset, short classId) { } } - private InternalFieldInfo[] buildFieldsInfo() { + private FieldGroups.SerializationFieldInfo[] buildFieldsInfo() { List descriptors = new ArrayList<>(); if (RecordUtils.isRecord(type)) { RecordComponent[] components = RecordUtils.getRecordComponents(type); @@ -1037,12 +1118,8 @@ private InternalFieldInfo[] buildFieldsInfo() { } DescriptorGrouper descriptorGrouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, descriptorGrouper); - fieldInfos = new InternalFieldInfo[descriptors.size()]; - System.arraycopy(infos.f0.f0, 0, fieldInfos, 0, infos.f0.f0.length); - System.arraycopy(infos.f1, 0, fieldInfos, infos.f0.f0.length, infos.f1.length); - System.arraycopy(infos.f2, 0, fieldInfos, fieldInfos.length - infos.f2.length, infos.f2.length); + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + fieldInfos = fieldGroups.allFields; if (isRecord) { List fieldNames = Arrays.stream(fieldInfos) @@ -1053,207 +1130,7 @@ private InternalFieldInfo[] buildFieldsInfo() { return fieldInfos; } - public static InternalFieldInfo[] buildFieldsInfo(Fory fory, List fields) { - List descriptors = new ArrayList<>(); - for (Field field : fields) { - if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) { - descriptors.add(new Descriptor(field, TypeRef.of(field.getGenericType()), null, null)); - } - } - DescriptorGrouper descriptorGrouper = - fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, descriptorGrouper); - InternalFieldInfo[] fieldInfos = new InternalFieldInfo[descriptors.size()]; - System.arraycopy(infos.f0.f0, 0, fieldInfos, 0, infos.f0.f0.length); - System.arraycopy(infos.f1, 0, fieldInfos, infos.f0.f0.length, infos.f1.length); - System.arraycopy(infos.f2, 0, fieldInfos, fieldInfos.length - infos.f2.length, infos.f2.length); - return fieldInfos; - } - protected T newBean() { return objectCreator.newInstance(); } - - static Tuple3, GenericTypeField[], GenericTypeField[]> - buildFieldInfos(Fory fory, DescriptorGrouper grouper) { - // When a type is both Collection/Map and final, add it to collection/map fields to keep - // consistent with jit. - Collection primitives = grouper.getPrimitiveDescriptors(); - Collection boxed = grouper.getBoxedDescriptors(); - Collection finals = grouper.getFinalDescriptors(); - FinalTypeField[] finalFields = - new FinalTypeField[primitives.size() + boxed.size() + finals.size()]; - int cnt = 0; - for (Descriptor d : primitives) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - for (Descriptor d : boxed) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - for (Descriptor d : finals) { - finalFields[cnt++] = new FinalTypeField(fory, d); - } - boolean[] isFinal = new boolean[finalFields.length]; - for (int i = 0; i < isFinal.length; i++) { - FinalTypeField finalField = finalFields[i]; - ClassInfo classInfo = finalField.classInfo; - isFinal[i] = classInfo != null && fory.getClassResolver().isMonomorphic(finalField.descriptor); - } - cnt = 0; - GenericTypeField[] otherFields = new GenericTypeField[grouper.getOtherDescriptors().size()]; - for (Descriptor descriptor : grouper.getOtherDescriptors()) { - GenericTypeField genericTypeField = new GenericTypeField(fory, descriptor); - otherFields[cnt++] = genericTypeField; - } - cnt = 0; - Collection collections = grouper.getCollectionDescriptors(); - Collection maps = grouper.getMapDescriptors(); - GenericTypeField[] containerFields = new GenericTypeField[collections.size() + maps.size()]; - for (Descriptor d : collections) { - containerFields[cnt++] = new GenericTypeField(fory, d); - } - for (Descriptor d : maps) { - containerFields[cnt++] = new GenericTypeField(fory, d); - } - return Tuple3.of(Tuple2.of(finalFields, isFinal), otherFields, containerFields); - } - - public static class InternalFieldInfo { - protected final Descriptor descriptor; - protected final TypeRef typeRef; - protected final short classId; - protected final String qualifiedFieldName; - protected final FieldAccessor fieldAccessor; - protected final FieldConverter fieldConverter; - protected final RefMode refMode; - protected final boolean nullable; - protected final boolean trackingRef; - protected final boolean isPrimitive; - - private InternalFieldInfo(Fory fory, Descriptor d, short classId) { - this.descriptor = d; - this.typeRef = d.getTypeRef(); - this.classId = classId; - this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); - if (d.getField() != null) { - this.fieldAccessor = FieldAccessor.createAccessor(d.getField()); - isPrimitive = d.getField().getType().isPrimitive(); - } else { - this.fieldAccessor = null; - isPrimitive = d.getTypeRef().getRawType().isPrimitive(); - } - fieldConverter = d.getFieldConverter(); - nullable = d.isNullable(); - // descriptor.isTrackingRef() already includes the needToWriteRef check - trackingRef = d.isTrackingRef(); - refMode = RefMode.of(trackingRef, nullable); - } - - @Override - public String toString() { - String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); - return "InternalFieldInfo{" - + "fieldName='" - + rsplit[1] - + ", typeRef=" - + typeRef - + ", classId=" - + classId - + ", fieldAccessor=" - + fieldAccessor - + ", nullable=" - + nullable - + '}'; - } - } - - static final class FinalTypeField extends InternalFieldInfo { - final ClassInfo classInfo; - - private FinalTypeField(Fory fory, Descriptor d) { - super(fory, d, getRegisteredClassId(fory, d)); - // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. - if (typeRef.getRawType() == FinalObjectTypeStub.class) { - // `FinalObjectTypeStub` has no fields, using its `classInfo` - // will make deserialization failed. - classInfo = null; - } else { - classInfo = SerializationUtils.getClassInfo(fory, typeRef.getRawType()); - if (!fory.isShareMeta() - && !fory.isCompatible() - && classInfo.getSerializer() instanceof ReplaceResolveSerializer) { - // overwrite replace resolve serializer for final field - classInfo.setSerializer(new FinalFieldReplaceResolveSerializer(fory, classInfo.getCls())); - } - } - } - } - - static final class GenericTypeField extends InternalFieldInfo { - final GenericType genericType; - final ClassInfoHolder classInfoHolder; - final boolean isArray; - final ClassInfo containerClassInfo; - - private GenericTypeField(Fory fory, Descriptor d) { - super(fory, d, getRegisteredClassId(fory, d)); - // TODO support generics in Pojo, see ComplexObjectSerializer.getGenericTypes - ClassResolver classResolver = fory.getClassResolver(); - GenericType t = classResolver.buildGenericType(typeRef); - Class cls = t.getCls(); - if (t.getTypeParametersCount() > 0) { - boolean skip = - Arrays.stream(t.getTypeParameters()).allMatch(p -> p.getCls() == Object.class); - if (skip) { - t = new GenericType(t.getTypeRef(), t.isMonomorphic()); - } - } - genericType = t; - classInfoHolder = classResolver.nilClassInfoHolder(); - isArray = cls.isArray(); - if (!fory.isCrossLanguage()) { - containerClassInfo = null; - } else { - if (classResolver.isMap(cls) - || classResolver.isCollection(cls) - || classResolver.isSet(cls)) { - containerClassInfo = fory.getXtypeResolver().getClassInfo(cls); - } else { - containerClassInfo = null; - } - } - } - - @Override - public String toString() { - String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); - return "GenericTypeField{" - + "fieldName=" - + rsplit[1] - + ", genericType=" - + genericType - + ", classInfoHolder=" - + classInfoHolder - + ", trackingRef=" - + trackingRef - + ", typeRef=" - + typeRef - + ", classId=" - + classId - + ", nullable=" - + nullable - + '}'; - } - } - - private static short getRegisteredClassId(Fory fory, Descriptor d) { - Field field = d.getField(); - Class cls = d.getTypeRef().getRawType(); - if (TypeUtils.unwrap(cls).isPrimitive() && field != null) { - return fory.getClassResolver().getRegisteredClassId(field.getType()); - } - Short classId = fory.getClassResolver().getRegisteredClassId(cls); - return classId == null ? ClassResolver.NO_CLASS_ID : classId; - } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java new file mode 100644 index 0000000000..7107a8ea8b --- /dev/null +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.serializer; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import org.apache.fory.Fory; +import org.apache.fory.reflect.FieldAccessor; +import org.apache.fory.reflect.TypeRef; +import org.apache.fory.resolver.ClassInfo; +import org.apache.fory.resolver.ClassInfoHolder; +import org.apache.fory.resolver.ClassResolver; +import org.apache.fory.resolver.RefMode; +import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.converter.FieldConverter; +import org.apache.fory.type.Descriptor; +import org.apache.fory.type.DescriptorGrouper; +import org.apache.fory.type.FinalObjectTypeStub; +import org.apache.fory.type.GenericType; +import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.StringUtils; + +public class FieldGroups { + public final SerializationFieldInfo[] buildInFields; + public final SerializationFieldInfo[] userTypeFields; + public final SerializationFieldInfo[] containerFields; + public final SerializationFieldInfo[] allFields; + + public FieldGroups( + SerializationFieldInfo[] buildInFields, + SerializationFieldInfo[] containerFields, + SerializationFieldInfo[] userTypeFields) { + this.buildInFields = buildInFields; + this.userTypeFields = userTypeFields; + this.containerFields = containerFields; + SerializationFieldInfo[] fields = + new SerializationFieldInfo + [buildInFields.length + userTypeFields.length + containerFields.length]; + System.arraycopy(buildInFields, 0, fields, 0, buildInFields.length); + System.arraycopy(containerFields, 0, fields, buildInFields.length, containerFields.length); + System.arraycopy( + userTypeFields, + 0, + fields, + buildInFields.length + containerFields.length, + userTypeFields.length); + allFields = fields; + } + + public static FieldGroups buildFieldsInfo(Fory fory, List fields) { + List descriptors = new ArrayList<>(); + for (Field field : fields) { + if (!Modifier.isTransient(field.getModifiers()) && !Modifier.isStatic(field.getModifiers())) { + descriptors.add(new Descriptor(field, TypeRef.of(field.getGenericType()), null, null)); + } + } + DescriptorGrouper descriptorGrouper = + fory.getClassResolver().createDescriptorGrouper(descriptors, false); + return buildFieldInfos(fory, descriptorGrouper); + } + + public static FieldGroups buildFieldInfos(Fory fory, DescriptorGrouper grouper) { + // When a type is both Collection/Map and final, add it to collection/map fields to keep + // consistent with jit. + Collection primitives = grouper.getPrimitiveDescriptors(); + Collection boxed = grouper.getBoxedDescriptors(); + Collection buildIn = grouper.getBuildInDescriptors(); + SerializationFieldInfo[] allBuildIn = + new SerializationFieldInfo[primitives.size() + boxed.size() + buildIn.size()]; + int cnt = 0; + for (Descriptor d : primitives) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : boxed) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : buildIn) { + allBuildIn[cnt++] = new SerializationFieldInfo(fory, d); + } + cnt = 0; + SerializationFieldInfo[] otherFields = + new SerializationFieldInfo[grouper.getOtherDescriptors().size()]; + for (Descriptor descriptor : grouper.getOtherDescriptors()) { + SerializationFieldInfo genericTypeField = new SerializationFieldInfo(fory, descriptor); + otherFields[cnt++] = genericTypeField; + } + cnt = 0; + Collection collections = grouper.getCollectionDescriptors(); + Collection maps = grouper.getMapDescriptors(); + SerializationFieldInfo[] containerFields = + new SerializationFieldInfo[collections.size() + maps.size()]; + for (Descriptor d : collections) { + containerFields[cnt++] = new SerializationFieldInfo(fory, d); + } + for (Descriptor d : maps) { + containerFields[cnt++] = new SerializationFieldInfo(fory, d); + } + return new FieldGroups(allBuildIn, containerFields, otherFields); + } + + static short getRegisteredClassId(Fory fory, Descriptor d) { + Field field = d.getField(); + Class cls = d.getTypeRef().getRawType(); + if (TypeUtils.unwrap(cls).isPrimitive() && field != null) { + return fory.getClassResolver().getRegisteredClassId(field.getType()); + } + Short classId = fory.getClassResolver().getRegisteredClassId(cls); + return classId == null ? ClassResolver.NO_CLASS_ID : classId; + } + + public static final class SerializationFieldInfo { + public final Descriptor descriptor; + public final TypeRef typeRef; + public final short classId; + public final ClassInfo classInfo; + public final Serializer serializer; + public final String qualifiedFieldName; + public final FieldAccessor fieldAccessor; + public final FieldConverter fieldConverter; + public final RefMode refMode; + public final boolean nullable; + public final boolean trackingRef; + public final boolean isPrimitive; + // Use declared type for serialization/deserialization + public final boolean useDeclaredTypeInfo; + + public final GenericType genericType; + public final ClassInfoHolder classInfoHolder; + public final boolean isArray; + public final ClassInfo containerClassInfo; + + SerializationFieldInfo(Fory fory, Descriptor d) { + this.descriptor = d; + this.typeRef = d.getTypeRef(); + this.classId = getRegisteredClassId(fory, d); + TypeResolver resolver = fory._getTypeResolver(); + // invoke `copy` to avoid ObjectSerializer construct clear serializer by `clearSerializer`. + if (typeRef.getRawType() == FinalObjectTypeStub.class) { + // `FinalObjectTypeStub` has no fields, using its `classInfo` + // will make deserialization failed. + classInfo = null; + } else { + if (resolver.isMonomorphic(descriptor)) { + classInfo = SerializationUtils.getClassInfo(fory, typeRef.getRawType()); + if (!fory.isShareMeta() + && !fory.isCompatible() + && classInfo.getSerializer() instanceof ReplaceResolveSerializer) { + // overwrite replace resolve serializer for final field + classInfo.setSerializer( + new FinalFieldReplaceResolveSerializer(fory, classInfo.getCls())); + } + } else { + classInfo = null; + } + } + useDeclaredTypeInfo = classInfo != null && fory.getClassResolver().isMonomorphic(descriptor); + if (classInfo != null) { + serializer = classInfo.getSerializer(); + } else { + serializer = null; + } + + this.qualifiedFieldName = d.getDeclaringClass() + "." + d.getName(); + if (d.getField() != null) { + this.fieldAccessor = FieldAccessor.createAccessor(d.getField()); + isPrimitive = d.getField().getType().isPrimitive(); + } else { + this.fieldAccessor = null; + isPrimitive = d.getTypeRef().getRawType().isPrimitive(); + } + fieldConverter = d.getFieldConverter(); + nullable = d.isNullable(); + // descriptor.isTrackingRef() already includes the needToWriteRef check + trackingRef = d.isTrackingRef(); + refMode = RefMode.of(trackingRef, nullable); + + GenericType t = resolver.buildGenericType(typeRef); + Class cls = t.getCls(); + if (t.getTypeParametersCount() > 0) { + boolean skip = + Arrays.stream(t.getTypeParameters()).allMatch(p -> p.getCls() == Object.class); + if (skip) { + t = new GenericType(t.getTypeRef(), t.isMonomorphic()); + } + } + genericType = t; + classInfoHolder = resolver.nilClassInfoHolder(); + isArray = cls.isArray(); + if (!fory.isCrossLanguage()) { + containerClassInfo = null; + } else { + if (resolver.isMap(cls) || resolver.isCollection(cls) || resolver.isSet(cls)) { + containerClassInfo = resolver.getClassInfo(cls); + } else { + containerClassInfo = null; + } + } + } + + @Override + public String toString() { + String[] rsplit = StringUtils.rsplit(qualifiedFieldName, ".", 1); + return "InternalFieldInfo{" + + "fieldName='" + + rsplit[1] + + ", typeRef=" + + typeRef + + ", classId=" + + classId + + ", fieldAccessor=" + + fieldAccessor + + ", nullable=" + + nullable + + '}'; + } + } +} diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java index cad3dcb1b0..af0da89a9d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedLayerSerializer.java @@ -23,8 +23,7 @@ import org.apache.fory.Fory; import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.ObjectArray; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; +import org.apache.fory.collection.ObjectIntMap; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; @@ -34,6 +33,7 @@ import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; @@ -55,10 +55,9 @@ public class MetaSharedLayerSerializer extends MetaSharedLayerSerializerBase { private final ClassDef layerClassDef; private final Class layerMarkerClass; - private final ObjectSerializer.FinalTypeField[] finalFields; - private final boolean[] isFinal; - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final ClassInfoHolder classInfoHolder; private final SerializationBinding binding; private final TypeResolver typeResolver; @@ -83,16 +82,10 @@ public MetaSharedLayerSerializer( // Build field infos from layerClassDef Collection descriptors = layerClassDef.getDescriptors(typeResolver, type); DescriptorGrouper descriptorGrouper = typeResolver.createDescriptorGrouper(descriptors, false); - - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - infos = AbstractObjectSerializer.buildFieldInfos(fory, descriptorGrouper); - this.finalFields = infos.f0.f0; - this.isFinal = infos.f0.f1; - this.otherFields = infos.f1; - this.containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + this.buildInFields = fieldGroups.buildInFields; + this.otherFields = fieldGroups.userTypeFields; + this.containerFields = fieldGroups.containerFields; } @Override @@ -129,8 +122,7 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -145,7 +137,7 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { fory, buffer, fieldValue, classId); if (writeBasicObjectResult) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinal[i]) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { if (!fieldInfo.trackingRef) { binding.writeNullable(buffer, fieldValue, serializer, nullable); } else { @@ -168,19 +160,19 @@ private void writeFinalFields(MemoryBuffer buffer, T value) { private void writeContainerFields(MemoryBuffer buffer, T value) { Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); } } private void writeOtherFields(MemoryBuffer buffer, T value) { - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - ObjectSerializer.writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -208,7 +200,7 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { // Read fields in order: final, container, other readFinalFields(buffer, obj); readContainerFields(buffer, obj); - readOtherFields(buffer, obj); + readUserTypeFields(buffer, obj); return obj; } @@ -235,11 +227,8 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { Fory fory = this.fory; RefResolver refResolver = this.refResolver; ClassResolver classResolver = this.classResolver; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinalField = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { boolean nullable = fieldInfo.nullable; @@ -253,7 +242,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory, buffer, obj, fieldAccessor, classId))) { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -263,7 +252,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { fory.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } } @@ -272,7 +261,7 @@ private void readFinalFields(MemoryBuffer buffer, T obj) { private void readContainerFields(MemoryBuffer buffer, T obj) { Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -282,8 +271,8 @@ private void readContainerFields(MemoryBuffer buffer, T obj) { } } - private void readOtherFields(MemoryBuffer buffer, T obj) { - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + private void readUserTypeFields(MemoryBuffer buffer, T obj) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -314,7 +303,7 @@ public Class getLayerMarkerClass() { /** Returns the number of fields in this layer. */ public int getNumFields() { - return finalFields.length + containerFields.length + otherFields.length; + return buildInFields.length + containerFields.length + otherFields.length; } /** @@ -331,30 +320,26 @@ public void writeFieldsValues(MemoryBuffer buffer, Object[] vals) { // Write field values from array int index = 0; // Write final fields - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : buildInFields) { Object fieldValue = vals[index++]; - writeFieldValueFromArray(buffer, fieldInfo, fieldValue, isFinal[i]); + writeFieldValueFromArray(buffer, fieldInfo, fieldValue); } // Write container fields Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = vals[index++]; - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, typeResolver, generics, fieldInfo, buffer, fieldValue); } // Write other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = vals[index++]; - ObjectSerializer.writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + AbstractObjectSerializer.writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } private void writeFieldValueFromArray( - MemoryBuffer buffer, - ObjectSerializer.FinalTypeField fieldInfo, - Object fieldValue, - boolean isFinalField) { + MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object fieldValue) { short classId = fieldInfo.classId; boolean nullable = fieldInfo.nullable; @@ -395,7 +380,7 @@ private void writeFieldValueFromArray( // Handle objects boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinalField) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { if (!fieldInfo.trackingRef) { binding.writeNullable(buffer, fieldValue, serializer, nullable); } else { @@ -426,28 +411,23 @@ public void readFields(MemoryBuffer buffer, Object[] vals) { } // Read field values into array int index = 0; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - // Read final fields - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinalField = !metaShareEnabled || this.isFinal[i]; - vals[index++] = readFieldValueToArray(buffer, fieldInfo, isFinalField); + for (SerializationFieldInfo fieldInfo : buildInFields) { + vals[index++] = readFieldValueToArray(buffer, fieldInfo); } // Read container fields Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { vals[index++] = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); } // Read other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { vals[index++] = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); } } - private Object readFieldValueToArray( - MemoryBuffer buffer, ObjectSerializer.FinalTypeField fieldInfo, boolean isFinalField) { + private Object readFieldValueToArray(MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { short classId = fieldInfo.classId; // Handle primitives @@ -458,7 +438,7 @@ private Object readFieldValueToArray( // Handle objects return AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinalField, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } /** @@ -471,32 +451,17 @@ private Object readFieldValueToArray( */ @Override @SuppressWarnings("rawtypes") - public void setFieldValuesFromPutFields( - Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, Object[] vals) { + public void setFieldValuesFromPutFields(Object obj, ObjectIntMap fieldIndexMap, Object[] vals) { // Set final fields - for (ObjectSerializer.FinalTypeField fieldInfo : finalFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - fieldAccessor.set(obj, vals[index]); - } - } - } - // Set container fields - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - fieldAccessor.set(obj, vals[index]); - } - } - } + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, buildInFields); + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, containerFields); + setFieldValuesFromPutFields(obj, fieldIndexMap, vals, otherFields); + } + + private void setFieldValuesFromPutFields( + Object obj, ObjectIntMap fieldIndexMap, Object[] vals, SerializationFieldInfo[] fieldInfos) { // Set other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : fieldInfos) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { String fieldName = fieldAccessor.getField().getName(); @@ -523,29 +488,20 @@ public Object[] getFieldValuesForPutFields( Object obj, org.apache.fory.collection.ObjectIntMap fieldIndexMap, int arraySize) { Object[] vals = new Object[arraySize]; // Get final fields - for (ObjectSerializer.FinalTypeField fieldInfo : finalFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - vals[index] = fieldAccessor.get(obj); - } - } - } + getFieldValuesForPutFields(obj, fieldIndexMap, vals, buildInFields); // Get container fields - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { - FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; - if (fieldAccessor != null) { - String fieldName = fieldAccessor.getField().getName(); - int index = fieldIndexMap.get(fieldName, -1); - if (index != -1 && index < vals.length) { - vals[index] = fieldAccessor.get(obj); - } - } - } + getFieldValuesForPutFields(obj, fieldIndexMap, vals, containerFields); // Get other fields - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + getFieldValuesForPutFields(obj, fieldIndexMap, vals, otherFields); + return vals; + } + + private void getFieldValuesForPutFields( + Object obj, + ObjectIntMap fieldIndexMap, + Object[] vals, + SerializationFieldInfo[] buildInFields) { + for (SerializationFieldInfo fieldInfo : buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { String fieldName = fieldAccessor.getField().getName(); @@ -555,6 +511,5 @@ public Object[] getFieldValuesForPutFields( } } } - return vals; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 8a4c539517..cafe11e802 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -25,8 +25,6 @@ import java.util.stream.Collectors; import org.apache.fory.Fory; import org.apache.fory.builder.MetaSharedCodecBuilder; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.logging.Logger; @@ -39,6 +37,7 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; @@ -65,21 +64,12 @@ * @see MetaSharedCodecBuilder * @see ObjectSerializer */ -@SuppressWarnings({"unchecked"}) public class MetaSharedSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(MetaSharedSerializer.class); - private final ObjectSerializer.FinalTypeField[] finalFields; - - /** - * Whether write class def for non-inner final types. - * - * @see ClassResolver#isMonomorphic(Class) - */ - private final boolean[] isFinal; - - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] containerFields; + private final SerializationFieldInfo[] otherFields; private final RecordInfo recordInfo; private Serializer serializer; private final ClassInfoHolder classInfoHolder; @@ -117,15 +107,10 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { } } // d.getField() may be null if not exists in this class when meta share enabled. - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - infos = AbstractObjectSerializer.buildFieldInfos(fory, descriptorGrouper); - finalFields = infos.f0.f0; - isFinal = infos.f0.f1; - otherFields = infos.f1; - containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, descriptorGrouper); + buildInFields = fieldGroups.buildInFields; + containerFields = fieldGroups.containerFields; + otherFields = fieldGroups.userTypeFields; classInfoHolder = this.classResolver.nilClassInfoHolder(); if (isRecord) { List fieldNames = @@ -140,15 +125,13 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { boolean hasDefaultValues = false; DefaultValueUtils.DefaultValueField[] defaultValueFields = new DefaultValueUtils.DefaultValueField[0]; - DefaultValueUtils.DefaultValueSupport defaultValueSupport = null; + DefaultValueUtils.DefaultValueSupport defaultValueSupport; if (fory.getConfig().isScalaOptimizationEnabled()) { defaultValueSupport = DefaultValueUtils.getScalaDefaultValueSupport(); - if (defaultValueSupport != null) { - hasDefaultValues = defaultValueSupport.hasDefaultValues(type); - defaultValueFields = - defaultValueSupport.buildDefaultValueFields( - fory, type, descriptorGrouper.getSortedDescriptors()); - } + hasDefaultValues = defaultValueSupport.hasDefaultValues(type); + defaultValueFields = + defaultValueSupport.buildDefaultValueFields( + fory, type, descriptorGrouper.getSortedDescriptors()); } if (!hasDefaultValues) { DefaultValueUtils.DefaultValueSupport kotlinDefaultValueSupport = @@ -182,7 +165,7 @@ public void xwrite(MemoryBuffer buffer, T value) { public T read(MemoryBuffer buffer) { if (isRecord) { Object[] fieldValues = - new Object[finalFields.length + otherFields.length + containerFields.length]; + new Object[buildInFields.length + otherFields.length + containerFields.length]; readFields(buffer, fieldValues); fieldValues = RecordUtils.remapping(recordInfo, fieldValues); T t = objectCreator.newInstanceWithArguments(fieldValues); @@ -196,10 +179,7 @@ public T read(MemoryBuffer buffer) { SerializationBinding binding = this.binding; refResolver.reference(obj); // read order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = this.finalFields; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; if (fieldAccessor != null) { @@ -221,7 +201,7 @@ public T read(MemoryBuffer buffer) { assert fieldInfo.classInfo != null; Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -230,19 +210,19 @@ public T read(MemoryBuffer buffer) { if (skipPrimitiveFieldValueFailed(fory, fieldInfo.classId, buffer)) { if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. - fory.readRef(buffer, classInfoHolder); + binding.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } } else { - compatibleRead(buffer, fieldInfo, isFinal, obj); + compatibleRead(buffer, fieldInfo, obj); } } } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; @@ -250,7 +230,7 @@ public T read(MemoryBuffer buffer) { fieldAccessor.putObject(obj, fieldValue); } } - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; if (fieldAccessor != null) { @@ -260,8 +240,7 @@ public T read(MemoryBuffer buffer) { return obj; } - private void compatibleRead( - MemoryBuffer buffer, FinalTypeField fieldInfo, boolean isFinal, Object obj) { + private void compatibleRead(MemoryBuffer buffer, SerializationFieldInfo fieldInfo, Object obj) { Object fieldValue; short classId = fieldInfo.classId; if (classId >= ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID @@ -270,7 +249,7 @@ private void compatibleRead( } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } fieldInfo.fieldConverter.set(obj, fieldValue); } @@ -297,10 +276,8 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { ClassResolver classResolver = this.classResolver; SerializationBinding binding = this.binding; // read order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = this.finalFields; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = this.isFinal[i]; + SerializationFieldInfo[] finalFields = this.buildInFields; + for (SerializationFieldInfo fieldInfo : finalFields) { if (fieldInfo.fieldAccessor != null) { assert fieldInfo.classInfo != null; short classId = fieldInfo.classId; @@ -311,7 +288,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { } else { Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); fields[counter++] = fieldValue; } } else { @@ -322,19 +299,19 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { fory.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal, buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } // remapping will handle those extra fields from peers. fields[counter++] = null; } } - for (ObjectSerializer.GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); fields[counter++] = fieldValue; } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); fields[counter++] = fieldValue; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 4213343dc4..81c362bd2f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -19,7 +19,7 @@ package org.apache.fory.serializer; -import static org.apache.fory.serializer.ObjectSerializer.writeOtherFieldValue; +import static org.apache.fory.serializer.AbstractObjectSerializer.writeOtherFieldValue; import static org.apache.fory.serializer.SerializationUtils.getTypeResolver; import java.util.ArrayList; @@ -29,8 +29,6 @@ import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; import org.apache.fory.collection.MapEntry; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.resolver.ClassInfo; @@ -40,6 +38,7 @@ import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.NonexistentClass.NonexistentEnum; import org.apache.fory.serializer.Serializers.CrossLanguageCompatibleSerializer; import org.apache.fory.type.Descriptor; @@ -51,22 +50,15 @@ public final class NonexistentClassSerializers { private static final class ClassFieldsInfo { - private final ObjectSerializer.FinalTypeField[] finalFields; - private final boolean[] isFinal; - private final ObjectSerializer.GenericTypeField[] otherFields; - private final ObjectSerializer.GenericTypeField[] containerFields; + private final SerializationFieldInfo[] finalFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final int classVersionHash; - private ClassFieldsInfo( - ObjectSerializer.FinalTypeField[] finalFields, - boolean[] isFinal, - ObjectSerializer.GenericTypeField[] otherFields, - ObjectSerializer.GenericTypeField[] containerFields, - int classVersionHash) { - this.finalFields = finalFields; - this.isFinal = isFinal; - this.otherFields = otherFields; - this.containerFields = containerFields; + private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { + this.finalFields = fieldGroups.buildInFields; + this.otherFields = fieldGroups.userTypeFields; + this.containerFields = fieldGroups.containerFields; this.classVersionHash = classVersionHash; } } @@ -124,16 +116,14 @@ public void write(MemoryBuffer buffer, Object v) { buffer.writeInt32(fieldsInfo.classVersionHash); } // write order: primitive,boxed,final,other,collection,map - ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields; - boolean[] isFinal = fieldsInfo.isFinal; - for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + SerializationFieldInfo[] finalFields = fieldsInfo.finalFields; + for (SerializationFieldInfo fieldInfo : finalFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); ClassInfo classInfo = fieldInfo.classInfo; if (classResolver.isPrimitive(fieldInfo.classId)) { classInfo.getSerializer().write(buffer, fieldValue); } else { - if (isFinal[i]) { + if (fieldInfo.useDeclaredTypeInfo) { // whether tracking ref is recorded in `fieldInfo.serializer`, so it's still // consistent with jit serializer. Serializer serializer = classInfo.getSerializer(); @@ -144,14 +134,14 @@ public void write(MemoryBuffer buffer, Object v) { } } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - ObjectSerializer.writeContainerFieldValue( + AbstractObjectSerializer.writeContainerFieldValue( binding, refResolver, classResolver, generics, fieldInfo, buffer, fieldValue); } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); - writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -165,17 +155,12 @@ private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { resolver, NonexistentClass.NonexistentSkip.class, classDef); DescriptorGrouper grouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); - Tuple3< - Tuple2, - ObjectSerializer.GenericTypeField[], - ObjectSerializer.GenericTypeField[]> - tuple = AbstractObjectSerializer.buildFieldInfos(fory, grouper); + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, grouper); int classVersionHash = 0; if (fory.checkClassVersion()) { classVersionHash = ObjectSerializer.computeStructHash(fory, grouper); } - fieldsInfo = - new ClassFieldsInfo(tuple.f0.f0, tuple.f0.f1, tuple.f1, tuple.f2, classVersionHash); + fieldsInfo = new ClassFieldsInfo(fieldGroups, classVersionHash); fieldsInfoMap.put(classDef.getId(), fieldsInfo); } return fieldsInfo; @@ -192,10 +177,9 @@ public Object read(MemoryBuffer buffer) { List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); - ObjectSerializer.FinalTypeField[] finalFields = fieldsInfo.finalFields; - boolean[] isFinal = fieldsInfo.isFinal; + SerializationFieldInfo[] finalFields = fieldsInfo.finalFields; for (int i = 0; i < finalFields.length; i++) { - ObjectSerializer.FinalTypeField fieldInfo = finalFields[i]; + SerializationFieldInfo fieldInfo = finalFields[i]; Object fieldValue; if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. @@ -206,18 +190,18 @@ public Object read(MemoryBuffer buffer) { } else { fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, isFinal[i], buffer); + binding, refResolver, classResolver, fieldInfo, buffer); } } entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } Generics generics = fory.getGenerics(); - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.containerFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.containerFields) { Object fieldValue = AbstractObjectSerializer.readContainerFieldValue(binding, generics, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); } - for (ObjectSerializer.GenericTypeField fieldInfo : fieldsInfo.otherFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.otherFields) { Object fieldValue = AbstractObjectSerializer.readOtherFieldValue(binding, fieldInfo, buffer); entries.add(new MapEntry(fieldInfo.qualifiedFieldName, fieldValue)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index 82a568c061..376ed5c7d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -25,24 +25,21 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.fory.Fory; -import org.apache.fory.collection.Tuple2; -import org.apache.fory.collection.Tuple3; import org.apache.fory.exception.ForyException; import org.apache.fory.logging.Logger; import org.apache.fory.logging.LoggerFactory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.meta.ClassDef; import org.apache.fory.reflect.FieldAccessor; -import org.apache.fory.resolver.ClassInfo; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.struct.Fingerprint; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; import org.apache.fory.util.MurmurHash3; -import org.apache.fory.util.Preconditions; import org.apache.fory.util.Utils; import org.apache.fory.util.record.RecordInfo; import org.apache.fory.util.record.RecordUtils; @@ -66,17 +63,9 @@ public final class ObjectSerializer extends AbstractObjectSerializer { private static final Logger LOG = LoggerFactory.getLogger(ObjectSerializer.class); private final RecordInfo recordInfo; - private final FinalTypeField[] finalFields; - - /** - * Whether write class def for non-inner final types. - * - * @see ClassResolver#isMonomorphic(Class) - */ - private final boolean[] isFinal; - - private final GenericTypeField[] otherFields; - private final GenericTypeField[] containerFields; + private final SerializationFieldInfo[] buildInFields; + private final SerializationFieldInfo[] otherFields; + private final SerializationFieldInfo[] containerFields; private final int classVersionHash; private final SerializationBinding binding; private final TypeResolver typeResolver; @@ -135,12 +124,10 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { } else { classVersionHash = 0; } - Tuple3, GenericTypeField[], GenericTypeField[]> infos = - buildFieldInfos(fory, grouper); - finalFields = infos.f0.f0; - isFinal = infos.f0.f1; - otherFields = infos.f1; - containerFields = infos.f2; + FieldGroups fieldGroups = FieldGroups.buildFieldInfos(fory, grouper); + buildInFields = fieldGroups.buildInFields; + otherFields = fieldGroups.userTypeFields; + containerFields = fieldGroups.containerFields; } @Override @@ -158,16 +145,16 @@ public void write(MemoryBuffer buffer, T value) { buffer.writeInt32(classVersionHash); } // write order: primitive,boxed,final,other,collection,map - writeFinalFields(buffer, value, fory, refResolver, typeResolver); + writeBuildInFields(buffer, value, fory, refResolver, typeResolver); writeContainerFields(buffer, value, fory, refResolver, typeResolver); writeOtherFields(buffer, value); } private void writeOtherFields(MemoryBuffer buffer, T value) { - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); - writeOtherFieldValue(binding, typeResolver, buffer, fieldInfo, fieldValue); + writeOtherFieldValue(binding, buffer, fieldInfo, fieldValue); } } @@ -176,12 +163,10 @@ public void xwrite(MemoryBuffer buffer, T value) { write(buffer, value); } - private void writeFinalFields( + private void writeBuildInFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { - FinalTypeField[] finalFields = this.finalFields; boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -193,7 +178,7 @@ private void writeFinalFields( : writeBasicObjectFieldValue(fory, buffer, fieldValue, classId); if (needWrite) { Serializer serializer = fieldInfo.classInfo.getSerializer(); - if (!metaShareEnabled || isFinal[i]) { + if (!metaShareEnabled || fieldInfo.useDeclaredTypeInfo) { switch (fieldInfo.refMode) { case NONE: binding.write(buffer, serializer, fieldValue); @@ -243,7 +228,7 @@ private void writeFinalFields( private void writeContainerFields( MemoryBuffer buffer, T value, Fory fory, RefResolver refResolver, TypeResolver typeResolver) { Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; Object fieldValue = fieldAccessor.getObject(value); writeContainerFieldValue( @@ -251,97 +236,12 @@ private void writeContainerFields( } } - static void writeContainerFieldValue( - SerializationBinding binding, - RefResolver refResolver, - TypeResolver typeResolver, - Generics generics, - GenericTypeField fieldInfo, - MemoryBuffer buffer, - Object fieldValue) { - switch (fieldInfo.refMode) { - case NONE: - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue( - buffer, - fieldValue, - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder)); - generics.popGenericType(); - } - break; - case TRACKING: - if (!refResolver.writeRefOrNull(buffer, fieldValue)) { - ClassInfo classInfo = - typeResolver.getClassInfo(fieldValue.getClass(), fieldInfo.classInfoHolder); - generics.pushGenericType(fieldInfo.genericType); - binding.writeContainerFieldValue(buffer, fieldValue, classInfo); - generics.popGenericType(); - } - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } - - static void writeOtherFieldValue( - SerializationBinding binding, - TypeResolver typeResolver, - MemoryBuffer buffer, - GenericTypeField fieldInfo, - Object fieldValue) { - // Enum has special handling for xlang compatibility - no ref tracking for enums - if (fieldInfo.genericType.getCls().isEnum()) { - if (fieldValue == null) { - Preconditions.checkArgument( - fieldInfo.nullable, "Enum field value is null but not nullable"); - buffer.writeByte(Fory.NULL_FLAG); - } else { - // Only write null flag when the field is nullable (for xlang compatibility) - if (fieldInfo.nullable) { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - } - fieldInfo.genericType.getSerializer(typeResolver).write(buffer, fieldValue); - } - return; - } - switch (fieldInfo.refMode) { - case NONE: - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - case NULL_ONLY: - if (fieldValue == null) { - buffer.writeByte(Fory.NULL_FLAG); - } else { - buffer.writeByte(Fory.NOT_NULL_VALUE_FLAG); - binding.writeNonRef(buffer, fieldValue, fieldInfo.classInfoHolder); - } - break; - case TRACKING: - binding.writeRef(buffer, fieldValue, fieldInfo.classInfoHolder); - break; - default: - throw new IllegalStateException("Unexpected refMode: " + fieldInfo.refMode); - } - } - @Override public T read(MemoryBuffer buffer) { if (isRecord) { Object[] fields = readFields(buffer); fields = RecordUtils.remapping(recordInfo, fields); - T obj = (T) objectCreator.newInstanceWithArguments(fields); + T obj = objectCreator.newInstanceWithArguments(fields); Arrays.fill(recordInfo.getRecordComponents(), null); return obj; } @@ -361,34 +261,29 @@ public Object[] readFields(MemoryBuffer buffer) { TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); - checkClassVersion(fory, hash, classVersionHash); + checkClassVersion(type, hash, classVersionHash); } Object[] fieldValues = - new Object[finalFields.length + otherFields.length + containerFields.length]; + new Object[buildInFields.length + otherFields.length + containerFields.length]; int counter = 0; // read order: primitive,boxed,final,other,collection,map - FinalTypeField[] finalFields = this.finalFields; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { short classId = fieldInfo.classId; if (classId >= ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID && classId <= ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID) { fieldValues[counter++] = Serializers.readPrimitiveValue(fory, buffer, classId); } else { Object fieldValue = - readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, isFinal, buffer); + readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } } Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); fieldValues[counter++] = fieldValue; } @@ -401,14 +296,10 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { TypeResolver typeResolver = this.typeResolver; if (fory.checkClassVersion()) { int hash = buffer.readInt32(); - checkClassVersion(fory, hash, classVersionHash); + checkClassVersion(type, hash, classVersionHash); } // read order: primitive,boxed,final,other,collection,map - FinalTypeField[] finalFields = this.finalFields; - boolean metaShareEnabled = fory.getConfig().isMetaShareEnabled(); - for (int i = 0; i < finalFields.length; i++) { - FinalTypeField fieldInfo = finalFields[i]; - boolean isFinal = !metaShareEnabled || this.isFinal[i]; + for (SerializationFieldInfo fieldInfo : this.buildInFields) { FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; boolean nullable = fieldInfo.nullable; short classId = fieldInfo.classId; @@ -417,18 +308,17 @@ public T readAndSetFields(MemoryBuffer buffer, T obj) { ? readBasicNullableObjectFieldValue(fory, buffer, obj, fieldAccessor, classId) : readBasicObjectFieldValue(fory, buffer, obj, fieldAccessor, classId))) { Object fieldValue = - readFinalObjectFieldValue( - binding, refResolver, typeResolver, fieldInfo, isFinal, buffer); + readFinalObjectFieldValue(binding, refResolver, typeResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } Generics generics = fory.getGenerics(); - for (GenericTypeField fieldInfo : containerFields) { + for (SerializationFieldInfo fieldInfo : containerFields) { Object fieldValue = readContainerFieldValue(binding, generics, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); } - for (GenericTypeField fieldInfo : otherFields) { + for (SerializationFieldInfo fieldInfo : otherFields) { Object fieldValue = readOtherFieldValue(binding, fieldInfo, buffer); FieldAccessor fieldAccessor = fieldInfo.fieldAccessor; fieldAccessor.putObject(obj, fieldValue); @@ -456,12 +346,12 @@ public static int computeStructHash(Fory fory, DescriptorGrouper grouper) { return hash; } - public static void checkClassVersion(Fory fory, int readHash, int classVersionHash) { + public static void checkClassVersion(Class cls, int readHash, int classVersionHash) { if (readHash != classVersionHash) { throw new ForyException( String.format( "Read class %s version %s is not consistent with %s", - fory.getClassResolver().getCurrentReadClass(), readHash, classVersionHash)); + cls, readHash, classVersionHash)); } } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java index 2309e3a895..4ce0ebaf0c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationBinding.java @@ -20,7 +20,6 @@ package org.apache.fory.serializer; import static org.apache.fory.Fory.NOT_NULL_VALUE_FLAG; -import static org.apache.fory.serializer.AbstractObjectSerializer.GenericTypeField; import org.apache.fory.Fory; import org.apache.fory.logging.Logger; @@ -32,13 +31,15 @@ import org.apache.fory.resolver.RefResolver; import org.apache.fory.resolver.TypeResolver; import org.apache.fory.resolver.XtypeResolver; -import org.apache.fory.util.Utils; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; // This polymorphic interface has cost, do not expose it as a public class // If it's used in other packages in fory, duplicate it in those packages. @SuppressWarnings({"rawtypes", "unchecked"}) // noinspection Duplicates abstract class SerializationBinding { + private static final Logger LOG = LoggerFactory.getLogger(SerializationBinding.class); + protected final Fory fory; protected final RefResolver refResolver; protected final TypeResolver typeResolver; @@ -59,7 +60,7 @@ abstract class SerializationBinding { abstract void writeNonRef(MemoryBuffer buffer, Object obj); - abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo); + abstract void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer); abstract void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfoHolder classInfoHolder); @@ -86,7 +87,7 @@ abstract void writeContainerFieldValue( abstract T readRef(MemoryBuffer buffer, Serializer serializer); - abstract Object readRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readRef(MemoryBuffer buffer, SerializationFieldInfo field); abstract Object readRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); @@ -96,16 +97,16 @@ abstract void writeContainerFieldValue( abstract Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder); - abstract Object readNonRef(MemoryBuffer buffer, GenericTypeField field); + abstract Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field); abstract Object readNullable(MemoryBuffer buffer, Serializer serializer); abstract Object readNullable( MemoryBuffer buffer, Serializer serializer, boolean nullable); - abstract Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field); + abstract Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field); - abstract Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo); + abstract Object readContainerFieldValueRef(MemoryBuffer buffer, SerializationFieldInfo fieldInfo); public int preserveRefId(int refId) { return refResolver.preserveRefId(refId); @@ -165,7 +166,10 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.readRef(buffer, field.classInfo.getSerializer()); + } return fory.readRef(buffer, field.classInfoHolder); } @@ -190,7 +194,10 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { } @Override - public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { + if (field.useDeclaredTypeInfo) { + return fory.readNonRef(buffer, field.classInfo); + } return fory.readNonRef(buffer, field.classInfoHolder); } @@ -210,12 +217,13 @@ public Object readNullable( } @Override - public Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field) { return fory.readNonRef(buffer, field.classInfoHolder); } @Override - public Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField fieldInfo) { + public Object readContainerFieldValueRef( + MemoryBuffer buffer, SerializationFieldInfo fieldInfo) { RefResolver refResolver = fory.getRefResolver(); int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { @@ -244,8 +252,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.writeNonRef(buffer, obj, classInfo); + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + fory.writeNonRef(buffer, obj, serializer); } @Override @@ -321,7 +329,6 @@ public void writeContainerFieldValue( } static final class XlangSerializationBinding extends SerializationBinding { - private static final Logger LOG = LoggerFactory.getLogger(XlangSerializationBinding.class); private final XtypeResolver xtypeResolver; XlangSerializationBinding(Fory fory) { @@ -336,7 +343,7 @@ public void writeRef(MemoryBuffer buffer, T obj) { @Override public void writeRef(MemoryBuffer buffer, T obj, Serializer serializer) { - fory.xwriteRef(buffer, obj, serializer); + fory.writeRef(buffer, obj, serializer); } @Override @@ -355,24 +362,22 @@ public T readRef(MemoryBuffer buffer, Serializer serializer) { } @Override - public Object readRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.isArray) { fory.getGenerics().pushGenericType(field.genericType); - Object o = fory.xreadRef(buffer); + Object o; + if (field.useDeclaredTypeInfo) { + o = fory.xreadRef(buffer, field.serializer); + } else { + o = fory.xreadRef(buffer); + } fory.getGenerics().popGenericType(); return o; } else { - // In COMPATIBLE mode (meta share enabled): read type info for schema evolution - // In SCHEMA_CONSISTENT mode: don't read type info, use serializer directly - if (fory.getConfig().isMetaShareEnabled()) { - return fory.xreadRef(buffer); + if (field.useDeclaredTypeInfo) { + return fory.xreadRef(buffer, field.serializer); } else { - // SCHEMA_CONSISTENT mode: resolve serializer and read without type info - ClassInfo classInfo = field.classInfoHolder.classInfo; - if (classInfo.getSerializer() == null) { - classInfo = xtypeResolver.getClassInfo(classInfo.getCls(), field.classInfoHolder); - } - return fory.xreadRef(buffer, classInfo.getSerializer()); + return fory.xreadRef(buffer); } } } @@ -398,14 +403,23 @@ public Object readNonRef(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { } @Override - public Object readNonRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readNonRef(MemoryBuffer buffer, SerializationFieldInfo field) { if (field.isArray) { fory.getGenerics().pushGenericType(field.genericType); - Object o = fory.xreadNonRef(buffer); + Object o; + if (field.useDeclaredTypeInfo) { + o = fory.xreadNonRef(buffer, field.serializer); + } else { + o = fory.xreadNonRef(buffer); + } fory.getGenerics().popGenericType(); return o; } else { - return fory.xreadNonRef(buffer); + if (field.useDeclaredTypeInfo) { + return fory.xreadNonRef(buffer, field.serializer); + } else { + return fory.xreadNonRef(buffer); + } } } @@ -425,12 +439,12 @@ public Object readNullable( } @Override - public Object readContainerFieldValue(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValue(MemoryBuffer buffer, SerializationFieldInfo field) { return fory.xreadNonRef(buffer, field.containerClassInfo); } @Override - public Object readContainerFieldValueRef(MemoryBuffer buffer, GenericTypeField field) { + public Object readContainerFieldValueRef(MemoryBuffer buffer, SerializationFieldInfo field) { int nextReadRefId = refResolver.tryPreserveRefId(buffer); if (nextReadRefId >= NOT_NULL_VALUE_FLAG) { Object o = fory.xreadNonRef(buffer, field.containerClassInfo); @@ -457,8 +471,8 @@ public void writeNonRef(MemoryBuffer buffer, Object obj) { } @Override - public void writeNonRef(MemoryBuffer buffer, Object obj, ClassInfo classInfo) { - fory.xwriteNonRef(buffer, obj, classInfo); + public void writeNonRef(MemoryBuffer buffer, Object obj, Serializer serializer) { + fory.xwriteNonRef(buffer, obj, serializer); } @Override diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java index 923974e75a..98247cff92 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/ChildContainerSerializers.java @@ -43,7 +43,8 @@ import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.serializer.AbstractObjectSerializer; -import org.apache.fory.serializer.AbstractObjectSerializer.InternalFieldInfo; +import org.apache.fory.serializer.FieldGroups; +import org.apache.fory.serializer.FieldGroups.SerializationFieldInfo; import org.apache.fory.serializer.JavaSerializer; import org.apache.fory.serializer.MetaSharedLayerSerializer; import org.apache.fory.serializer.ObjectSerializer; @@ -131,7 +132,7 @@ public static class ChildCollectionSerializer ArrayList.class, LinkedList.class, ArrayDeque.class, Vector.class, HashSet.class // PriorityQueue/TreeSet/ConcurrentSkipListSet need comparator as constructor argument ); - protected InternalFieldInfo[] fieldInfos; + protected SerializationFieldInfo[] fieldInfos; protected final Serializer[] slotsSerializers; public ChildCollectionSerializer(Fory fory, Class cls) { @@ -159,7 +160,7 @@ public Collection newCollection(Collection originCollection) { Collection newCollection = super.newCollection(originCollection); if (fieldInfos == null) { List fields = ReflectionUtils.getFieldsWithoutSuperClasses(type, superClasses); - fieldInfos = AbstractObjectSerializer.buildFieldsInfo(fory, fields); + fieldInfos = FieldGroups.buildFieldsInfo(fory, fields).allFields; } AbstractObjectSerializer.copyFields(fory, fieldInfos, originCollection, newCollection); return newCollection; @@ -193,7 +194,7 @@ public static class ChildMapSerializer extends MapSerializer { // TreeMap/ConcurrentSkipListMap need comparator as constructor argument ); private final Serializer[] slotsSerializers; - private InternalFieldInfo[] fieldInfos; + private SerializationFieldInfo[] fieldInfos; public ChildMapSerializer(Fory fory, Class cls) { super(fory, cls); @@ -221,7 +222,7 @@ public Map newMap(Map originMap) { Map newMap = super.newMap(originMap); if (fieldInfos == null || fieldInfos.length == 0) { List fields = ReflectionUtils.getFieldsWithoutSuperClasses(type, superClasses); - fieldInfos = AbstractObjectSerializer.buildFieldsInfo(fory, fields); + fieldInfos = FieldGroups.buildFieldsInfo(fory, fields).allFields; } AbstractObjectSerializer.copyFields(fory, fieldInfos, originMap, newMap); return newMap; diff --git a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java index dbbeb06338..da25315d9b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/DescriptorGrouper.java @@ -82,7 +82,7 @@ public static String getFieldSortKey(Descriptor descriptor) { return c; }; private final Collection descriptors; - private final Predicate isMonomorphic; + private final Predicate isBuildIn; private final Function descriptorUpdater; private final boolean descriptorsGroupedOrdered; private boolean sorted = false; @@ -167,13 +167,13 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole private final Collection collectionDescriptors; // The key/value type should be final. private final Collection mapDescriptors; - private final Collection finalDescriptors; + private final Collection buildInDescriptors; private Collection otherDescriptors; /** * Create a descriptor grouper. * - * @param isMonomorphic whether the class is monomorphic. + * @param isBuildIn whether the class is build-in types. * @param descriptors descriptors may have field with same name. * @param descriptorsGroupedOrdered whether the descriptors are grouped and ordered. * @param descriptorUpdater create a new descriptor from original one. @@ -181,14 +181,14 @@ private static boolean isCompressedType(Class cls, boolean compressInt, boole * @param comparator comparator for non-primitive fields. */ private DescriptorGrouper( - Predicate isMonomorphic, + Predicate isBuildIn, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdater, Comparator primitiveComparator, Comparator comparator) { this.descriptors = descriptors; - this.isMonomorphic = isMonomorphic; + this.isBuildIn = isBuildIn; this.descriptorUpdater = descriptorUpdater; this.descriptorsGroupedOrdered = descriptorsGroupedOrdered; this.primitiveDescriptors = @@ -198,7 +198,7 @@ private DescriptorGrouper( this.collectionDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); this.mapDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); - this.finalDescriptors = + this.buildInDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); this.otherDescriptors = descriptorsGroupedOrdered ? new ArrayList<>() : new TreeSet<>(comparator); @@ -232,8 +232,8 @@ public DescriptorGrouper sort() { collectionDescriptors.add(descriptorUpdater.apply(descriptor)); } else if (TypeUtils.isMap(descriptor.getRawType())) { mapDescriptors.add(descriptorUpdater.apply(descriptor)); - } else if (isMonomorphic.test(descriptor)) { - finalDescriptors.add(descriptorUpdater.apply(descriptor)); + } else if (isBuildIn.test(descriptor)) { + buildInDescriptors.add(descriptorUpdater.apply(descriptor)); } else { otherDescriptors.add(descriptorUpdater.apply(descriptor)); } @@ -247,7 +247,7 @@ public List getSortedDescriptors() { List descriptors = new ArrayList<>(getNumDescriptors()); descriptors.addAll(getPrimitiveDescriptors()); descriptors.addAll(getBoxedDescriptors()); - descriptors.addAll(getFinalDescriptors()); + descriptors.addAll(getBuildInDescriptors()); descriptors.addAll(getCollectionDescriptors()); descriptors.addAll(getMapDescriptors()); descriptors.addAll(getOtherDescriptors()); @@ -274,9 +274,9 @@ public Collection getMapDescriptors() { return mapDescriptors; } - public Collection getFinalDescriptors() { + public Collection getBuildInDescriptors() { Preconditions.checkArgument(sorted); - return finalDescriptors; + return buildInDescriptors; } public Collection getOtherDescriptors() { @@ -297,7 +297,7 @@ private static Descriptor createDescriptor(Descriptor d) { } public static DescriptorGrouper createDescriptorGrouper( - Predicate isMonomorphic, + Predicate isBuildIn, Collection descriptors, boolean descriptorsGroupedOrdered, Function descriptorUpdator, @@ -305,7 +305,7 @@ public static DescriptorGrouper createDescriptorGrouper( boolean compressLong, Comparator comparator) { return new DescriptorGrouper( - isMonomorphic, + isBuildIn, descriptors, descriptorsGroupedOrdered, descriptorUpdator == null ? DescriptorGrouper::createDescriptor : descriptorUpdator, @@ -319,7 +319,7 @@ public int getNumDescriptors() { + boxedDescriptors.size() + collectionDescriptors.size() + mapDescriptors.size() - + finalDescriptors.size() + + buildInDescriptors.size() + otherDescriptors.size(); } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 39968373da..44df2f95cc 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -452,10 +452,9 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableCollectionSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers$UnmodifiableMapSerializer,\ org.apache.fory.serializer.collection.UnmodifiableSerializers,\ - org.apache.fory.serializer.AbstractObjectSerializer$ForyFieldInfo,\ org.apache.fory.serializer.ObjectSerializer,\ - org.apache.fory.serializer.ObjectSerializer$FinalTypeField,\ - org.apache.fory.serializer.ObjectSerializer$GenericTypeField,\ + org.apache.fory.serializer.FieldGroups,\ + org.apache.fory.serializer.FieldGroups.SerializationFieldInfo,\ org.apache.fory.serializer.LazySerializer,\ org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\ org.apache.fory.serializer.shim.ShimDispatcher,\ diff --git a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java index 487680d20f..1f96d17869 100644 --- a/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/type/DescriptorGrouperTest.java @@ -244,7 +244,7 @@ public void testGrouper() { } { List> classes = - grouper.getFinalDescriptors().stream() + grouper.getBuildInDescriptors().stream() .map(Descriptor::getRawType) .collect(Collectors.toList()); assertEquals(classes, Arrays.asList(String.class, Instant.class, Instant.class)); From e36b99ecbb180ea3f4bc6304a31d257ea675c8dd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 22:34:11 +0800 Subject: [PATCH 08/35] move xlang tests dir --- .github/workflows/ci.yml | 12 +++++++----- AGENTS.md | 10 +++++----- .../org/apache/fory/{ => xlang}/CPPXlangTest.java | 6 ++++-- .../org/apache/fory/{ => xlang}/GoXlangTest.java | 4 +++- .../org/apache/fory/xlang/MetaSharedXlangTest.java | 4 ++-- .../PyCrossLanguageTest.java} | 8 +++++--- .../org/apache/fory/{ => xlang}/PythonXlangTest.java | 6 ++++-- .../org/apache/fory/{ => xlang}/RustXlangTest.java | 2 +- .../org/apache/fory/{ => xlang}/XlangTestBase.java | 4 +++- 9 files changed, 34 insertions(+), 22 deletions(-) rename java/fory-core/src/test/java/org/apache/fory/{ => xlang}/CPPXlangTest.java (98%) rename java/fory-core/src/test/java/org/apache/fory/{ => xlang}/GoXlangTest.java (99%) rename java/fory-core/src/test/java/org/apache/fory/{CrossLanguageTest.java => xlang/PyCrossLanguageTest.java} (99%) rename java/fory-core/src/test/java/org/apache/fory/{ => xlang}/PythonXlangTest.java (98%) rename java/fory-core/src/test/java/org/apache/fory/{ => xlang}/RustXlangTest.java (99%) rename java/fory-core/src/test/java/org/apache/fory/{ => xlang}/XlangTestBase.java (99%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2487372b..42f8607e2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -326,7 +326,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.RustXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.RustXlangTest cpp: name: C++ CI @@ -412,7 +412,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CPPXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.CPPXlangTest cpp_examples: name: C++ Examples @@ -533,8 +533,10 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.PythonXlangTest - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.CrossLanguageTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PythonXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.PyCrossLanguageTest + cd ../fory-format + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.format.CrossLanguageTest go: name: Golang CI @@ -599,7 +601,7 @@ jobs: cd java mvn -T16 --no-transfer-progress clean install -DskipTests cd fory-core - mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.GoXlangTest + mvn -T16 --no-transfer-progress test -Dtest=org.apache.fory.xlang.GoXlangTest lint: name: Code Style Check diff --git a/AGENTS.md b/AGENTS.md index 2322cb945a..9202efd6a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,7 @@ Run C++ xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.CPPXlangTest +FORY_CPP_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.CPPXlangTest ``` ### Python Development @@ -125,9 +125,9 @@ cd java mvn -T16 install -DskipTests cd fory-core # disable fory cython for faster debugging -FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest +FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest # enable fory cython -FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.PythonXlangTest +FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest ``` ### Golang Development @@ -159,7 +159,7 @@ Run Go xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.GoXlangTest +FORY_GO_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.GoXlangTest ``` ### Rust Development @@ -215,7 +215,7 @@ Run Rust xlang tests: cd java mvn -T16 install -DskipTests cd fory-core -FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.RustXlangTest +FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.RustXlangTest ``` ### JavaScript/TypeScript Development diff --git a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java similarity index 98% rename from java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index 7097e50cce..8dd2c97cca 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -29,6 +29,8 @@ import java.util.HashSet; import java.util.List; import java.util.concurrent.TimeUnit; + +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; @@ -46,7 +48,7 @@ public class CPPXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_CPP_JAVA_CI"); if (!"1".equals(enabled)) { - // throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); + throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); } boolean bazelAvailable = true; try { diff --git a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index 919b250838..dde5a2878a 100644 --- a/java/fory-core/src/test/java/org/apache/fory/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -29,6 +29,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; + +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 903f63f68d..2cf950065b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -20,8 +20,8 @@ package org.apache.fory.xlang; import lombok.Data; -import org.apache.fory.CrossLanguageTest.Bar; -import org.apache.fory.CrossLanguageTest.Foo; +import org.apache.fory.xlang.PyCrossLanguageTest.Bar; +import org.apache.fory.xlang.PyCrossLanguageTest.Foo; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; diff --git a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java index 47fef66f8f..6d22bbae36 100644 --- a/java/fory-core/src/test/java/org/apache/fory/CrossLanguageTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PyCrossLanguageTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import static org.testng.Assert.assertEquals; @@ -56,6 +56,8 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.ForyBuilder; import org.apache.fory.config.Language; @@ -82,8 +84,8 @@ /** Tests in this class need fory python installed. */ @Test -public class CrossLanguageTest extends ForyTestBase { - private static final Logger LOG = LoggerFactory.getLogger(CrossLanguageTest.class); +public class PyCrossLanguageTest extends ForyTestBase { + private static final Logger LOG = LoggerFactory.getLogger(PyCrossLanguageTest.class); private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language"; private static final String PYTHON_EXECUTABLE = "python"; diff --git a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java similarity index 98% rename from java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java index d079c883ed..4f599cc4fd 100644 --- a/java/fory-core/src/test/java/org/apache/fory/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; @@ -28,6 +28,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; + +import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.memory.MemoryBuffer; @@ -137,7 +139,7 @@ public void testInteger() throws IOException { // ============================================================================ // Explicitly re-declare inherited test methods to enable running individual - // tests via Maven: mvn test -Dtest=org.apache.fory.PythonXlangTest#testXxx + // tests via Maven: mvn test -Dtest=org.apache.fory.xlang.PythonXlangTest#testXxx // // Maven Surefire cannot find inherited test methods when using the #methodName // syntax for test selection. By overriding and forwarding to the parent class, diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index 60c00a0024..278aa9ec3e 100644 --- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import java.io.File; diff --git a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java similarity index 99% rename from java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java rename to java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index be2fa72ab5..9b9f722041 100644 --- a/java/fory-core/src/test/java/org/apache/fory/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.fory; +package org.apache.fory.xlang; import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; @@ -33,6 +33,8 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; import lombok.Data; +import org.apache.fory.Fory; +import org.apache.fory.ForyTestBase; import org.apache.fory.annotation.ForyField; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; From d006d97a25eac2fdd67094be9517d698d6ebb728 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 23:03:09 +0800 Subject: [PATCH 09/35] amek all xlang tests cover codegen --- .../apache/fory/resolver/ClassResolver.java | 26 +- .../apache/fory/resolver/TypeResolver.java | 25 +- .../apache/fory/resolver/XtypeResolver.java | 19 ++ .../serializer/AbstractObjectSerializer.java | 2 + .../fory/serializer/MetaSharedSerializer.java | 10 +- .../NonexistentClassSerializers.java | 13 +- .../org/apache/fory/xlang/CPPXlangTest.java | 190 +++++++------- .../org/apache/fory/xlang/GoXlangTest.java | 188 +++++++------- .../apache/fory/xlang/PythonXlangTest.java | 129 +++++----- .../org/apache/fory/xlang/RustXlangTest.java | 192 +++++++------- .../org/apache/fory/xlang/XlangTestBase.java | 234 ++++++++---------- 11 files changed, 512 insertions(+), 516 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 5ffc874360..3c13eb92d9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -78,6 +78,7 @@ import org.apache.fory.Fory; import org.apache.fory.ForyCopyable; import org.apache.fory.annotation.CodegenInvoke; +import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; @@ -687,6 +688,22 @@ public String getTypeAlias(Class cls) { return cls.getName(); } + @Override + public boolean isMonomorphic(Descriptor descriptor) { + ForyField foryField = descriptor.getForyField(); + if (foryField != null) { + switch (foryField.morphic()) { + case POLYMORPHIC: + return false; + case FINAL: + return true; + default: + return isMonomorphic(descriptor.getRawType()); + } + } + return isMonomorphic(descriptor.getRawType()); + } + /** * Mark non-inner registered final types as non-final to write class def for those types. Note if * a class is registered but not an inner class with inner serializer, it will still be taken as @@ -1798,15 +1815,6 @@ public void resetRead() {} public void resetWrite() {} - private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); - - @CodegenInvoke - public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { - Map map = - extRegistry.classGenericTypes.computeIfAbsent(cls, this::buildGenericMap); - return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); - } - @Override public GenericType buildGenericType(TypeRef typeRef) { return GenericType.build( diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index b7b1cb4d61..f66f0550da 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -37,6 +37,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import org.apache.fory.Fory; +import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; @@ -97,6 +98,7 @@ public abstract class TypeResolver { static final String SET_META__CONTEXT_MSG = "Meta context must be set before serialization, " + "please set meta context by SerializationContext.setMetaContext"; + private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class); final Fory fory; final boolean metaContextShareEnabled; @@ -225,20 +227,7 @@ public final boolean needToWriteClassDef(Serializer serializer) { public abstract boolean isBuildIn(Descriptor descriptor); - public boolean isMonomorphic(Descriptor descriptor) { - ForyField foryField = descriptor.getForyField(); - if (foryField != null) { - switch (foryField.morphic()) { - case POLYMORPHIC: - return false; - case FINAL: - return true; - default: - return isMonomorphic(descriptor.getRawType()); - } - } - return isMonomorphic(descriptor.getRawType()); - } + public abstract boolean isMonomorphic(Descriptor descriptor); public abstract boolean isMonomorphic(Class clz); @@ -496,6 +485,14 @@ final Class loadClass( public abstract GenericType buildGenericType(Type type); + @CodegenInvoke + public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { + Map map = + extRegistry.classGenericTypes.computeIfAbsent(cls, this::buildGenericMap); + return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); + } + + public abstract void initialize(); public abstract ClassDef getTypeDef(Class cls, boolean resolveParent); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index f213c0c0b0..cb6c52520c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -53,6 +53,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import org.apache.fory.Fory; +import org.apache.fory.annotation.ForyField; import org.apache.fory.annotation.Internal; import org.apache.fory.collection.IdentityObjectIntMap; import org.apache.fory.collection.LongMap; @@ -433,6 +434,24 @@ public boolean isRegisteredByName(Class cls) { } } + @Override + public boolean isMonomorphic(Descriptor descriptor) { + ForyField foryField = descriptor.getForyField(); + ForyField.Morphic morphic = foryField != null ? foryField.morphic() : ForyField.Morphic.AUTO; + switch (morphic) { + case POLYMORPHIC: + return false; + case FINAL: + return true; + default: + Class rawType = descriptor.getRawType(); + if (rawType.isEnum()) { + return true; + } + return !ReflectionUtils.isAbstract(rawType); + } + } + @Override public boolean isMonomorphic(Class clz) { if (TypeUtils.unwrap(clz).isPrimitive() || clz.isEnum() || clz == String.class) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 1d485c966c..437584f8b3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -47,6 +47,7 @@ public abstract class AbstractObjectSerializer extends Serializer { protected final RefResolver refResolver; protected final ClassResolver classResolver; + protected final TypeResolver typeResolver; protected final boolean isRecord; protected final ObjectCreator objectCreator; private FieldGroups.SerializationFieldInfo[] fieldInfos; @@ -60,6 +61,7 @@ public AbstractObjectSerializer(Fory fory, Class type, ObjectCreator objec super(fory, type); this.refResolver = fory.getRefResolver(); this.classResolver = fory.getClassResolver(); + this.typeResolver = fory._getTypeResolver(); this.isRecord = RecordUtils.isRecord(type); this.objectCreator = objectCreator; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index cafe11e802..60c4044572 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -150,6 +150,7 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { @Override public void write(MemoryBuffer buffer, T value) { if (serializer == null) { + // xlang mode will register class and create serializer in advance, it won't go to here. serializer = this.classResolver.createSerializerSafe(type, () -> new ObjectSerializer<>(fory, type)); } @@ -175,7 +176,7 @@ public T read(MemoryBuffer buffer) { T obj = newInstance(); Fory fory = this.fory; RefResolver refResolver = this.refResolver; - ClassResolver classResolver = this.classResolver; + TypeResolver typeResolver = this.typeResolver; SerializationBinding binding = this.binding; refResolver.reference(obj); // read order: primitive,boxed,final,other,collection,map @@ -201,7 +202,7 @@ public T read(MemoryBuffer buffer) { assert fieldInfo.classInfo != null; Object fieldValue = AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + binding, refResolver, typeResolver, fieldInfo, buffer); fieldAccessor.putObject(obj, fieldValue); } } else { @@ -213,7 +214,7 @@ public T read(MemoryBuffer buffer) { binding.readRef(buffer, classInfoHolder); } else { AbstractObjectSerializer.readFinalObjectFieldValue( - binding, refResolver, classResolver, fieldInfo, buffer); + binding, refResolver, typeResolver, fieldInfo, buffer); } } } else { @@ -276,8 +277,7 @@ private void readFields(MemoryBuffer buffer, Object[] fields) { ClassResolver classResolver = this.classResolver; SerializationBinding binding = this.binding; // read order: primitive,boxed,final,other,collection,map - SerializationFieldInfo[] finalFields = this.buildInFields; - for (SerializationFieldInfo fieldInfo : finalFields) { + for (SerializationFieldInfo fieldInfo : this.buildInFields) { if (fieldInfo.fieldAccessor != null) { assert fieldInfo.classInfo != null; short classId = fieldInfo.classId; diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 81c362bd2f..83366f1286 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -50,13 +50,13 @@ public final class NonexistentClassSerializers { private static final class ClassFieldsInfo { - private final SerializationFieldInfo[] finalFields; + private final SerializationFieldInfo[] buildInFields; private final SerializationFieldInfo[] otherFields; private final SerializationFieldInfo[] containerFields; private final int classVersionHash; private ClassFieldsInfo(FieldGroups fieldGroups, int classVersionHash) { - this.finalFields = fieldGroups.buildInFields; + this.buildInFields = fieldGroups.buildInFields; this.otherFields = fieldGroups.userTypeFields; this.containerFields = fieldGroups.containerFields; this.classVersionHash = classVersionHash; @@ -68,7 +68,6 @@ public static final class NonexistentClassSerializer extends Serializer { private final ClassInfoHolder classInfoHolder; private final LongMap fieldsInfoMap; private final SerializationBinding binding; - private final TypeResolver typeResolver; public NonexistentClassSerializer(Fory fory, ClassDef classDef) { super(fory, NonexistentClass.NonexistentMetaShared.class); @@ -76,7 +75,6 @@ public NonexistentClassSerializer(Fory fory, ClassDef classDef) { classInfoHolder = fory.getClassResolver().nilClassInfoHolder(); fieldsInfoMap = new LongMap<>(); binding = SerializationBinding.createBinding(fory); - typeResolver = fory._getTypeResolver(); Preconditions.checkArgument(fory.getConfig().isMetaShareEnabled()); } @@ -116,8 +114,7 @@ public void write(MemoryBuffer buffer, Object v) { buffer.writeInt32(fieldsInfo.classVersionHash); } // write order: primitive,boxed,final,other,collection,map - SerializationFieldInfo[] finalFields = fieldsInfo.finalFields; - for (SerializationFieldInfo fieldInfo : finalFields) { + for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { Object fieldValue = value.get(fieldInfo.qualifiedFieldName); ClassInfo classInfo = fieldInfo.classInfo; if (classResolver.isPrimitive(fieldInfo.classId)) { @@ -177,9 +174,7 @@ public Object read(MemoryBuffer buffer) { List entries = new ArrayList<>(); // read order: primitive,boxed,final,other,collection,map ClassFieldsInfo fieldsInfo = getClassFieldsInfo(classDef); - SerializationFieldInfo[] finalFields = fieldsInfo.finalFields; - for (int i = 0; i < finalFields.length; i++) { - SerializationFieldInfo fieldInfo = finalFields[i]; + for (SerializationFieldInfo fieldInfo : fieldsInfo.buildInFields) { Object fieldValue; if (fieldInfo.classInfo == null) { // TODO(chaokunyang) support registered serializer in peer with ref tracking disabled. diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index 8dd2c97cca..279ad261fb 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -153,152 +153,152 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test - public void testStringSerializer() throws Exception { - super.testStringSerializer(); + @Test(dataProvider = "enableCodegen") + public void testStringSerializer(boolean enableCodegen) throws Exception { + super.testStringSerializer(enableCodegen); } - @Test - public void testCrossLanguageSerializer() throws Exception { - super.testCrossLanguageSerializer(); + @Test(dataProvider = "enableCodegen") + public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { + super.testCrossLanguageSerializer(enableCodegen); } - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); } - @Test - public void testList() throws java.io.IOException { - super.testList(); + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); } - @Test - public void testMap() throws java.io.IOException { - super.testMap(); + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); } - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); } - @Test - public void testItem() throws java.io.IOException { - super.testItem(); + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); } - @Test - public void testColor() throws java.io.IOException { - super.testColor(); + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); } - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); } - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); } - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); } - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); } - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); } - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); } - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); } - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); } - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { - super.testEnumSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testEnumSchemaEvolutionCompatible(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { // C++ has proper std::optional support and sends actual null values, // unlike Rust which sends default values. Override with C++-specific expectations. String caseName = "test_nullable_field_compatible_null"; @@ -306,7 +306,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -364,20 +364,20 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Assert.assertEquals(result, obj); } - @Test @Override - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { // Skip: C++ doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: C++ Union xlang support not implemented"); } - @Test - public void testRefSchemaConsistent() throws java.io.IOException { - super.testRefSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testRefSchemaConsistent(enableCodegen); } - @Test - public void testRefCompatible() throws java.io.IOException { - super.testRefCompatible(); + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index dde5a2878a..e4a548f0e9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -117,129 +117,129 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test - public void testStringSerializer() throws Exception { - super.testStringSerializer(); + @Test(dataProvider = "enableCodegen") + public void testStringSerializer(boolean enableCodegen) throws Exception { + super.testStringSerializer(enableCodegen); } - @Test - public void testCrossLanguageSerializer() throws Exception { - super.testCrossLanguageSerializer(); + @Test(dataProvider = "enableCodegen") + public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { + super.testCrossLanguageSerializer(enableCodegen); } - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); } - @Test - public void testList() throws java.io.IOException { - super.testList(); + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); } - @Test - public void testMap() throws java.io.IOException { - super.testMap(); + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); } - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); } - @Test - public void testItem() throws java.io.IOException { - super.testItem(); + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); } - @Test - public void testColor() throws java.io.IOException { - super.testColor(); + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); } - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); } - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); } - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); } - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); } - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); } - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); } - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); } - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); } - @Test @Override - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { // Go-specific override: Go writes null for nil pointers (nullable=true by default) String caseName = "test_enum_schema_evolution_compatible"; // Fory for TwoEnumFieldStruct @@ -304,26 +304,26 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } - @Test @Override - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { // Go-specific override: Unlike Rust which has non-nullable reference types (Vec), // Go's slices and maps can be nil and default to nullable in COMPATIBLE mode. // So Go sends null for nil values, not empty collections like Rust does. @@ -332,7 +332,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -426,23 +426,23 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Assert.assertEquals(result, expected); } - @Test @Override - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { // Skip: Go doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Go Union xlang support not implemented"); } @Override - @Test - public void testRefSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { // Run the test to debug hash mismatch - super.testRefSchemaConsistent(); + super.testRefSchemaConsistent(enableCodegen); } @Override - @Test - public void testRefCompatible() throws java.io.IOException { - super.testRefCompatible(); + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java index 4f599cc4fd..0b1369c5c6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java @@ -72,60 +72,59 @@ protected CommandContext buildCommandContext(String caseName, Path dataFile) { // ============================================================================ @Override - @Test + @Test(dataProvider = "enableCodegen") public void testBuffer() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test + @Test(dataProvider = "enableCodegen") public void testMurmurHash3() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testCrossLanguageSerializer() throws Exception { + @Test(dataProvider = "enableCodegen") + public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testList() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testMap() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test - public void testItem() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: simple struct tests covered in CrossLanguageTest"); } @Override - @Test - public void testColor() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: enum tests covered in CrossLanguageTest"); } @Override - @Test - public void testStructWithList() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: struct with list covered in CrossLanguageTest"); } @Override - @Test - public void testStructWithMap() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: struct with map covered in CrossLanguageTest"); } - @Override @Test public void testBufferVar() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); @@ -133,7 +132,7 @@ public void testBufferVar() throws IOException { @Override @Test - public void testInteger() throws IOException { + public void testInteger(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @@ -147,80 +146,80 @@ public void testInteger() throws IOException { // ============================================================================ @Override - @Test - public void testStringSerializer() throws Exception { - super.testStringSerializer(); + @Test(dataProvider = "enableCodegen") + public void testStringSerializer(boolean enableCodegen) throws Exception { + super.testStringSerializer(enableCodegen); } @Override - @Test - public void testSimpleStruct() throws IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws IOException { + super.testSimpleStruct(enableCodegen); } @Override - @Test - public void testSimpleNamedStruct() throws IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws IOException { + super.testSimpleNamedStruct(enableCodegen); } @Override - @Test - public void testSkipIdCustom() throws IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws IOException { + super.testSkipIdCustom(enableCodegen); } @Override - @Test - public void testSkipNameCustom() throws IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws IOException { + super.testSkipNameCustom(enableCodegen); } @Override - @Test - public void testConsistentNamed() throws IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws IOException { + super.testConsistentNamed(enableCodegen); } @Override - @Test - public void testStructVersionCheck() throws IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws IOException { + super.testStructVersionCheck(enableCodegen); } @Override - @Test - public void testPolymorphicList() throws IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws IOException { + super.testPolymorphicList(enableCodegen); } @Override - @Test - public void testPolymorphicMap() throws IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws IOException { + super.testPolymorphicMap(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNotNull() throws IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldSchemaConsistentNull() throws IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNull() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws IOException { // Python properly supports Optional and sends actual null values, // unlike Rust which sends default values. Override with Python-specific expectations. String caseName = "test_nullable_field_compatible_null"; @@ -228,7 +227,7 @@ public void testNullableFieldCompatibleNull() throws IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -287,19 +286,19 @@ public void testNullableFieldCompatibleNull() throws IOException { } @Override - @Test - public void testUnionXlang() throws IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws IOException { // Skip: Python doesn't have Union xlang support yet throw new SkipException("Skipping testUnionXlang: Python Union xlang support not implemented"); } - @Test - public void testRefSchemaConsistent() throws IOException { - super.testRefSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws IOException { + super.testRefSchemaConsistent(enableCodegen); } - @Test - public void testRefCompatible() throws IOException { - super.testRefCompatible(); + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws IOException { + super.testRefCompatible(enableCodegen); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index 278aa9ec3e..c36d1940e2 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -106,165 +106,165 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test - public void testStringSerializer() throws Exception { - super.testStringSerializer(); + @Test(dataProvider = "enableCodegen") + public void testStringSerializer(boolean enableCodegen) throws Exception { + super.testStringSerializer(enableCodegen); } - @Test - public void testCrossLanguageSerializer() throws Exception { - super.testCrossLanguageSerializer(); + @Test(dataProvider = "enableCodegen") + public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { + super.testCrossLanguageSerializer(enableCodegen); } - @Test - public void testSimpleStruct() throws java.io.IOException { - super.testSimpleStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleStruct(enableCodegen); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { - super.testSimpleNamedStruct(); + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { + super.testSimpleNamedStruct(enableCodegen); } - @Test - public void testList() throws java.io.IOException { - super.testList(); + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { + super.testList(enableCodegen); } - @Test - public void testMap() throws java.io.IOException { - super.testMap(); + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { + super.testMap(enableCodegen); } - @Test - public void testInteger() throws java.io.IOException { - super.testInteger(); + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { + super.testInteger(enableCodegen); } - @Test - public void testItem() throws java.io.IOException { - super.testItem(); + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { + super.testItem(enableCodegen); } - @Test - public void testColor() throws java.io.IOException { - super.testColor(); + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { + super.testColor(enableCodegen); } - @Test - public void testStructWithList() throws java.io.IOException { - super.testStructWithList(); + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { + super.testStructWithList(enableCodegen); } - @Test - public void testStructWithMap() throws java.io.IOException { - super.testStructWithMap(); + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { + super.testStructWithMap(enableCodegen); } - @Test - public void testSkipIdCustom() throws java.io.IOException { - super.testSkipIdCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipIdCustom(enableCodegen); } - @Test - public void testSkipNameCustom() throws java.io.IOException { - super.testSkipNameCustom(); + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { + super.testSkipNameCustom(enableCodegen); } - @Test - public void testConsistentNamed() throws java.io.IOException { - super.testConsistentNamed(); + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { + super.testConsistentNamed(enableCodegen); } - @Test - public void testStructVersionCheck() throws java.io.IOException { - super.testStructVersionCheck(); + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { + super.testStructVersionCheck(enableCodegen); } - @Test - public void testPolymorphicList() throws java.io.IOException { - super.testPolymorphicList(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicList(enableCodegen); } - @Test - public void testPolymorphicMap() throws java.io.IOException { - super.testPolymorphicMap(); + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { + super.testPolymorphicMap(enableCodegen); } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { - super.testOneStringFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { - super.testOneStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneStringFieldCompatible(enableCodegen); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { - super.testTwoStringFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoStringFieldCompatible(enableCodegen); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { - super.testSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { - super.testOneEnumFieldSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldSchemaConsistent(enableCodegen); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { - super.testOneEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testOneEnumFieldCompatible(enableCodegen); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { - super.testTwoEnumFieldCompatible(); + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { + super.testTwoEnumFieldCompatible(enableCodegen); } - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { - super.testEnumSchemaEvolutionCompatible(); + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { + super.testEnumSchemaEvolutionCompatible(enableCodegen); } - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { - super.testNullableFieldSchemaConsistentNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldSchemaConsistentNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { - super.testNullableFieldCompatibleNotNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNotNull(enableCodegen); } @Override - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { - super.testNullableFieldCompatibleNull(); + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { + super.testNullableFieldCompatibleNull(enableCodegen); } - @Test - public void testUnionXlang() throws java.io.IOException { - super.testUnionXlang(); + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { + super.testUnionXlang(enableCodegen); } - @Test - public void testRefSchemaConsistent() throws java.io.IOException { - super.testRefSchemaConsistent(); + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { + super.testRefSchemaConsistent(enableCodegen); } - @Test - public void testRefCompatible() throws java.io.IOException { - super.testRefCompatible(); + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { + super.testRefCompatible(enableCodegen); } } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index 9b9f722041..cd331bf234 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -416,19 +416,21 @@ private void _testStringSerializer(Fory fory, String caseName) throws IOExceptio } } - @Test - public void testStringSerializer() throws Exception { + @Test(dataProvider = "enableCodegen") + public void testStringSerializer(boolean enableCodegen) throws Exception { String caseName = "test_string_serializer"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); _testStringSerializer(fory, caseName); Fory foryCompress = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .withStringCompressed(true) .withWriteNumUtf16BytesForUtf8Encoding(false) .build(); @@ -442,8 +444,8 @@ enum Color { White, } - @Test - public void testCrossLanguageSerializer() throws Exception { + @Test(dataProvider = "enableCodegen") + public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { String caseName = "test_cross_language_serializer"; List strList = Arrays.asList("hello", "world"); Set strSet = new HashSet<>(strList); @@ -456,6 +458,7 @@ public void testCrossLanguageSerializer() throws Exception { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, 101); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -544,14 +547,14 @@ static class SimpleStruct { int last; // Changed from Integer to int to match Rust } - @Test - public void testSimpleStruct() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSimpleStruct(boolean enableCodegen) throws java.io.IOException { String caseName = "test_simple_struct"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, 101); fory.register(Item.class, 102); @@ -582,14 +585,14 @@ public void testSimpleStruct() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), obj); } - @Test - public void testSimpleNamedStruct() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSimpleNamedStruct(boolean enableCodegen) throws java.io.IOException { String caseName = "test_named_simple_struct"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, "demo", "color"); fory.register(Item.class, "demo", "item"); @@ -620,14 +623,14 @@ public void testSimpleNamedStruct() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), obj); } - @Test - public void testList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -655,14 +658,14 @@ public void testList() throws java.io.IOException { assertEqualsNullTolerant(fory.deserialize(buffer2), itemList2); } - @Test - public void testMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); MemoryBuffer buffer = MemoryUtils.buffer(64); @@ -700,13 +703,13 @@ static class Item1 { Integer f6; } - @Test - public void testInteger() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testInteger(boolean enableCodegen) throws java.io.IOException { String caseName = "test_integer"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) - .withCodegen(false) + .withCodegen(enableCodegen) .withCompatibleMode(CompatibleMode.COMPATIBLE) .build(); fory.register(Item1.class, 101); @@ -754,14 +757,14 @@ public void testInteger() throws java.io.IOException { Assert.assertEquals(fory.deserialize(buffer2), 0); } - @Test - public void testItem() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testItem(boolean enableCodegen) throws java.io.IOException { String caseName = "test_item"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Item.class, 102); @@ -792,14 +795,14 @@ public void testItem() throws java.io.IOException { Assert.assertEquals(readItem3.name, ""); } - @Test - public void testColor() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testColor(boolean enableCodegen) throws java.io.IOException { String caseName = "test_color"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Color.class, 101); @@ -832,14 +835,14 @@ static class StructWithUnion2 { org.apache.fory.type.union.Union2 union; } - @Test - public void testUnionXlang() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testUnionXlang(boolean enableCodegen) throws java.io.IOException { String caseName = "test_union_xlang"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithUnion2.class, 301); @@ -873,14 +876,14 @@ static class StructWithList { List items; } - @Test - public void testStructWithList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_with_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithList.class, 201); @@ -910,14 +913,14 @@ static class StructWithMap { Map data; } - @Test - public void testStructWithMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructWithMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_with_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(StructWithMap.class, 202); @@ -1012,8 +1015,7 @@ private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOE MyWrapper wrapper = new MyWrapper(); wrapper.color = Color.White; MyStruct myStruct = new MyStruct(42); - MyExt myExt = new MyExt(43); - wrapper.myExt = myExt; + wrapper.myExt = new MyExt(43); wrapper.myStruct = myStruct; byte[] serialize = fory1.serialize(wrapper); ExecutionContext ctx = prepareExecution(caseName, serialize); @@ -1023,14 +1025,14 @@ private void _testSkipCustom(Fory fory1, Fory fory2, String caseName) throws IOE Assert.assertEquals(newWrapper, new EmptyWrapper()); } - @Test - public void testSkipIdCustom() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSkipIdCustom(boolean enableCodegen) throws java.io.IOException { String caseName = "test_skip_id_custom"; Fory fory1 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory1.register(Color.class, 101); fory1.register(MyStruct.class, 102); @@ -1041,7 +1043,7 @@ public void testSkipIdCustom() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory2.register(MyExt.class, 103); fory2.registerSerializer(MyExt.class, MyExtSerializer.class); @@ -1049,14 +1051,14 @@ public void testSkipIdCustom() throws java.io.IOException { _testSkipCustom(fory1, fory2, caseName); } - @Test - public void testSkipNameCustom() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSkipNameCustom(boolean enableCodegen) throws java.io.IOException { String caseName = "test_skip_name_custom"; Fory fory1 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory1.register(Color.class, "color"); fory1.register(MyStruct.class, "my_struct"); @@ -1067,7 +1069,7 @@ public void testSkipNameCustom() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory2.register(MyExt.class, "my_ext"); fory2.registerSerializer(MyExt.class, MyExtSerializer.class); @@ -1075,14 +1077,14 @@ public void testSkipNameCustom() throws java.io.IOException { _testSkipCustom(fory1, fory2, caseName); } - @Test - public void testConsistentNamed() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testConsistentNamed(boolean enableCodegen) throws java.io.IOException { String caseName = "test_consistent_named"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .withClassVersionCheck(true) .build(); fory.register(Color.class, "color"); @@ -1129,14 +1131,14 @@ static class VersionCheckStruct { double f3; } - @Test - public void testStructVersionCheck() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testStructVersionCheck(boolean enableCodegen) throws java.io.IOException { String caseName = "test_struct_version_check"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .withClassVersionCheck(true) .build(); fory.register(VersionCheckStruct.class, 201); @@ -1211,14 +1213,14 @@ static class AnimalMapHolder { Map animal_map; } - @Test - public void testPolymorphicList() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testPolymorphicList(boolean enableCodegen) throws java.io.IOException { String caseName = "test_polymorphic_list"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); // Register concrete types, not the interface fory.register(Dog.class, 302); @@ -1270,14 +1272,14 @@ public void testPolymorphicList() throws java.io.IOException { Assert.assertEquals(((Cat) readHolder.animals.get(1)).lives, 7); } - @Test - public void testPolymorphicMap() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testPolymorphicMap(boolean enableCodegen) throws java.io.IOException { String caseName = "test_polymorphic_map"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(Dog.class, 302); fory.register(Cat.class, 303); @@ -1368,13 +1370,14 @@ static class TwoStringFieldStruct { String f2; } - @Test - public void testOneStringFieldSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_string_field_schema"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) .build(); fory.register(OneStringFieldStruct.class, 200); @@ -1392,13 +1395,14 @@ public void testOneStringFieldSchemaConsistent() throws java.io.IOException { Assert.assertEquals(result.f1, "hello"); } - @Test - public void testOneStringFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_string_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(OneStringFieldStruct.class, 200); @@ -1416,13 +1420,14 @@ public void testOneStringFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f1, "hello"); } - @Test - public void testTwoStringFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testTwoStringFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_two_string_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TwoStringFieldStruct.class, 201); @@ -1442,14 +1447,15 @@ public void testTwoStringFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f2, "second"); } - @Test - public void testSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_schema_evolution_compatible"; // Fory for TwoStringFieldStruct Fory fory2 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory2.register(TwoStringFieldStruct.class, 200); @@ -1458,6 +1464,7 @@ public void testSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); foryEmpty.register(EmptyStruct.class, 200); @@ -1465,6 +1472,7 @@ public void testSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory1.register(OneStringFieldStruct.class, 200); @@ -1524,13 +1532,14 @@ static class TwoEnumFieldStruct { TestEnum f2; } - @Test - public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldSchemaConsistent(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_enum_field_schema"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(OneEnumFieldStruct.class, 211); @@ -1549,13 +1558,14 @@ public void testOneEnumFieldSchemaConsistent() throws java.io.IOException { Assert.assertEquals(result.f1, TestEnum.VALUE_B); } - @Test - public void testOneEnumFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testOneEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_one_enum_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(OneEnumFieldStruct.class, 211); @@ -1574,13 +1584,14 @@ public void testOneEnumFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f1, TestEnum.VALUE_A); } - @Test - public void testTwoEnumFieldCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testTwoEnumFieldCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_two_enum_field_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory.register(TestEnum.class, 210); fory.register(TwoEnumFieldStruct.class, 212); @@ -1601,14 +1612,15 @@ public void testTwoEnumFieldCompatible() throws java.io.IOException { Assert.assertEquals(result.f2, TestEnum.VALUE_C); } - @Test - public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_enum_schema_evolution_compatible"; // Fory for TwoEnumFieldStruct Fory fory2 = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory2.register(TestEnum.class, 210); fory2.register(TwoEnumFieldStruct.class, 211); @@ -1618,6 +1630,7 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); foryEmpty.register(TestEnum.class, 210); foryEmpty.register(EmptyStruct.class, 211); @@ -1626,6 +1639,7 @@ public void testEnumSchemaEvolutionCompatible() throws java.io.IOException { Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) + .withCodegen(enableCodegen) .build(); fory1.register(TestEnum.class, 210); fory1.register(OneEnumFieldStruct.class, 211); @@ -1734,14 +1748,14 @@ static class NullableComprehensiveSchemaConsistent { Map nullableMap; } - @Test - public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_not_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(NullableComprehensiveSchemaConsistent.class, 401); @@ -1792,14 +1806,14 @@ public void testNullableFieldSchemaConsistentNotNull() throws java.io.IOExceptio Assert.assertEquals(result, obj); } - @Test - public void testNullableFieldSchemaConsistentNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(NullableComprehensiveSchemaConsistent.class, 401); @@ -1917,14 +1931,14 @@ static class NullableComprehensiveCompatible { Map nullableMap2; } - @Test - public void testNullableFieldCompatibleNotNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNotNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_compatible_not_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -1983,14 +1997,14 @@ public void testNullableFieldCompatibleNotNull() throws java.io.IOException { Assert.assertEquals(result, obj); } - @Test - public void testNullableFieldCompatibleNull() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testNullableFieldCompatibleNull(boolean enableCodegen) throws java.io.IOException { String caseName = "test_nullable_field_compatible_null"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(NullableComprehensiveCompatible.class, 402); @@ -2082,43 +2096,7 @@ public void testNullableFieldCompatibleNull() throws java.io.IOException { Assert.assertEquals(result, expected); } - // Keep the old simple structs for backward compatibility with existing tests - @Data - static class NullableFieldStruct { - int intField; - long longField; - float floatField; - double doubleField; - boolean boolField; - String stringField; - - @ForyField(nullable = true) - String nullableString1; - - @ForyField(nullable = true) - String nullableString2; - } - - @Data - static class NullableFieldStructCompatible { - int intField; - long longField; - float floatField; - double doubleField; - boolean boolField; - String stringField; - - @ForyField(nullable = true) - String nullableString1; - - @ForyField(nullable = true) - String nullableString2; - - @ForyField(nullable = true) - String nullableString3; - } - - @SuppressWarnings("unchecked") + @SuppressWarnings({"unchecked", "rawtypes"}) private void assertStringEquals(Object actual, Object expected, boolean useToString) { if (useToString) { if (expected instanceof Map) { @@ -2158,7 +2136,6 @@ private void assertStringEquals(Object actual, Object expected, boolean useToStr * null strings become empty strings "". This method first tries direct comparison, then * normalizes null to empty string and compares again. Prints normalized values on mismatch. */ - @SuppressWarnings("unchecked") protected void assertEqualsNullTolerant(Object actual, Object expected) { // First try direct comparison if (Objects.equals(actual, expected)) { @@ -2218,15 +2195,15 @@ static class RefOuterSchemaConsistent { * with two fields pointing to the same inner struct instance. Verifies that after * serialization/deserialization across languages, both fields still reference the same object. */ - @Test - public void testRefSchemaConsistent() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testRefSchemaConsistent(boolean enableCodegen) throws java.io.IOException { String caseName = "test_ref_schema_consistent"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT) .withRefTracking(true) - .withCodegen(false) + .withCodegen(enableCodegen) .build(); fory.register(RefInnerSchemaConsistent.class, 501); fory.register(RefOuterSchemaConsistent.class, 502); @@ -2295,15 +2272,15 @@ static class RefOuterCompatible { * fields pointing to the same inner struct instance. Verifies that after * serialization/deserialization across languages, both fields still reference the same object. */ - @Test - public void testRefCompatible() throws java.io.IOException { + @Test(dataProvider = "enableCodegen") + public void testRefCompatible(boolean enableCodegen) throws java.io.IOException { String caseName = "test_ref_compatible"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) .withRefTracking(true) - .withCodegen(false) + .withCodegen(enableCodegen) .withMetaCompressor(new NoOpMetaCompressor()) .build(); fory.register(RefInnerCompatible.class, 503); @@ -2346,7 +2323,6 @@ public void testRefCompatible() throws java.io.IOException { } /** Normalize null values to empty strings in collections and maps recursively. */ - @SuppressWarnings("unchecked") private Object normalizeNulls(Object obj) { if (obj == null) { return ""; From fe0ad1f38eb52b5293dd02e25a1e4262131cf046 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Fri, 2 Jan 2026 23:59:52 +0800 Subject: [PATCH 10/35] fix isBuildIn check --- .../fory/builder/MetaSharedCodecBuilder.java | 6 +----- .../org/apache/fory/resolver/ClassResolver.java | 2 +- .../org/apache/fory/resolver/TypeResolver.java | 1 - .../org/apache/fory/resolver/XtypeResolver.java | 14 ++++++++------ .../org/apache/fory/serializer/FieldGroups.java | 2 +- .../java/org/apache/fory/xlang/CPPXlangTest.java | 9 +++++---- .../java/org/apache/fory/xlang/GoXlangTest.java | 7 ++++--- .../org/apache/fory/xlang/MetaSharedXlangTest.java | 4 ++-- .../org/apache/fory/xlang/PythonXlangTest.java | 1 - .../java/org/apache/fory/xlang/RustXlangTest.java | 8 +++++--- .../java/org/apache/fory/xlang/XlangTestBase.java | 6 ++++-- 11 files changed, 31 insertions(+), 29 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index e88e4b4807..865ded99cc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -91,11 +91,7 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) this.classDef = classDef; Collection descriptors = fory( - f -> - MetaSharedSerializer.consolidateFields( - f.isCrossLanguage() ? f.getXtypeResolver() : f.getClassResolver(), - beanClass, - classDef)); + f -> MetaSharedSerializer.consolidateFields(f._getTypeResolver(), beanClass, classDef)); DescriptorGrouper grouper = typeResolver(r -> r.createDescriptorGrouper(descriptors, false)); List sortedDescriptors = grouper.getSortedDescriptors(); if (org.apache.fory.util.Utils.debugOutputEnabled()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 3c13eb92d9..c026bdeaa7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -689,7 +689,7 @@ public String getTypeAlias(Class cls) { } @Override - public boolean isMonomorphic(Descriptor descriptor) { + public boolean isMonomorphic(Descriptor descriptor) { ForyField foryField = descriptor.getForyField(); if (foryField != null) { switch (foryField.morphic()) { diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java index f66f0550da..73bd5f7ec9 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java @@ -492,7 +492,6 @@ public GenericType getGenericTypeInStruct(Class cls, String genericTypeStr) { return map.getOrDefault(genericTypeStr, OBJECT_GENERIC_TYPE); } - public abstract void initialize(); public abstract ClassDef getTypeDef(Class cls, boolean resolveParent); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java index cb6c52520c..a4fc4079e7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java @@ -435,7 +435,7 @@ public boolean isRegisteredByName(Class cls) { } @Override - public boolean isMonomorphic(Descriptor descriptor) { + public boolean isMonomorphic(Descriptor descriptor) { ForyField foryField = descriptor.getForyField(); ForyField.Morphic morphic = foryField != null ? foryField.morphic() : ForyField.Morphic.AUTO; switch (morphic) { @@ -448,7 +448,11 @@ public boolean isMonomorphic(Descriptor descriptor) { if (rawType.isEnum()) { return true; } - return !ReflectionUtils.isAbstract(rawType); + byte xtypeId = getXtypeId(rawType); + if (fory.isCompatible()) { + return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; + } + return xtypeId != Types.UNKNOWN; } } @@ -481,7 +485,7 @@ public boolean isMonomorphic(Class clz) { public boolean isBuildIn(Descriptor descriptor) { Class rawType = descriptor.getRawType(); byte xtypeId = getXtypeId(rawType); - return !Types.isUserDefinedType(xtypeId); + return !Types.isUserDefinedType(xtypeId) && xtypeId != Types.UNKNOWN; } @Override @@ -965,8 +969,6 @@ public DescriptorGrouper createDescriptorGrouper( .sort(); } - private static final int UNKNOWN_TYPE_ID = Types.UNKNOWN; - private byte getXtypeId(Class cls) { if (isSet(cls)) { return Types.SET; @@ -992,7 +994,7 @@ private byte getXtypeId(Class cls) { if (ReflectionUtils.isMonomorphic(cls)) { throw new UnsupportedOperationException(cls + " is not supported for xlang serialization"); } - return UNKNOWN_TYPE_ID; + return Types.UNKNOWN; } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 7107a8ea8b..1bf28314a3 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -174,7 +174,7 @@ public static final class SerializationFieldInfo { classInfo = null; } } - useDeclaredTypeInfo = classInfo != null && fory.getClassResolver().isMonomorphic(descriptor); + useDeclaredTypeInfo = classInfo != null && resolver.isMonomorphic(descriptor); if (classInfo != null) { serializer = classInfo.getSerializer(); } else { diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index 279ad261fb..ece36c0d46 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -29,7 +29,6 @@ import java.util.HashSet; import java.util.List; import java.util.concurrent.TimeUnit; - import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; @@ -48,7 +47,7 @@ public class CPPXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_CPP_JAVA_CI"); if (!"1".equals(enabled)) { - throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); + throw new SkipException("Skipping CPPXlangTest: FORY_CPP_JAVA_CI not set to 1"); } boolean bazelAvailable = true; try { @@ -280,13 +279,15 @@ public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java @Override @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNull(enableCodegen); } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index e4a548f0e9..b453c9182b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -29,7 +29,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; - import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; @@ -305,13 +304,15 @@ public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java @Override @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Override @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNull(enableCodegen); } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 2cf950065b..e03353093b 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -20,13 +20,13 @@ package org.apache.fory.xlang; import lombok.Data; -import org.apache.fory.xlang.PyCrossLanguageTest.Bar; -import org.apache.fory.xlang.PyCrossLanguageTest.Foo; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; import org.apache.fory.test.bean.BeanB; +import org.apache.fory.xlang.PyCrossLanguageTest.Bar; +import org.apache.fory.xlang.PyCrossLanguageTest.Foo; import org.testng.annotations.Test; public class MetaSharedXlangTest extends ForyTestBase { diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java index 0b1369c5c6..8a686c1fad 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; - import org.apache.fory.Fory; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index c36d1940e2..acb504c0b3 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -53,7 +53,7 @@ public class RustXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_RUST_JAVA_CI"); if (!"1".equals(enabled)) { - throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); + // throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); } boolean rustInstalled = true; try { @@ -232,12 +232,14 @@ public void testEnumSchemaEvolutionCompatible(boolean enableCodegen) throws java } @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNotNull(enableCodegen); } @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { super.testNullableFieldSchemaConsistentNull(enableCodegen); } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index cd331bf234..eaf0eb46a7 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -1749,7 +1749,8 @@ static class NullableComprehensiveSchemaConsistent { } @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) + throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_not_null"; Fory fory = Fory.builder() @@ -1807,7 +1808,8 @@ public void testNullableFieldSchemaConsistentNotNull(boolean enableCodegen) thro } @Test(dataProvider = "enableCodegen") - public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) throws java.io.IOException { + public void testNullableFieldSchemaConsistentNull(boolean enableCodegen) + throws java.io.IOException { String caseName = "test_nullable_field_schema_consistent_null"; Fory fory = Fory.builder() From 12224b871291ad1bc2fc51fe6f964cfe896b8c9b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 11:01:11 +0800 Subject: [PATCH 11/35] fix xlang tests --- cpp/fory/serialization/type_info.h | 6 +- cpp/fory/serialization/type_resolver.h | 7 +- cpp/fory/serialization/xlang_test_main.cc | 6 +- go/fory/type_def.go | 11 ++ go/fory/type_resolver.go | 17 ++- .../org/apache/fory/annotation/ForyField.java | 2 +- .../fory/builder/BaseObjectCodecBuilder.java | 120 ++++++++++-------- .../serializer/AbstractObjectSerializer.java | 120 +++++++++--------- .../apache/fory/serializer/FieldGroups.java | 5 +- .../org/apache/fory/xlang/CPPXlangTest.java | 12 +- .../org/apache/fory/xlang/GoXlangTest.java | 12 +- .../apache/fory/xlang/PythonXlangTest.java | 16 +-- .../org/apache/fory/xlang/RustXlangTest.java | 12 +- .../org/apache/fory/xlang/XlangTestBase.java | 14 +- python/pyfory/meta/typedef_encoder.py | 4 +- rust/fory-core/src/serializer/arc.rs | 6 +- rust/fory-core/src/serializer/rc.rs | 6 +- rust/fory-derive/src/object/read.rs | 8 +- rust/fory-derive/src/object/write.rs | 6 +- rust/tests/tests/test_cross_language.rs | 32 ++++- 20 files changed, 247 insertions(+), 175 deletions(-) diff --git a/cpp/fory/serialization/type_info.h b/cpp/fory/serialization/type_info.h index 12a8110299..9334611223 100644 --- a/cpp/fory/serialization/type_info.h +++ b/cpp/fory/serialization/type_info.h @@ -56,12 +56,12 @@ struct Harness { using WriteFn = void (*)(const void *value, WriteContext &ctx, RefMode ref_mode, bool write_type_info, bool has_generics); - using ReadFn = void *(*)(ReadContext &ctx, RefMode ref_mode, + using ReadFn = void *(*)(ReadContext & ctx, RefMode ref_mode, bool read_type_info); using WriteDataFn = void (*)(const void *value, WriteContext &ctx, bool has_generics); - using ReadDataFn = void *(*)(ReadContext &ctx); - using ReadCompatibleFn = void *(*)(ReadContext &ctx, + using ReadDataFn = void *(*)(ReadContext & ctx); + using ReadCompatibleFn = void *(*)(ReadContext & ctx, const struct TypeInfo *type_info); using SortedFieldInfosFn = Result, Error> (*)(TypeResolver &); diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index fe4b0a8d22..29380ad7e1 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -528,10 +528,9 @@ template struct FieldInfoBuilder { field_type.ref_mode = make_ref_mode(is_nullable, track_ref); // DEBUG: Print field info for debugging fingerprint mismatch std::cerr << "[xlang][debug] FieldInfoBuilder T=" << typeid(T).name() - << " Index=" << Index << " field=" << field_name - << " has_tags=" << ::fory::detail::has_field_tags_v - << " is_nullable=" << is_nullable << " track_ref=" << track_ref - << std::endl; + << " Index=" << Index << " field=" << field_name << " has_tags=" + << ::fory::detail::has_field_tags_v << " is_nullable=" + << is_nullable << " track_ref=" << track_ref << std::endl; return FieldInfo(std::move(field_name), std::move(field_type)); } }; diff --git a/cpp/fory/serialization/xlang_test_main.cc b/cpp/fory/serialization/xlang_test_main.cc index 220b3e2cb6..9cc1bb9d8d 100644 --- a/cpp/fory/serialization/xlang_test_main.cc +++ b/cpp/fory/serialization/xlang_test_main.cc @@ -2253,7 +2253,8 @@ void RunTestNullableFieldCompatibleNull(const std::string &data_file) { void RunTestRefSchemaConsistent(const std::string &data_file) { auto bytes = ReadFile(data_file); - // SCHEMA_CONSISTENT mode: compatible=false, xlang=true, check_struct_version=true, track_ref=true + // SCHEMA_CONSISTENT mode: compatible=false, xlang=true, + // check_struct_version=true, track_ref=true auto fory = BuildFory(false, true, true, true); EnsureOk(fory.register_struct(501), "register RefInnerSchemaConsistent"); @@ -2298,7 +2299,8 @@ void RunTestRefSchemaConsistent(const std::string &data_file) { void RunTestRefCompatible(const std::string &data_file) { auto bytes = ReadFile(data_file); - // COMPATIBLE mode: compatible=true, xlang=true, check_struct_version=false, track_ref=true + // COMPATIBLE mode: compatible=true, xlang=true, check_struct_version=false, + // track_ref=true auto fory = BuildFory(true, true, false, true); EnsureOk(fory.register_struct(503), "register RefInnerCompatible"); diff --git a/go/fory/type_def.go b/go/fory/type_def.go index e15c3304e0..2d547631ce 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -462,10 +462,21 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { } // Calculate ref tracking - use tag override if explicitly set + // In xlang mode, registered types (primitives, strings) don't use ref tracking + // because they are value types, not reference types. trackingRef := fory.config.TrackRef if foryTag.RefSet { trackingRef = foryTag.Ref } + // Disable ref tracking for simple types (primitives, strings) in xlang mode + // These types don't benefit from ref tracking and Java doesn't expect ref flags for them + if fory.config.IsXlang && trackingRef { + // Check if this is a simple field type (primitives, strings, enums, etc.) + // SimpleFieldType represents built-in types that don't need ref tracking + if _, ok := ft.(*SimpleFieldType); ok { + trackingRef = false + } + } fieldInfo := FieldDef{ name: fieldName, diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 4511a0bb0e..6b11b69b78 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -1412,12 +1412,25 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } } + // Determine key/value referencability + // In xlang mode, strings are value types and should not be reference-tracked + keyReferencable := nullable(type_.Key()) + valueReferencable := nullable(type_.Elem()) + if r.isXlang { + // In xlang mode, strings are value types (not reference-tracked) + if type_.Key().Kind() == reflect.String { + keyReferencable = false + } + if type_.Elem().Kind() == reflect.String { + valueReferencable = false + } + } return &mapSerializer{ type_: type_, keySerializer: keySerializer, valueSerializer: valueSerializer, - keyReferencable: nullable(type_.Key()), - valueReferencable: nullable(type_.Elem()), + keyReferencable: keyReferencable, + valueReferencable: valueReferencable, mapInStruct: mapInStruct, }, nil } else { diff --git a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java index 11c0976439..aae0721967 100644 --- a/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java +++ b/java/fory-core/src/main/java/org/apache/fory/annotation/ForyField.java @@ -31,7 +31,7 @@ /** Controls polymorphism behavior for struct fields in cross-language serialization. */ enum Morphic { /** - * Auto-detect based on declared type (default): + * Auto-detect based on declared type (default). * *
    *
  • Xlang mode: only interface/abstract class are treated as POLYMORPHIC, concrete classes diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 90b92a6635..ce894ab38d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1976,22 +1976,27 @@ protected Expression deserializeForCollection( protected Expression readCollectionCodegen( Expression buffer, Expression collection, Expression size, TypeRef elementType) { ListExpression builder = new ListExpression(); - Invoke flags = new Invoke(buffer, "readByte", "flags", PRIMITIVE_INT_TYPE, false); + Invoke flags = new Invoke(buffer, "readUnsignedByte", "flags", PRIMITIVE_INT_TYPE, false); builder.add(flags); Class elemClass = TypeUtils.getRawType(elementType); walkPath.add(elementType.toString()); boolean finalType = isMonomorphic(elemClass); - boolean trackingRef = typeResolver(resolver -> resolver.needToWriteRef(elementType)); + // Read TRACKING_REF flag from bitmap at runtime for xlang compatibility. + // This ensures Java correctly reads data serialized by other languages (C++, Go, Rust, etc.) + // that may set TRACKING_REF based on element type (e.g., shared_ptr) rather than global config. + Literal trackingRefFlag = ofInt(CollectionFlags.TRACKING_REF); + Expression trackingRef = eq(new BitAnd(flags, trackingRefFlag), trackingRefFlag, "trackingRef"); + Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); + // Don't use flags.inline() - it mutates the flags object to prevent variable generation + Expression hasNull = eq(new BitAnd(flags, hasNullFlag), hasNullFlag, "hasNull"); if (finalType) { - if (trackingRef) { - builder.add(readContainerElements(elementType, true, null, null, buffer, collection, size)); - } else { - Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); - Expression hasNull = eq(new BitAnd(flags.inline(), hasNullFlag), hasNullFlag, "hasNull"); - builder.add( - hasNull, - readContainerElements(elementType, false, null, hasNull, buffer, collection, size)); - } + builder.add(hasNull); + // Use runtime trackingRef flag to determine if ref flags are present + Expression trackingRefRead = + readContainerElements(elementType, true, null, null, buffer, collection, size); + Expression noTrackingRefRead = + readContainerElements(elementType, false, null, hasNull, buffer, collection, size); + builder.add(new If(trackingRef, trackingRefRead, noTrackingRefRead)); } else { Literal isSameTypeFlag = ofInt(CollectionFlags.IS_SAME_TYPE); Expression sameElementClass = @@ -2016,45 +2021,60 @@ protected Expression readCollectionCodegen( elemSerializer = cast(serializer.inline(), serializerType); } elemSerializer = uninline(elemSerializer); - builder.add(sameElementClass); - Expression action; - if (trackingRef) { - // Same element class read start - ListExpression readBuilder = new ListExpression(elemSerializer); - readBuilder.add( - readContainerElements( - elementType, true, elemSerializer, null, buffer, collection, size)); - // Same element class read end - Set cutPoint = ofHashSet(buffer, collection, size); - Expression differentElemTypeRead = - invokeGenerated( - ctx, - cutPoint, - readContainerElements(elementType, true, null, null, buffer, collection, size), - "differentTypeElemsRead", - false); - action = new If(sameElementClass, readBuilder, differentElemTypeRead); - } else { - Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); - Expression hasNull = eq(new BitAnd(flags, hasNullFlag), hasNullFlag, "hasNull"); - builder.add(hasNull); - // Same element class read start - ListExpression readBuilder = new ListExpression(elemSerializer); - readBuilder.add( - readContainerElements( - elementType, false, elemSerializer, hasNull, buffer, collection, size)); - // Same element class read end - Set cutPoint = ofHashSet(buffer, collection, size, hasNull); - Expression differentTypeElemsRead = - invokeGenerated( - ctx, - cutPoint, - readContainerElements(elementType, false, null, hasNull, buffer, collection, size), - "differentTypeElemsRead", - false); - action = new If(sameElementClass, readBuilder, differentTypeElemsRead); - } - builder.add(action); + // For xlang compatibility, we must read the TRACKING_REF flag from the serialized data + // because different languages may set this flag differently (e.g., C++ sets it for + // shared_ptr). + // The elemSerializer (which contains readClassInfo) must be conditionally evaluated: + // - It should only be evaluated when sameElementClass == true + // - It should be in scope for both tracking ref branches + // Wrap elemSerializer in a conditional that only evaluates when sameElementClass is true + Expression elemSerializerInit = + new If(sameElementClass, elemSerializer, nullValue(serializerType)); + builder.add(sameElementClass, hasNull, elemSerializerInit); + // Build both tracking and non-tracking paths, then branch at runtime based on flags byte + // Tracking ref path - same element class + // Pass elemSerializerInit (not elemSerializer) so that we reference the variable + // defined by the If expression, not regenerate the readClassInfo call + ListExpression trackingRefSameBuilder = new ListExpression(); + trackingRefSameBuilder.add( + readContainerElements( + elementType, true, elemSerializerInit, null, buffer, collection, size)); + // Tracking ref path - different element class + Set cutPoint1 = ofHashSet(buffer, collection, size); + Expression trackingRefDifferent = + invokeGenerated( + ctx, + cutPoint1, + readContainerElements(elementType, true, null, null, buffer, collection, size), + "differentTypeElemsReadTrackingRef", + false); + Expression trackingRefAction = + new If(sameElementClass, trackingRefSameBuilder, trackingRefDifferent); + + // No tracking ref path - same element class + // Pass elemSerializerInit to reference the same variable as the tracking ref path + ListExpression noTrackingRefSameBuilder = new ListExpression(); + noTrackingRefSameBuilder.add( + readContainerElements( + elementType, false, elemSerializerInit, hasNull, buffer, collection, size)); + // No tracking ref path - different element class + Set cutPoint2 = ofHashSet(buffer, collection, size, hasNull); + Expression noTrackingRefDifferent = + invokeGenerated( + ctx, + cutPoint2, + readContainerElements(elementType, false, null, hasNull, buffer, collection, size), + "differentTypeElemsRead", + false); + Expression noTrackingRefAction = + new If(sameElementClass, noTrackingRefSameBuilder, noTrackingRefDifferent); + + // Use neq to check: if (flags & TRACKING_REF) != 0 + // This is the standard way to check if a bit flag is set + Expression trackingRefCheck = + neq(new BitAnd(flags, trackingRefFlag), ofInt(0), "isTrackingRef"); + builder.add(trackingRefCheck); + builder.add(new If(trackingRefCheck, trackingRefAction, noTrackingRefAction)); } walkPath.removeLast(); // place newCollection as last as expr value diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java index 437584f8b3..6c0a0ee193 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java @@ -208,6 +208,66 @@ static boolean writePrimitiveFieldValue( } } + /** + * Write a primitive field value to buffer using the field accessor. + * + * @param fory the fory instance for compression settings + * @param buffer the buffer to write to + * @param targetObject the object containing the field + * @param fieldAccessor the accessor to get the field value + * @param classId the class ID of the primitive type + * @return true if classId is not a primitive type and needs further write handling + */ + static boolean writePrimitiveFieldValue( + Fory fory, + MemoryBuffer buffer, + Object targetObject, + FieldAccessor fieldAccessor, + short classId) { + long fieldOffset = fieldAccessor.getFieldOffset(); + if (fieldOffset != -1) { + return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); + } + switch (classId) { + case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: + buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: + buffer.writeByte((Byte) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: + buffer.writeChar((Character) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: + buffer.writeInt16((Short) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_INT_CLASS_ID: + { + int fieldValue = (Integer) fieldAccessor.get(targetObject); + if (fory.compressInt()) { + buffer.writeVarInt32(fieldValue); + } else { + buffer.writeInt32(fieldValue); + } + return false; + } + case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: + buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); + return false; + case ClassResolver.PRIMITIVE_LONG_CLASS_ID: + { + long fieldValue = (long) fieldAccessor.get(targetObject); + fory.writeInt64(buffer, fieldValue); + return false; + } + case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: + buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); + return false; + default: + return true; + } + } + /** * Write field value to buffer. This method handle the situation which all fields are not null. * @@ -550,66 +610,6 @@ static Object readContainerFieldValue( return fieldValue; } - /** - * Write a primitive field value to buffer using the field accessor. - * - * @param fory the fory instance for compression settings - * @param buffer the buffer to write to - * @param targetObject the object containing the field - * @param fieldAccessor the accessor to get the field value - * @param classId the class ID of the primitive type - * @return true if classId is not a primitive type and needs further write handling - */ - static boolean writePrimitiveFieldValue( - Fory fory, - MemoryBuffer buffer, - Object targetObject, - FieldAccessor fieldAccessor, - short classId) { - long fieldOffset = fieldAccessor.getFieldOffset(); - if (fieldOffset != -1) { - return writePrimitiveFieldValue(fory, buffer, targetObject, fieldOffset, classId); - } - switch (classId) { - case ClassResolver.PRIMITIVE_BOOLEAN_CLASS_ID: - buffer.writeBoolean((Boolean) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_BYTE_CLASS_ID: - buffer.writeByte((Byte) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_CHAR_CLASS_ID: - buffer.writeChar((Character) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_SHORT_CLASS_ID: - buffer.writeInt16((Short) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_INT_CLASS_ID: - { - int fieldValue = (Integer) fieldAccessor.get(targetObject); - if (fory.compressInt()) { - buffer.writeVarInt32(fieldValue); - } else { - buffer.writeInt32(fieldValue); - } - return false; - } - case ClassResolver.PRIMITIVE_FLOAT_CLASS_ID: - buffer.writeFloat32((Float) fieldAccessor.get(targetObject)); - return false; - case ClassResolver.PRIMITIVE_LONG_CLASS_ID: - { - long fieldValue = (long) fieldAccessor.get(targetObject); - fory.writeInt64(buffer, fieldValue); - return false; - } - case ClassResolver.PRIMITIVE_DOUBLE_CLASS_ID: - buffer.writeFloat64((Double) fieldAccessor.get(targetObject)); - return false; - default: - return true; - } - } - /** * Read a primitive value from buffer and set it to field referenced by fieldAccessor * of targetObject. diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java index 1bf28314a3..dbcc3362d6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/FieldGroups.java @@ -54,9 +54,8 @@ public FieldGroups( this.buildInFields = buildInFields; this.userTypeFields = userTypeFields; this.containerFields = containerFields; - SerializationFieldInfo[] fields = - new SerializationFieldInfo - [buildInFields.length + userTypeFields.length + containerFields.length]; + int len = buildInFields.length + userTypeFields.length + containerFields.length; + SerializationFieldInfo[] fields = new SerializationFieldInfo[len]; System.arraycopy(buildInFields, 0, fields, 0, buildInFields.length); System.arraycopy(containerFields, 0, fields, buildInFields.length, containerFields.length); System.arraycopy( diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java index ece36c0d46..3f7ba9a191 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/CPPXlangTest.java @@ -152,14 +152,14 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test(dataProvider = "enableCodegen") - public void testStringSerializer(boolean enableCodegen) throws Exception { - super.testStringSerializer(enableCodegen); + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); } - @Test(dataProvider = "enableCodegen") - public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { - super.testCrossLanguageSerializer(enableCodegen); + @Test + public void testCrossLanguageSerializer() throws Exception { + super.testCrossLanguageSerializer(); } @Test(dataProvider = "enableCodegen") diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java index b453c9182b..e6b60ccd06 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/GoXlangTest.java @@ -116,14 +116,14 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test(dataProvider = "enableCodegen") - public void testStringSerializer(boolean enableCodegen) throws Exception { - super.testStringSerializer(enableCodegen); + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); } - @Test(dataProvider = "enableCodegen") - public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { - super.testCrossLanguageSerializer(enableCodegen); + @Test + public void testCrossLanguageSerializer() throws Exception { + super.testCrossLanguageSerializer(); } @Test(dataProvider = "enableCodegen") diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java index 8a686c1fad..8953c8acc9 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/PythonXlangTest.java @@ -71,20 +71,20 @@ protected CommandContext buildCommandContext(String caseName, Path dataFile) { // ============================================================================ @Override - @Test(dataProvider = "enableCodegen") + @Test public void testBuffer() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test(dataProvider = "enableCodegen") + @Test public void testMurmurHash3() throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @Override - @Test(dataProvider = "enableCodegen") - public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { + @Test + public void testCrossLanguageSerializer() throws Exception { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @@ -130,7 +130,7 @@ public void testBufferVar() throws IOException { } @Override - @Test + @Test(dataProvider = "enableCodegen") public void testInteger(boolean enableCodegen) throws IOException { throw new SkipException("Skipping: similar test already covered in CrossLanguageTest"); } @@ -145,9 +145,9 @@ public void testInteger(boolean enableCodegen) throws IOException { // ============================================================================ @Override - @Test(dataProvider = "enableCodegen") - public void testStringSerializer(boolean enableCodegen) throws Exception { - super.testStringSerializer(enableCodegen); + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); } @Override diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index acb504c0b3..f322d56f33 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -106,14 +106,14 @@ public void testMurmurHash3() throws java.io.IOException { super.testMurmurHash3(); } - @Test(dataProvider = "enableCodegen") - public void testStringSerializer(boolean enableCodegen) throws Exception { - super.testStringSerializer(enableCodegen); + @Test + public void testStringSerializer() throws Exception { + super.testStringSerializer(); } - @Test(dataProvider = "enableCodegen") - public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { - super.testCrossLanguageSerializer(enableCodegen); + @Test + public void testCrossLanguageSerializer() throws Exception { + super.testCrossLanguageSerializer(); } @Test(dataProvider = "enableCodegen") diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java index eaf0eb46a7..0eded0f2b6 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/XlangTestBase.java @@ -416,21 +416,21 @@ private void _testStringSerializer(Fory fory, String caseName) throws IOExceptio } } - @Test(dataProvider = "enableCodegen") - public void testStringSerializer(boolean enableCodegen) throws Exception { + @Test + public void testStringSerializer() throws Exception { String caseName = "test_string_serializer"; Fory fory = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(enableCodegen) + .withCodegen(false) .build(); _testStringSerializer(fory, caseName); Fory foryCompress = Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(enableCodegen) + .withCodegen(false) .withStringCompressed(true) .withWriteNumUtf16BytesForUtf8Encoding(false) .build(); @@ -444,8 +444,8 @@ enum Color { White, } - @Test(dataProvider = "enableCodegen") - public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception { + @Test + public void testCrossLanguageSerializer() throws Exception { String caseName = "test_cross_language_serializer"; List strList = Arrays.asList("hello", "world"); Set strSet = new HashSet<>(strList); @@ -458,7 +458,7 @@ public void testCrossLanguageSerializer(boolean enableCodegen) throws Exception Fory.builder() .withLanguage(Language.XLANG) .withCompatibleMode(CompatibleMode.COMPATIBLE) - .withCodegen(enableCodegen) + .withCodegen(false) .build(); fory.register(Color.class, 101); MemoryBuffer buffer = MemoryUtils.buffer(64); diff --git a/python/pyfory/meta/typedef_encoder.py b/python/pyfory/meta/typedef_encoder.py index fd4d7d3423..ae0e56bb65 100644 --- a/python/pyfory/meta/typedef_encoder.py +++ b/python/pyfory/meta/typedef_encoder.py @@ -97,8 +97,8 @@ def encode_typedef(type_resolver, cls): # Write fields info write_fields_info(type_resolver, buffer, field_infos) - # Get the encoded binary - binary = buffer.to_bytes() + # Get the encoded binary (only the written portion, not the full buffer) + binary = buffer.to_bytes(0, buffer.writer_index) # Compress if beneficial compressed_binary = type_resolver.get_meta_compressor().compress(binary) diff --git a/rust/fory-core/src/serializer/arc.rs b/rust/fory-core/src/serializer/arc.rs index a581e2cdd4..7aecffa452 100644 --- a/rust/fory-core/src/serializer/arc.rs +++ b/rust/fory-core/src/serializer/arc.rs @@ -69,7 +69,11 @@ impl Serializer for Arc } } - fn fory_write_data_generic(&self, context: &mut WriteContext, has_generics: bool) -> Result<(), Error> { + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { if T::fory_is_shared_ref() { return Err(Error::not_allowed( "Arc where T is a shared ref type is not allowed for serialization.", diff --git a/rust/fory-core/src/serializer/rc.rs b/rust/fory-core/src/serializer/rc.rs index 39832996f2..4bfd6373f8 100644 --- a/rust/fory-core/src/serializer/rc.rs +++ b/rust/fory-core/src/serializer/rc.rs @@ -68,7 +68,11 @@ impl Serializer for Rc { } } - fn fory_write_data_generic(&self, context: &mut WriteContext, has_generics: bool) -> Result<(), Error> { + fn fory_write_data_generic( + &self, + context: &mut WriteContext, + has_generics: bool, + ) -> Result<(), Error> { if T::fory_is_shared_ref() { return Err(Error::not_allowed( "Rc where T is a shared ref type is not allowed for serialization.", diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs index ed16babea8..2e7aa12f50 100644 --- a/rust/fory-derive/src/object/read.rs +++ b/rust/fory-derive/src/object/read.rs @@ -22,9 +22,8 @@ use syn::Field; use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, - get_primitive_reader_method, get_struct_name, is_debug_enabled, - is_direct_primitive_type, is_primitive_type, is_skip_field, - should_skip_type_info_for_field, FieldRefMode, StructField, + get_primitive_reader_method, get_struct_name, is_debug_enabled, is_direct_primitive_type, + is_primitive_type, is_skip_field, should_skip_type_info_for_field, FieldRefMode, StructField, }; use crate::util::SourceField; @@ -233,7 +232,8 @@ pub fn gen_read_field(field: &Field, private_ident: &Ident, field_name: &str) -> } else { // Numeric primitives: use direct buffer methods let reader_method = get_primitive_reader_method(&type_name); - let reader_ident = syn::Ident::new(reader_method, proc_macro2::Span::call_site()); + let reader_ident = + syn::Ident::new(reader_method, proc_macro2::Span::call_site()); quote! { let #private_ident = context.reader.#reader_ident()?; } diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs index 1ab0b3c7d2..3960860103 100644 --- a/rust/fory-derive/src/object/write.rs +++ b/rust/fory-derive/src/object/write.rs @@ -19,8 +19,7 @@ use super::util::{ classify_trait_object_field, create_wrapper_types_arc, create_wrapper_types_rc, determine_field_ref_mode, extract_type_name, gen_struct_version_hash_ts, get_field_accessor, get_field_name, get_filtered_source_fields_iter, get_primitive_writer_method, get_struct_name, - get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_type, FieldRefMode, - StructField, + get_type_id_by_type_ast, is_debug_enabled, is_direct_primitive_type, FieldRefMode, StructField, }; use crate::util::SourceField; use fory_core::types::TypeId; @@ -262,7 +261,8 @@ fn gen_write_field_impl( } else { // Numeric primitives: use direct buffer methods let writer_method = get_primitive_writer_method(&type_name); - let writer_ident = syn::Ident::new(writer_method, proc_macro2::Span::call_site()); + let writer_ident = + syn::Ident::new(writer_method, proc_macro2::Span::call_site()); // For primitives: // - use_self=true: #value_ts is `self.field`, which is T (copy happens automatically) // - use_self=false: #value_ts is `field` from pattern match on &self, which is &T diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 7b83e7d35a..532b75b639 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1700,7 +1700,10 @@ fn test_ref_schema_consistent() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().compatible(false).xlang(true).ref_tracking(true); + let mut fory = Fory::default() + .compatible(false) + .xlang(true) + .ref_tracking(true); fory.register::(501).unwrap(); fory.register::(502).unwrap(); @@ -1716,10 +1719,17 @@ fn test_ref_schema_consistent() { assert_eq!(inner1.id, 42); assert_eq!(inner1.name, "shared_inner"); // Compare the values (Rc contents) - assert_eq!(inner1.as_ref(), inner2.as_ref(), "inner1 and inner2 should have equal values"); + assert_eq!( + inner1.as_ref(), + inner2.as_ref(), + "inner1 and inner2 should have equal values" + ); // With Rc, after deserialization with ref tracking, both fields should point to the same Rc - assert!(Rc::ptr_eq(inner1, inner2), "inner1 and inner2 should be the same Rc (reference identity)"); + assert!( + Rc::ptr_eq(inner1, inner2), + "inner1 and inner2 should be the same Rc (reference identity)" + ); // Re-serialize and write back let new_bytes = fory.serialize(&outer).unwrap(); @@ -1737,7 +1747,10 @@ fn test_ref_compatible() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default().compatible(true).xlang(true).ref_tracking(true); + let mut fory = Fory::default() + .compatible(true) + .xlang(true) + .ref_tracking(true); fory.register::(503).unwrap(); fory.register::(504).unwrap(); @@ -1753,10 +1766,17 @@ fn test_ref_compatible() { assert_eq!(inner1.id, 99); assert_eq!(inner1.name, "compatible_shared"); // Compare the values (Rc contents) - assert_eq!(inner1.as_ref(), inner2.as_ref(), "inner1 and inner2 should have equal values"); + assert_eq!( + inner1.as_ref(), + inner2.as_ref(), + "inner1 and inner2 should have equal values" + ); // With Rc, after deserialization with ref tracking, both fields should point to the same Rc - assert!(Rc::ptr_eq(inner1, inner2), "inner1 and inner2 should be the same Rc (reference identity)"); + assert!( + Rc::ptr_eq(inner1, inner2), + "inner1 and inner2 should be the same Rc (reference identity)" + ); // Re-serialize and write back let new_bytes = fory.serialize(&outer).unwrap(); From 944eaf1ce92406d9ce16ba14f0878310d151be2a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 11:14:14 +0800 Subject: [PATCH 12/35] fix(codegen): return FalseLiteral for non-nullable If expression isNull When a non-nullable If expression was nested inside a nullable outer If, the inner If's isNull variable was referenced but never declared, causing compilation errors in generated code (e.g., "isNull1 = isNull0;" where isNull0 doesn't exist). The fix returns FalseLiteral instead of a variable reference when nullable=false, matching the behavior of other expressions like Reference. --- .../main/java/org/apache/fory/codegen/Expression.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index ce6bc390a4..073c2ffdd7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -1866,8 +1866,14 @@ public ExprCode doGenCode(CodegenContext ctx) { falseEval.value()); } codeBuilder.append(StringUtils.stripBlankLines(ifCode)); - return new ExprCode( - codeBuilder.toString(), Code.isNullVariable(isNull), Code.variable(rawType, value)); + if (nullable) { + return new ExprCode( + codeBuilder.toString(), Code.isNullVariable(isNull), Code.variable(rawType, value)); + } else { + // When not nullable, return FalseLiteral instead of a variable that was never declared + return new ExprCode( + codeBuilder.toString(), FalseLiteral, Code.variable(rawType, value)); + } } else { String ifCode; if (falseExpr != null) { From 7b9b0d5513cde9b7773359b381092050dc8d010b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 11:25:33 +0800 Subject: [PATCH 13/35] fix(cpp): respect write_type parameter in shared_ptr serialization In map serialization for polymorphic types, the map already writes type info at the chunk level and passes write_type=false. The shared_ptr serializer was incorrectly overriding this to write type info again in compatible mode, causing deserialization failures due to byte stream corruption. The fix respects the write_type parameter passed by the caller. --- cpp/fory/serialization/smart_ptr_serializers.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index 019feb902c..23229f04e4 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -333,11 +333,6 @@ template struct Serializer> { ctx.write_int8(NOT_NULL_VALUE_FLAG); } - // In compatible mode with ref tracking, first occurrence (RefValue) - // requires type info to be written after the ref flag. - const bool should_write_type = - ctx.is_compatible() && is_first_occurrence ? true : write_type; - // For polymorphic types, serialize the concrete type dynamically if constexpr (is_polymorphic) { // Get the concrete type_index from the actual object @@ -352,7 +347,7 @@ template struct Serializer> { const TypeInfo *type_info = type_info_res.value(); // Write type info if requested - if (should_write_type) { + if (write_type) { auto write_res = ctx.write_any_typeinfo( static_cast(TypeId::UNKNOWN), concrete_type_id); if (!write_res.ok()) { @@ -369,7 +364,7 @@ template struct Serializer> { // Non-polymorphic path Serializer::write( *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - should_write_type); + write_type); } } From 097c24351863ce53d0a5de628c13256c8b1292d0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 11:37:54 +0800 Subject: [PATCH 14/35] fix(go): treat slices and maps as nullable in xlang mode In xlang mode, slices and maps can be nil and should have null flags written/read to match the codegen serializer behavior. This ensures compatibility between codegen and reflect mode serialization. --- go/fory/struct.go | 11 +++++++++-- go/fory/type_def.go | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/go/fory/struct.go b/go/fory/struct.go index 93cf4316fc..6b3a6a7956 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -1092,10 +1092,17 @@ func (s *structSerializer) initFieldsFromTypeResolver(typeResolver *TypeResolver isEnum := internalId == ENUM || internalId == NAMED_ENUM // Determine nullable based on mode + // In both xlang and native mode, nil-able types (ptr, slice, map, interface) are nullable. + // This is necessary for correct null flag handling when serializing/deserializing. + // Note: In xlang mode, we don't set nullable for interface{} fields because the + // actual type determines nullability; for slices and maps, they can be nil. var nullableFlag bool if typeResolver.fory.config.IsXlang { - // xlang mode: only pointer types are nullable by default per xlang spec - nullableFlag = fieldType.Kind() == reflect.Ptr + // xlang mode: pointers, slices, and maps are nullable + // (they can be nil and need null flag handling) + nullableFlag = fieldType.Kind() == reflect.Ptr || + fieldType.Kind() == reflect.Slice || + fieldType.Kind() == reflect.Map } else { // Native mode: Go's natural semantics - all nil-able types are nullable nullableFlag = fieldType.Kind() == reflect.Ptr || diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 2d547631ce..98839af6a2 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -441,10 +441,17 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { internalId := TypeId(typeId & 0xFF) isEnumField := internalId == ENUM || internalId == NAMED_ENUM // Determine nullable based on mode + // In both xlang and native mode, nil-able types (ptr, slice, map, interface) are nullable. + // This is necessary for correct null flag handling when serializing/deserializing. + // Note: In xlang mode, we don't set nullable for interface{} fields because the + // actual type determines nullability; for slices and maps, they can be nil. var nullableFlag bool if fory.config.IsXlang { - // xlang mode: only pointer types are nullable by default per xlang spec - nullableFlag = field.Type.Kind() == reflect.Ptr + // xlang mode: pointers, slices, and maps are nullable + // (they can be nil and need null flag handling) + nullableFlag = field.Type.Kind() == reflect.Ptr || + field.Type.Kind() == reflect.Slice || + field.Type.Kind() == reflect.Map } else { // Native mode: Go's natural semantics - all nil-able types are nullable nullableFlag = field.Type.Kind() == reflect.Ptr || From d89b3296d46cecee1efcb75c34d86b7da397ef02 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 14:02:32 +0800 Subject: [PATCH 15/35] fix rust tests --- rust/fory-core/src/serializer/weak.rs | 8 + rust/tests/expanded.rs | 1584 +++++++++++++++++++++++++ rust/tests/tests/test_weak.rs | 18 +- 3 files changed, 1601 insertions(+), 9 deletions(-) create mode 100644 rust/tests/expanded.rs diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 1cc91f40a0..09181ea15d 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -330,6 +330,10 @@ impl Serializer for RcWeak { .ref_writer .try_write_rc_ref(&mut context.writer, &rc) { + // Target not previously registered - serialize its data. + // Note: For circular references, ref_tracking should be enabled + // on the Fory instance to ensure proper reference resolution. + // Use `Fory::default().ref_tracking(true)` for circular references. if write_type_info { T::fory_write_type_info(context)?; } @@ -490,6 +494,10 @@ impl Serializer for ArcWeak .ref_writer .try_write_arc_ref(&mut context.writer, &arc) { + // Target not previously registered - serialize its data. + // Note: For circular references with Mutex/RwLock, this may cause deadlock + // if ref_tracking is not enabled on the Fory instance. + // Use `Fory::default().ref_tracking(true)` for circular references. if write_type_info { T::fory_write_type_info(context)?; } diff --git a/rust/tests/expanded.rs b/rust/tests/expanded.rs new file mode 100644 index 0000000000..db2be92537 --- /dev/null +++ b/rust/tests/expanded.rs @@ -0,0 +1,1584 @@ +#![feature(prelude_import)] +#[macro_use] +extern crate std; +#[prelude_import] +use std::prelude::rust_2021::*; +use fory_core::fory::Fory; +use fory_derive::ForyObject; +use std::collections::HashMap; +extern crate test; +#[rustc_test_marker = "test_simple"] +#[doc(hidden)] +pub const test_simple: test::TestDescAndFn = test::TestDescAndFn { + desc: test::TestDesc { + name: test::StaticTestName("test_simple"), + ignore: false, + ignore_message: ::core::option::Option::None, + source_file: "tests/tests/test_one_struct.rs", + start_line: 23usize, + start_col: 4usize, + end_line: 23usize, + end_col: 15usize, + compile_fail: false, + no_run: false, + should_panic: test::ShouldPanic::No, + test_type: test::TestType::IntegrationTest, + }, + testfn: test::StaticTestFn( + #[coverage(off)] + || test::assert_test_result(test_simple()), + ), +}; +fn test_simple() { + #[fory_debug] + struct Animal1 { + f1: HashMap>, + f2: String, + f3: Vec, + f5: String, + f6: Vec, + f7: i8, + last: i8, + } + use fory_core::ForyDefault as _; + impl fory_core::ForyDefault for Animal1 { + fn fory_default() -> Self { + Self { + f7: ::fory_default(), + last: ::fory_default(), + f2: ::fory_default(), + f5: ::fory_default(), + f3: as fory_core::ForyDefault>::fory_default(), + f6: as fory_core::ForyDefault>::fory_default(), + f1: > as fory_core::ForyDefault>::fory_default(), + } + } + } + impl std::default::Default for Animal1 { + fn default() -> Self { + Self::fory_default() + } + } + impl fory_core::StructSerializer for Animal1 { + #[inline(always)] + fn fory_type_index() -> u32 { + 0u32 + } + #[inline(always)] + fn fory_actual_type_id( + type_id: u32, + register_by_name: bool, + compatible: bool, + ) -> u32 { + fory_core::serializer::struct_::actual_type_id( + type_id, + register_by_name, + compatible, + ) + } + fn fory_get_sorted_field_names() -> &'static [&'static str] { + &["f7", "last", "f2", "f5", "f3", "f6", "f1"] + } + fn fory_fields_info( + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result, fory_core::error::Error> { + let field_infos: Vec = <[_]>::into_vec( + ::alloc::boxed::box_new([ + fory_core::meta::FieldInfo::new( + "f7", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "last", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f2", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f5", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f3", + fory_core::meta::FieldType::new( + 31u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f6", + fory_core::meta::FieldType::new( + 31u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f1", + fory_core::meta::FieldType::new( + , + > as fory_core::serializer::Serializer>::fory_get_type_id( + type_resolver, + )?, + true, + <[_]>::into_vec( + ::alloc::boxed::box_new([ + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + fory_core::meta::FieldType::new( + 31u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ]), + ) as Vec, + ), + ), + ]), + ); + Ok(field_infos) + } + #[inline] + fn fory_read_compatible( + context: &mut fory_core::resolver::context::ReadContext, + type_info: std::rc::Rc, + ) -> Result { + let fields = type_info.get_type_meta().get_field_infos().clone(); + let mut _f7: i8 = 0 as i8; + let mut _last: i8 = 0 as i8; + let mut _f2: Option = None; + let mut _f5: Option = None; + let mut _f3: Option> = None; + let mut _f6: Option> = None; + let mut _f1: Option>> = None; + let meta = context + .get_type_info(&std::any::TypeId::of::())? + .get_type_meta(); + let local_type_hash = meta.get_hash(); + let remote_type_hash = type_info.get_type_meta().get_hash(); + if remote_type_hash == local_type_hash { + ::fory_read_data(context) + } else { + for _field in fields.iter() { + match _field.field_id { + 0i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f7", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f7 = ::fory_read( + context, + true, + false, + )?; + } else { + _f7 = ::fory_read_data( + context, + )?; + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f7", + (&_f7) as &dyn std::any::Any, + context, + ); + } + 1i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "last", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _last = ::fory_read( + context, + true, + false, + )?; + } else { + _last = ::fory_read_data( + context, + )?; + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "last", + (&_last) as &dyn std::any::Any, + context, + ); + } + 2i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f2", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f2 = Some( + ::fory_read( + context, + true, + false, + )?, + ); + } else { + _f2 = Some( + ::fory_read_data(context)?, + ); + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f2", + (&_f2) as &dyn std::any::Any, + context, + ); + } + 3i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f5", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f5 = Some( + ::fory_read( + context, + true, + false, + )?, + ); + } else { + _f5 = Some( + ::fory_read_data(context)?, + ); + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f5", + (&_f5) as &dyn std::any::Any, + context, + ); + } + 4i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f3", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f3 = Some( + as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f3 = Some( + as fory_core::Serializer>::fory_read_data(context)?, + ); + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f3", + (&_f3) as &dyn std::any::Any, + context, + ); + } + 5i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f6", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f6 = Some( + as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f6 = Some( + as fory_core::Serializer>::fory_read_data(context)?, + ); + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f6", + (&_f6) as &dyn std::any::Any, + context, + ); + } + 6i16 => { + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f1", + context, + ); + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f1 = Some( + , + > as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f1 = Some( + , + > as fory_core::Serializer>::fory_read_data(context)?, + ); + } + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f1", + (&_f1) as &dyn std::any::Any, + context, + ); + } + _ => { + let field_type = &_field.field_type; + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + field_type.type_id, + field_type.nullable, + ); + let field_name = _field.field_name.as_str(); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + field_name, + context, + ); + fory_core::serializer::skip::skip_field_value( + context, + &field_type, + read_ref_flag, + )?; + let placeholder: &dyn std::any::Any = &(); + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + field_name, + placeholder, + context, + ); + } + } + } + Ok(Self { + f7: _f7, + last: _last, + f2: _f2.unwrap_or_default(), + f5: _f5.unwrap_or_default(), + f3: _f3.unwrap_or_default(), + f6: _f6.unwrap_or_default(), + f1: _f1.unwrap_or_default(), + }) + } + } + } + impl fory_core::Serializer for Animal1 { + #[inline(always)] + fn fory_get_type_id( + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result { + type_resolver.get_type_id(&std::any::TypeId::of::(), 0u32) + } + #[inline(always)] + fn fory_type_id_dyn( + &self, + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) + } + #[inline(always)] + fn as_any(&self) -> &dyn std::any::Any { + self + } + #[inline(always)] + fn FORY_STATIC_TYPE_ID -> fory_core::TypeId + where + Self: Sized, + { + fory_core::TypeId::STRUCT + } + #[inline(always)] + fn fory_reserved_space() -> usize { + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + > as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + } + #[inline(always)] + fn fory_write( + &self, + context: &mut fory_core::resolver::context::WriteContext, + write_ref_info: bool, + write_type_info: bool, + _: bool, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::write::< + Self, + >(self, context, write_ref_info, write_type_info) + } + #[inline] + fn fory_write_data( + &self, + context: &mut fory_core::resolver::context::WriteContext, + ) -> Result<(), fory_core::error::Error> { + if context.is_check_struct_version() { + context.writer.write_i32(-362304397i32); + } + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f7", + (&self.f7) as &dyn std::any::Any, + context, + ); + ::fory_write_data(&self.f7, context)?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f7", + (&self.f7) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "last", + (&self.last) as &dyn std::any::Any, + context, + ); + ::fory_write_data(&self.last, context)?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "last", + (&self.last) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f2", + (&self.f2) as &dyn std::any::Any, + context, + ); + ::fory_write( + &self.f2, + context, + true, + false, + false, + )?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f2", + (&self.f2) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f5", + (&self.f5) as &dyn std::any::Any, + context, + ); + ::fory_write( + &self.f5, + context, + true, + false, + false, + )?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f5", + (&self.f5) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f3", + (&self.f3) as &dyn std::any::Any, + context, + ); + as fory_core::Serializer>::fory_write( + &self.f3, + context, + true, + false, + false, + )?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f3", + (&self.f3) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f6", + (&self.f6) as &dyn std::any::Any, + context, + ); + as fory_core::Serializer>::fory_write( + &self.f6, + context, + true, + false, + false, + )?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f6", + (&self.f6) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_write_field( + "Animal1", + "f1", + (&self.f1) as &dyn std::any::Any, + context, + ); + , + > as fory_core::Serializer>::fory_write( + &self.f1, + context, + true, + false, + true, + )?; + fory_core::serializer::struct_::struct_after_write_field( + "Animal1", + "f1", + (&self.f1) as &dyn std::any::Any, + context, + ); + Ok(()) + } + #[inline(always)] + fn fory_write_type_info( + context: &mut fory_core::resolver::context::WriteContext, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::write_type_info::(context) + } + #[inline(always)] + fn fory_read( + context: &mut fory_core::resolver::context::ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) + || ref_flag == (fory_core::RefFlag::RefValue as i8) + { + if context.is_compatible() { + let type_info = if read_type_info { + context.read_any_typeinfo()? + } else { + let rs_type_id = std::any::TypeId::of::(); + context.get_type_info(&rs_type_id)? + }; + ::fory_read_compatible( + context, + type_info, + ) + } else { + if read_type_info { + ::fory_read_type_info(context)?; + } + ::fory_read_data(context) + } + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) + } else { + Err( + fory_core::error::Error::invalid_ref( + ::alloc::__export::must_use({ + ::alloc::fmt::format( + format_args!("Unknown ref flag, value:{0}", ref_flag), + ) + }), + ), + ) + } + } + #[inline(always)] + fn fory_read_with_type_info( + context: &mut fory_core::resolver::context::ReadContext, + read_ref_info: bool, + type_info: std::rc::Rc, + ) -> Result { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) + || ref_flag == (fory_core::RefFlag::RefValue as i8) + { + if context.is_compatible() { + ::fory_read_compatible( + context, + type_info, + ) + } else { + ::fory_read_data(context) + } + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) + } else { + Err( + fory_core::error::Error::invalid_ref( + ::alloc::__export::must_use({ + ::alloc::fmt::format( + format_args!("Unknown ref flag, value:{0}", ref_flag), + ) + }), + ), + ) + } + } + #[inline] + fn fory_read_data( + context: &mut fory_core::resolver::context::ReadContext, + ) -> Result { + if context.is_check_struct_version() { + let read_version = context.reader.read_i32()?; + let type_name = std::any::type_name::(); + fory_core::meta::TypeMeta::check_struct_version( + read_version, + -362304397i32, + type_name, + )?; + } + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f7", + context, + ); + let _f7 = ::fory_read_data(context)?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f7", + (&_f7) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "last", + context, + ); + let _last = ::fory_read_data(context)?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "last", + (&_last) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f2", + context, + ); + let _f2 = ::fory_read( + context, + true, + false, + )?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f2", + (&_f2) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f5", + context, + ); + let _f5 = ::fory_read( + context, + true, + false, + )?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f5", + (&_f5) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f3", + context, + ); + let _f3 = as fory_core::Serializer>::fory_read(context, true, false)?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f3", + (&_f3) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f6", + context, + ); + let _f6 = as fory_core::Serializer>::fory_read(context, true, false)?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f6", + (&_f6) as &dyn std::any::Any, + context, + ); + fory_core::serializer::struct_::struct_before_read_field( + "Animal1", + "f1", + context, + ); + let _f1 = , + > as fory_core::Serializer>::fory_read(context, true, false)?; + fory_core::serializer::struct_::struct_after_read_field( + "Animal1", + "f1", + (&_f1) as &dyn std::any::Any, + context, + ); + Ok(Self { + f7: _f7, + last: _last, + f2: _f2, + f5: _f5, + f3: _f3, + f6: _f6, + f1: _f1, + }) + } + #[inline(always)] + fn fory_read_type_info( + context: &mut fory_core::resolver::context::ReadContext, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::read_type_info::(context) + } + } + #[automatically_derived] + impl ::core::fmt::Debug for Animal1 { + #[inline] + fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + let names: &'static _ = &["f1", "f2", "f3", "f5", "f6", "f7", "last"]; + let values: &[&dyn ::core::fmt::Debug] = &[ + &self.f1, + &self.f2, + &self.f3, + &self.f5, + &self.f6, + &self.f7, + &&self.last, + ]; + ::core::fmt::Formatter::debug_struct_fields_finish( + f, + "Animal1", + names, + values, + ) + } + } + struct Animal2 { + f1: HashMap>, + f3: Vec, + f4: String, + f5: i8, + f6: Vec, + f7: i16, + last: i8, + } + use fory_core::ForyDefault as _; + impl fory_core::ForyDefault for Animal2 { + fn fory_default() -> Self { + Self { + f7: ::fory_default(), + f5: ::fory_default(), + last: ::fory_default(), + f4: ::fory_default(), + f3: as fory_core::ForyDefault>::fory_default(), + f6: as fory_core::ForyDefault>::fory_default(), + f1: > as fory_core::ForyDefault>::fory_default(), + } + } + } + impl std::default::Default for Animal2 { + fn default() -> Self { + Self::fory_default() + } + } + impl fory_core::StructSerializer for Animal2 { + #[inline(always)] + fn fory_type_index() -> u32 { + 1u32 + } + #[inline(always)] + fn fory_actual_type_id( + type_id: u32, + register_by_name: bool, + compatible: bool, + ) -> u32 { + fory_core::serializer::struct_::actual_type_id( + type_id, + register_by_name, + compatible, + ) + } + fn fory_get_sorted_field_names() -> &'static [&'static str] { + &["f7", "f5", "last", "f4", "f3", "f6", "f1"] + } + fn fory_fields_info( + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result, fory_core::error::Error> { + let field_infos: Vec = <[_]>::into_vec( + ::alloc::boxed::box_new([ + fory_core::meta::FieldInfo::new( + "f7", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f5", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "last", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f4", + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f3", + fory_core::meta::FieldType::new( + 31u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f6", + fory_core::meta::FieldType::new( + 32u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ), + fory_core::meta::FieldInfo::new( + "f1", + fory_core::meta::FieldType::new( + , + > as fory_core::serializer::Serializer>::fory_get_type_id( + type_resolver, + )?, + true, + <[_]>::into_vec( + ::alloc::boxed::box_new([ + fory_core::meta::FieldType::new( + ::fory_get_type_id( + type_resolver, + )?, + false, + ::alloc::vec::Vec::new() as Vec, + ), + fory_core::meta::FieldType::new( + 31u32, + true, + ::alloc::vec::Vec::new() as Vec, + ), + ]), + ) as Vec, + ), + ), + ]), + ); + Ok(field_infos) + } + #[inline] + fn fory_read_compatible( + context: &mut fory_core::resolver::context::ReadContext, + type_info: std::rc::Rc, + ) -> Result { + let fields = type_info.get_type_meta().get_field_infos().clone(); + let mut _f7: i16 = 0 as i16; + let mut _f5: i8 = 0 as i8; + let mut _last: i8 = 0 as i8; + let mut _f4: Option = None; + let mut _f3: Option> = None; + let mut _f6: Option> = None; + let mut _f1: Option>> = None; + let meta = context + .get_type_info(&std::any::TypeId::of::())? + .get_type_meta(); + let local_type_hash = meta.get_hash(); + let remote_type_hash = type_info.get_type_meta().get_hash(); + if remote_type_hash == local_type_hash { + ::fory_read_data(context) + } else { + for _field in fields.iter() { + match _field.field_id { + 0i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f7 = ::fory_read( + context, + true, + false, + )?; + } else { + _f7 = ::fory_read_data( + context, + )?; + } + } + 1i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f5 = ::fory_read( + context, + true, + false, + )?; + } else { + _f5 = ::fory_read_data( + context, + )?; + } + } + 2i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _last = ::fory_read( + context, + true, + false, + )?; + } else { + _last = ::fory_read_data( + context, + )?; + } + } + 3i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f4 = Some( + ::fory_read( + context, + true, + false, + )?, + ); + } else { + _f4 = Some( + ::fory_read_data(context)?, + ); + } + } + 4i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f3 = Some( + as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f3 = Some( + as fory_core::Serializer>::fory_read_data(context)?, + ); + } + } + 5i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f6 = Some( + as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f6 = Some( + as fory_core::Serializer>::fory_read_data(context)?, + ); + } + } + 6i16 => { + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + _field.field_type.type_id, + _field.field_type.nullable, + ); + if read_ref_flag { + _f1 = Some( + , + > as fory_core::Serializer>::fory_read( + context, + true, + false, + )?, + ); + } else { + _f1 = Some( + , + > as fory_core::Serializer>::fory_read_data(context)?, + ); + } + } + _ => { + let field_type = &_field.field_type; + let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( + field_type.type_id, + field_type.nullable, + ); + fory_core::serializer::skip::skip_field_value( + context, + field_type, + read_ref_flag, + )?; + } + } + } + Ok(Self { + f7: _f7, + f5: _f5, + last: _last, + f4: _f4.unwrap_or_default(), + f3: _f3.unwrap_or_default(), + f6: _f6.unwrap_or_default(), + f1: _f1.unwrap_or_default(), + }) + } + } + } + impl fory_core::Serializer for Animal2 { + #[inline(always)] + fn fory_get_type_id( + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result { + type_resolver.get_type_id(&std::any::TypeId::of::(), 1u32) + } + #[inline(always)] + fn fory_type_id_dyn( + &self, + type_resolver: &fory_core::resolver::type_resolver::TypeResolver, + ) -> Result { + Self::fory_get_type_id(type_resolver) + } + #[inline(always)] + fn as_any(&self) -> &dyn std::any::Any { + self + } + #[inline(always)] + fn FORY_STATIC_TYPE_ID -> fory_core::TypeId + where + Self: Sized, + { + fory_core::TypeId::STRUCT + } + #[inline(always)] + fn fory_reserved_space() -> usize { + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + ::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + + > as fory_core::Serializer>::fory_reserved_space() + + fory_core::types::SIZE_OF_REF_AND_TYPE + } + #[inline(always)] + fn fory_write( + &self, + context: &mut fory_core::resolver::context::WriteContext, + write_ref_info: bool, + write_type_info: bool, + _: bool, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::write::< + Self, + >(self, context, write_ref_info, write_type_info) + } + #[inline] + fn fory_write_data( + &self, + context: &mut fory_core::resolver::context::WriteContext, + ) -> Result<(), fory_core::error::Error> { + if context.is_check_struct_version() { + context.writer.write_i32(-1615591914i32); + } + ::fory_write_data(&self.f7, context)?; + ::fory_write_data(&self.f5, context)?; + ::fory_write_data(&self.last, context)?; + ::fory_write( + &self.f4, + context, + true, + false, + false, + )?; + as fory_core::Serializer>::fory_write( + &self.f3, + context, + true, + false, + false, + )?; + as fory_core::Serializer>::fory_write( + &self.f6, + context, + true, + false, + false, + )?; + , + > as fory_core::Serializer>::fory_write( + &self.f1, + context, + true, + false, + true, + )?; + Ok(()) + } + #[inline(always)] + fn fory_write_type_info( + context: &mut fory_core::resolver::context::WriteContext, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::write_type_info::(context) + } + #[inline(always)] + fn fory_read( + context: &mut fory_core::resolver::context::ReadContext, + read_ref_info: bool, + read_type_info: bool, + ) -> Result { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) + || ref_flag == (fory_core::RefFlag::RefValue as i8) + { + if context.is_compatible() { + let type_info = if read_type_info { + context.read_any_typeinfo()? + } else { + let rs_type_id = std::any::TypeId::of::(); + context.get_type_info(&rs_type_id)? + }; + ::fory_read_compatible( + context, + type_info, + ) + } else { + if read_type_info { + ::fory_read_type_info(context)?; + } + ::fory_read_data(context) + } + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) + } else { + Err( + fory_core::error::Error::invalid_ref( + ::alloc::__export::must_use({ + ::alloc::fmt::format( + format_args!("Unknown ref flag, value:{0}", ref_flag), + ) + }), + ), + ) + } + } + #[inline(always)] + fn fory_read_with_type_info( + context: &mut fory_core::resolver::context::ReadContext, + read_ref_info: bool, + type_info: std::rc::Rc, + ) -> Result { + let ref_flag = if read_ref_info { + context.reader.read_i8()? + } else { + fory_core::RefFlag::NotNullValue as i8 + }; + if ref_flag == (fory_core::RefFlag::NotNullValue as i8) + || ref_flag == (fory_core::RefFlag::RefValue as i8) + { + if context.is_compatible() { + ::fory_read_compatible( + context, + type_info, + ) + } else { + ::fory_read_data(context) + } + } else if ref_flag == (fory_core::RefFlag::Null as i8) { + Ok(::fory_default()) + } else { + Err( + fory_core::error::Error::invalid_ref( + ::alloc::__export::must_use({ + ::alloc::fmt::format( + format_args!("Unknown ref flag, value:{0}", ref_flag), + ) + }), + ), + ) + } + } + #[inline] + fn fory_read_data( + context: &mut fory_core::resolver::context::ReadContext, + ) -> Result { + if context.is_check_struct_version() { + let read_version = context.reader.read_i32()?; + let type_name = std::any::type_name::(); + fory_core::meta::TypeMeta::check_struct_version( + read_version, + -1615591914i32, + type_name, + )?; + } + let _f7 = ::fory_read_data(context)?; + let _f5 = ::fory_read_data(context)?; + let _last = ::fory_read_data(context)?; + let _f4 = ::fory_read( + context, + true, + false, + )?; + let _f3 = as fory_core::Serializer>::fory_read(context, true, false)?; + let _f6 = as fory_core::Serializer>::fory_read(context, true, false)?; + let _f1 = , + > as fory_core::Serializer>::fory_read(context, true, false)?; + Ok(Self { + f7: _f7, + f5: _f5, + last: _last, + f4: _f4, + f3: _f3, + f6: _f6, + f1: _f1, + }) + } + #[inline(always)] + fn fory_read_type_info( + context: &mut fory_core::resolver::context::ReadContext, + ) -> Result<(), fory_core::error::Error> { + fory_core::serializer::struct_::read_type_info::(context) + } + } + #[automatically_derived] + impl ::core::fmt::Debug for Animal2 { + #[inline] + fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + let names: &'static _ = &["f1", "f3", "f4", "f5", "f6", "f7", "last"]; + let values: &[&dyn ::core::fmt::Debug] = &[ + &self.f1, + &self.f3, + &self.f4, + &self.f5, + &self.f6, + &self.f7, + &&self.last, + ]; + ::core::fmt::Formatter::debug_struct_fields_finish( + f, + "Animal2", + names, + values, + ) + } + } + let mut fory1 = Fory::default().compatible(true); + let mut fory2 = Fory::default().compatible(true); + fory1.register::(999).unwrap(); + fory2.register::(999).unwrap(); + let animal: Animal1 = Animal1 { + f1: HashMap::from([(1, <[_]>::into_vec(::alloc::boxed::box_new([2])))]), + f2: String::from("hello"), + f3: <[_]>::into_vec(::alloc::boxed::box_new([1, 2, 3])), + f5: String::from("f5"), + f6: <[_]>::into_vec(::alloc::boxed::box_new([42])), + f7: 43, + last: 44, + }; + let bin = fory1.serialize(&animal).unwrap(); + let obj: Animal2 = fory2.deserialize(&bin).unwrap(); + match (&animal.f1, &obj.f1) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&animal.f3, &obj.f3) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&obj.f4, &String::default()) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&obj.f5, &i8::default()) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&obj.f6, &Vec::::default()) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&obj.f7, &i16::default()) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; + match (&animal.last, &obj.last) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + let kind = ::core::panicking::AssertKind::Eq; + ::core::panicking::assert_failed( + kind, + &*left_val, + &*right_val, + ::core::option::Option::None, + ); + } + } + }; +} +#[rustc_main] +#[coverage(off)] +#[doc(hidden)] +pub fn main() -> () { + extern crate test; + test::test_main_static(&[&test_simple]) +} diff --git a/rust/tests/tests/test_weak.rs b/rust/tests/tests/test_weak.rs index 1a5490480a..e3b561b990 100644 --- a/rust/tests/tests/test_weak.rs +++ b/rust/tests/tests/test_weak.rs @@ -25,7 +25,7 @@ use std::sync::Mutex; #[test] fn test_rc_weak_null_serialization() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let weak: RcWeak = RcWeak::new(); @@ -37,7 +37,7 @@ fn test_rc_weak_null_serialization() { #[test] fn test_arc_weak_null_serialization() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let weak: ArcWeak = ArcWeak::new(); @@ -49,7 +49,7 @@ fn test_arc_weak_null_serialization() { #[test] fn test_rc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let weak = { let rc = Rc::new(42i32); @@ -69,7 +69,7 @@ fn test_rc_weak_dead_pointer_serializes_as_null() { #[test] fn test_arc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let weak = { let arc = Arc::new(String::from("test")); @@ -89,7 +89,7 @@ fn test_arc_weak_dead_pointer_serializes_as_null() { #[test] fn test_rc_weak_in_vec_circular_reference() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let data1 = Rc::new(42i32); let data2 = Rc::new(100i32); @@ -107,7 +107,7 @@ fn test_rc_weak_in_vec_circular_reference() { #[test] fn test_arc_weak_in_vec_circular_reference() { - let fory = Fory::default(); + let fory = Fory::default().ref_tracking(true); let data1 = Arc::new(String::from("hello")); let data2 = Arc::new(String::from("world")); @@ -133,7 +133,7 @@ fn test_rc_weak_field_in_struct() { weak_ref: RcWeak, } - let mut fory = Fory::default(); + let mut fory = Fory::default().ref_tracking(true); fory.register::(1000).unwrap(); let data = Rc::new(42i32); @@ -160,7 +160,7 @@ struct Node { #[test] fn test_node_circular_reference_with_parent_children() { // Register the Node type with Fory - let mut fory = Fory::default(); + let mut fory = Fory::default().ref_tracking(true); fory.register::(2000).unwrap(); // Create parent @@ -218,7 +218,7 @@ fn test_arc_mutex_circular_reference() { children: Vec>>, } - let mut fory = Fory::default(); + let mut fory = Fory::default().ref_tracking(true); fory.register::(6000).unwrap(); let parent = Arc::new(Mutex::new(Node { From 52c7282e82a77e42e573c2c60d6dd7d559f341be Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 14:11:58 +0800 Subject: [PATCH 16/35] fix(go): generate xlang-mode conditional code for slice/map null handling In xlang mode, slices and maps are NOT nullable by default (per xlang spec). In native Go mode, slices and maps can be nil and need null flags. This commit: - Updates Go codegen (encoder.go, decoder.go) to generate conditional code that checks isXlang at runtime to determine null flag behavior - Updates struct.go and type_def.go to correctly mark slices/maps as NOT nullable in xlang mode (only pointer types are nullable) - Adds IsXlang() method to TypeResolver for runtime mode checking - Updates Go tests that expect native Go nil preservation to use WithXlang(false) This ensures proper Java-Go cross-language interoperability while maintaining native Go nil preservation when using WithXlang(false). --- go/fory/codegen/decoder.go | 376 +++++++++---- go/fory/codegen/encoder.go | 399 +++++++++---- go/fory/fory_test.go | 3 +- go/fory/struct.go | 15 +- go/fory/tests/generator_test.go | 8 +- go/fory/tests/structs_fory_gen.go | 531 +++++++++++++++--- go/fory/type_def.go | 15 +- go/fory/type_resolver.go | 5 + .../fory/builder/BaseObjectCodecBuilder.java | 66 ++- .../org/apache/fory/codegen/Expression.java | 37 ++ 10 files changed, 1138 insertions(+), 317 deletions(-) diff --git a/go/fory/codegen/decoder.go b/go/fory/codegen/decoder.go index 64d3bd7765..1eadc41b04 100644 --- a/go/fory/codegen/decoder.go +++ b/go/fory/codegen/decoder.go @@ -156,13 +156,13 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := elemType.(*types.Interface); ok && iface.Empty() { // For []interface{}, we need to manually implement the deserialization // to match our custom encoding. - // Slices are nullable in Go, so read null flag first. + // In xlang mode, slices are NOT nullable by default. + // In native Go mode, slices can be nil and need null flags. fmt.Fprintf(buf, "\t// Dynamic slice []interface{} handling - manual deserialization\n") fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make([]interface{}, 0)\n", fieldAccess) @@ -176,6 +176,26 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem())\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make([]interface{}, 0)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\t\t// ReadData collection flags (ignore for now)\n") + fmt.Fprintf(buf, "\t\t\t\t\t_ = buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\t\t\t// Create slice with proper capacity\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make([]interface{}, sliceLen)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t// ReadData each element using ReadValue\n") + fmt.Fprintf(buf, "\t\t\t\t\tfor i := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t}\n") fmt.Fprintf(buf, "\t}\n") return nil @@ -351,68 +371,90 @@ func generateSliceReadInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAcc // Check if element type is referencable (needs ref tracking) elemIsReferencable := isReferencableType(elemType) - // Slices are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // ReadData slice with null flag - use block scope to avoid variable name conflicts + // ReadData slice with conditional null flag - use block scope to avoid variable name conflicts fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, 0)\n", fieldAccess, sliceType.String()) fmt.Fprintf(buf, "\t\t\t} else {\n") + // ReadData collection header in xlang mode + if err := writeSliceReadElements(buf, sliceType, elemType, fieldAccess, elemIsReferencable, "\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t}\n") // end else (sliceLen > 0) + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make(%s, 0)\n", fieldAccess, sliceType.String()) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + // ReadData collection header in native mode + if err := writeSliceReadElements(buf, sliceType, elemType, fieldAccess, elemIsReferencable, "\t\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t\t}\n") // end else (sliceLen > 0) + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writeSliceReadElements generates the element reading code for a slice with specified indentation +func writeSliceReadElements(buf *bytes.Buffer, sliceType *types.Slice, elemType types.Type, fieldAccess string, elemIsReferencable bool, indent string) error { // ReadData collection header - fmt.Fprintf(buf, "\t\t\t\tcollectFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\t\t\t// Check if CollectionIsDeclElementType is set (bit 2, value 4)\n") - fmt.Fprintf(buf, "\t\t\t\thasDeclType := (collectFlag & 4) != 0\n") + fmt.Fprintf(buf, "%scollectFlag := buf.ReadInt8(err)\n", indent) + fmt.Fprintf(buf, "%s// Check if CollectionIsDeclElementType is set (bit 2, value 4)\n", indent) + fmt.Fprintf(buf, "%shasDeclType := (collectFlag & 4) != 0\n", indent) if elemIsReferencable { - fmt.Fprintf(buf, "\t\t\t\t// Check if CollectionTrackingRef is set (bit 0, value 1)\n") - fmt.Fprintf(buf, "\t\t\t\ttrackRefs := (collectFlag & 1) != 0\n") + fmt.Fprintf(buf, "%s// Check if CollectionTrackingRef is set (bit 0, value 1)\n", indent) + fmt.Fprintf(buf, "%strackRefs := (collectFlag & 1) != 0\n", indent) } // Create slice - fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, sliceLen)\n", fieldAccess, sliceType.String()) + fmt.Fprintf(buf, "%s%s = make(%s, sliceLen)\n", indent, fieldAccess, sliceType.String()) // ReadData elements based on whether CollectionIsDeclElementType is set - fmt.Fprintf(buf, "\t\t\t\tif hasDeclType {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// Elements are written directly without type IDs\n") - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < sliceLen; i++ {\n") + fmt.Fprintf(buf, "%sif hasDeclType {\n", indent) + fmt.Fprintf(buf, "%s\t// Elements are written directly without type IDs\n", indent) + fmt.Fprintf(buf, "%s\tfor i := 0; i < sliceLen; i++ {\n", indent) if elemIsReferencable { - // For referencable elements (like strings), need to read ref flag when tracking - fmt.Fprintf(buf, "\t\t\t\t\t\tif trackRefs {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tif trackRefs {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) } - if err := generateSliceElementReadDirect(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess)); err != nil { + if err := generateSliceElementReadDirectIndented(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess), indent+"\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// Need to read type ID once if CollectionIsSameType is set\n") - fmt.Fprintf(buf, "\t\t\t\t\tif (collectFlag & 8) != 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t// ReadData element type ID once for all elements\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t_ = buf.ReadVaruint32(err)\n") - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < sliceLen; i++ {\n") + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s} else {\n", indent) + fmt.Fprintf(buf, "%s\t// Need to read type ID once if CollectionIsSameType is set\n", indent) + fmt.Fprintf(buf, "%s\tif (collectFlag & 8) != 0 {\n", indent) + fmt.Fprintf(buf, "%s\t\t// ReadData element type ID once for all elements\n", indent) + fmt.Fprintf(buf, "%s\t\t_ = buf.ReadVaruint32(err)\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s\tfor i := 0; i < sliceLen; i++ {\n", indent) if elemIsReferencable { - // For referencable elements (like strings), need to read ref flag when tracking - fmt.Fprintf(buf, "\t\t\t\t\t\tif trackRefs {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tif trackRefs {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t_ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag)\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) } - // For same type without declared type, read elements directly - if err := generateSliceElementReadDirect(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess)); err != nil { + if err := generateSliceElementReadDirectIndented(buf, elemType, fmt.Sprintf("%s[i]", fieldAccess), indent+"\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t}\n") // end else (sliceLen > 0) - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) return nil } @@ -422,47 +464,67 @@ func generatePrimitiveSliceReadInline(buf *bytes.Buffer, sliceType *types.Slice, elemType := sliceType.Elem() basic := elemType.Underlying().(*types.Basic) - // Slices are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // Read null flag first fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, read directly without null flag\n") + + // Read primitive slice in xlang mode + if err := writePrimitiveSliceReadCall(buf, basic, fieldAccess, "\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") - // Call the exported helper function for each primitive type + // Read primitive slice in native mode + if err := writePrimitiveSliceReadCall(buf, basic, fieldAccess, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writePrimitiveSliceReadCall writes the helper function call for reading a primitive slice +func writePrimitiveSliceReadCall(buf *bytes.Buffer, basic *types.Basic, fieldAccess string, indent string) error { switch basic.Kind() { case types.Bool: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadBoolSlice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadBoolSlice(buf, err)\n", indent, fieldAccess) case types.Int8: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt8Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt8Slice(buf, err)\n", indent, fieldAccess) case types.Uint8: - fmt.Fprintf(buf, "\t\t\tsizeBytes := buf.ReadLength(err)\n") - fmt.Fprintf(buf, "\t\t\t%s = make([]uint8, sizeBytes)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tif sizeBytes > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\traw := buf.ReadBinary(sizeBytes, err)\n") - fmt.Fprintf(buf, "\t\t\t\tif raw != nil {\n") - fmt.Fprintf(buf, "\t\t\t\t\tcopy(%s, raw)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "%ssizeBytes := buf.ReadLength(err)\n", indent) + fmt.Fprintf(buf, "%s%s = make([]uint8, sizeBytes)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%sif sizeBytes > 0 {\n", indent) + fmt.Fprintf(buf, "%s\traw := buf.ReadBinary(sizeBytes, err)\n", indent) + fmt.Fprintf(buf, "%s\tif raw != nil {\n", indent) + fmt.Fprintf(buf, "%s\t\tcopy(%s, raw)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\t}\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) case types.Int16: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt16Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt16Slice(buf, err)\n", indent, fieldAccess) case types.Int32: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt32Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt32Slice(buf, err)\n", indent, fieldAccess) case types.Int64: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadInt64Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadInt64Slice(buf, err)\n", indent, fieldAccess) case types.Float32: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadFloat32Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadFloat32Slice(buf, err)\n", indent, fieldAccess) case types.Float64: - fmt.Fprintf(buf, "\t\t\t%s = fory.ReadFloat64Slice(buf, err)\n", fieldAccess) + fmt.Fprintf(buf, "%s%s = fory.ReadFloat64Slice(buf, err)\n", indent, fieldAccess) default: return fmt.Errorf("unsupported primitive type for ARRAY protocol read: %s", basic.String()) } - - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope return nil } @@ -599,6 +661,43 @@ func generateSliceElementReadDirect(buf *bytes.Buffer, elemType types.Type, elem return fmt.Errorf("unsupported element type for direct read: %s", elemType.String()) } +// generateSliceElementReadDirectIndented generates code to read slice elements directly with custom indentation +func generateSliceElementReadDirectIndented(buf *bytes.Buffer, elemType types.Type, elemAccess string, indent string) error { + if basic, ok := elemType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Bool: + fmt.Fprintf(buf, "%s%s = buf.ReadBool(err)\n", indent, elemAccess) + case types.Int8: + fmt.Fprintf(buf, "%s%s = buf.ReadInt8(err)\n", indent, elemAccess) + case types.Int16: + fmt.Fprintf(buf, "%s%s = buf.ReadInt16(err)\n", indent, elemAccess) + case types.Int32: + fmt.Fprintf(buf, "%s%s = buf.ReadVarint32(err)\n", indent, elemAccess) + case types.Int, types.Int64: + fmt.Fprintf(buf, "%s%s = buf.ReadVarint64(err)\n", indent, elemAccess) + case types.Uint8: + fmt.Fprintf(buf, "%s%s = buf.ReadByte(err)\n", indent, elemAccess) + case types.Uint16: + fmt.Fprintf(buf, "%s%s = uint16(buf.ReadInt16(err))\n", indent, elemAccess) + case types.Uint32: + fmt.Fprintf(buf, "%s%s = uint32(buf.ReadInt32(err))\n", indent, elemAccess) + case types.Uint, types.Uint64: + fmt.Fprintf(buf, "%s%s = uint64(buf.ReadInt64(err))\n", indent, elemAccess) + case types.Float32: + fmt.Fprintf(buf, "%s%s = buf.ReadFloat32(err)\n", indent, elemAccess) + case types.Float64: + fmt.Fprintf(buf, "%s%s = buf.ReadFloat64(err)\n", indent, elemAccess) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, elemAccess) + default: + return fmt.Errorf("unsupported basic type for direct element read: %s", basic.String()) + } + return nil + } + + return fmt.Errorf("unsupported element type for direct read: %s", elemType.String()) +} + // generateMapReadInline generates inline map deserialization code following the chunk-based format // Uses error-aware methods for deferred error checking func generateMapReadInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess string) error { @@ -615,77 +714,100 @@ func generateMapReadInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess st valueIsInterface = true } - // Maps are nullable in Go (can be nil), so we need to read a null flag. - // This matches reflection behavior in struct.go where maps have nullable=true. + // In xlang mode, maps are NOT nullable by default (only pointer types are nullable). + // In native Go mode, maps can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // ReadData map with null flag + // ReadData map with conditional null flag fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tnullFlag := buf.ReadInt8(err)\n") - fmt.Fprintf(buf, "\t\tif nullFlag == -3 {\n") // NullFlag - fmt.Fprintf(buf, "\t\t\t%s = nil\n", fieldAccess) - fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: maps are not nullable, read directly without null flag\n") fmt.Fprintf(buf, "\t\t\tmapLen := int(buf.ReadVaruint32(err))\n") fmt.Fprintf(buf, "\t\t\tif mapLen == 0 {\n") fmt.Fprintf(buf, "\t\t\t\t%s = make(%s)\n", fieldAccess, mapType.String()) fmt.Fprintf(buf, "\t\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\t\t%s = make(%s, mapLen)\n", fieldAccess, mapType.String()) - fmt.Fprintf(buf, "\t\t\t\tmapSize := mapLen\n") + // Read map chunks in xlang mode + if err := writeMapReadChunks(buf, mapType, fieldAccess, keyType, valueType, keyIsInterface, valueIsInterface, "\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t}\n") // end else (mapLen > 0) + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: maps are nullable, read null flag\n") + fmt.Fprintf(buf, "\t\t\tnullFlag := buf.ReadInt8(err)\n") + fmt.Fprintf(buf, "\t\t\tif nullFlag == -3 {\n") // NullFlag + fmt.Fprintf(buf, "\t\t\t\t%s = nil\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tmapLen := int(buf.ReadVaruint32(err))\n") + fmt.Fprintf(buf, "\t\t\t\tif mapLen == 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t%s = make(%s)\n", fieldAccess, mapType.String()) + fmt.Fprintf(buf, "\t\t\t\t} else {\n") + // Read map chunks in native mode + if err := writeMapReadChunks(buf, mapType, fieldAccess, keyType, valueType, keyIsInterface, valueIsInterface, "\t\t\t\t\t"); err != nil { + return err + } + fmt.Fprintf(buf, "\t\t\t\t}\n") // end else (mapLen > 0) + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not null) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + + return nil +} + +// writeMapReadChunks generates the map chunk reading code with specified indentation +func writeMapReadChunks(buf *bytes.Buffer, mapType *types.Map, fieldAccess string, keyType, valueType types.Type, keyIsInterface, valueIsInterface bool, indent string) error { + fmt.Fprintf(buf, "%s%s = make(%s, mapLen)\n", indent, fieldAccess, mapType.String()) + fmt.Fprintf(buf, "%smapSize := mapLen\n", indent) // ReadData chunks - fmt.Fprintf(buf, "\t\t\t\tfor mapSize > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t// ReadData KV header\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader := buf.ReadByte(err)\n") - fmt.Fprintf(buf, "\t\t\t\t\tchunkSize := int(buf.ReadByte(err))\n") + fmt.Fprintf(buf, "%sfor mapSize > 0 {\n", indent) + fmt.Fprintf(buf, "%s\t// ReadData KV header\n", indent) + fmt.Fprintf(buf, "%s\tkvHeader := buf.ReadByte(err)\n", indent) + fmt.Fprintf(buf, "%s\tchunkSize := int(buf.ReadByte(err))\n", indent) // Parse header flags - fmt.Fprintf(buf, "\t\t\t\t\ttrackKeyRef := (kvHeader & 0x1) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\tkeyNotDeclared := (kvHeader & 0x4) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\ttrackValueRef := (kvHeader & 0x8) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\tvalueNotDeclared := (kvHeader & 0x20) != 0\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = trackKeyRef\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = keyNotDeclared\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = trackValueRef\n") - fmt.Fprintf(buf, "\t\t\t\t\t_ = valueNotDeclared\n") + fmt.Fprintf(buf, "%s\ttrackKeyRef := (kvHeader & 0x1) != 0\n", indent) + fmt.Fprintf(buf, "%s\tkeyNotDeclared := (kvHeader & 0x4) != 0\n", indent) + fmt.Fprintf(buf, "%s\ttrackValueRef := (kvHeader & 0x8) != 0\n", indent) + fmt.Fprintf(buf, "%s\tvalueNotDeclared := (kvHeader & 0x20) != 0\n", indent) + fmt.Fprintf(buf, "%s\t_ = trackKeyRef\n", indent) + fmt.Fprintf(buf, "%s\t_ = keyNotDeclared\n", indent) + fmt.Fprintf(buf, "%s\t_ = trackValueRef\n", indent) + fmt.Fprintf(buf, "%s\t_ = valueNotDeclared\n", indent) // ReadData key-value pairs in this chunk - fmt.Fprintf(buf, "\t\t\t\t\tfor i := 0; i < chunkSize; i++ {\n") + fmt.Fprintf(buf, "%s\tfor i := 0; i < chunkSize; i++ {\n", indent) // ReadData key if keyIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapKey interface{}\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem())\n") + fmt.Fprintf(buf, "%s\t\tvar mapKey interface{}\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem())\n", indent) } else { - // Declare key variable with appropriate type keyVarType := getGoTypeString(keyType) - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapKey %s\n", keyVarType) - if err := generateMapKeyRead(buf, keyType, "mapKey"); err != nil { + fmt.Fprintf(buf, "%s\t\tvar mapKey %s\n", indent, keyVarType) + if err := generateMapKeyReadIndented(buf, keyType, "mapKey", indent+"\t\t"); err != nil { return err } } // ReadData value if valueIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapValue interface{}\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem())\n") + fmt.Fprintf(buf, "%s\t\tvar mapValue interface{}\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem())\n", indent) } else { - // Declare value variable with appropriate type valueVarType := getGoTypeString(valueType) - fmt.Fprintf(buf, "\t\t\t\t\t\tvar mapValue %s\n", valueVarType) - if err := generateMapValueRead(buf, valueType, "mapValue"); err != nil { + fmt.Fprintf(buf, "%s\t\tvar mapValue %s\n", indent, valueVarType) + if err := generateMapValueReadIndented(buf, valueType, "mapValue", indent+"\t\t"); err != nil { return err } } // Set key-value pair in map - fmt.Fprintf(buf, "\t\t\t\t\t\t%s[mapKey] = mapValue\n", fieldAccess) - - fmt.Fprintf(buf, "\t\t\t\t\t}\n") // end chunk loop - fmt.Fprintf(buf, "\t\t\t\t\tmapSize -= chunkSize\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") // end mapSize > 0 loop + fmt.Fprintf(buf, "%s\t\t%s[mapKey] = mapValue\n", indent, fieldAccess) - fmt.Fprintf(buf, "\t\t\t}\n") // end else (mapLen > 0) - fmt.Fprintf(buf, "\t\t}\n") // end else (not null) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s\t}\n", indent) // end chunk loop + fmt.Fprintf(buf, "%s\tmapSize -= chunkSize\n", indent) + fmt.Fprintf(buf, "%s}\n", indent) // end mapSize > 0 loop return nil } @@ -751,3 +873,37 @@ func generateMapValueRead(buf *bytes.Buffer, valueType types.Type, varName strin fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", varName) return nil } + +// generateMapKeyReadIndented generates code to read a map key with custom indentation +func generateMapKeyReadIndented(buf *bytes.Buffer, keyType types.Type, varName string, indent string) error { + if basic, ok := keyType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + fmt.Fprintf(buf, "%s%s = int(buf.ReadInt64(err))\n", indent, varName) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, varName) + default: + return fmt.Errorf("unsupported map key type: %v", keyType) + } + return nil + } + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", indent, varName) + return nil +} + +// generateMapValueReadIndented generates code to read a map value with custom indentation +func generateMapValueReadIndented(buf *bytes.Buffer, valueType types.Type, varName string, indent string) error { + if basic, ok := valueType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + fmt.Fprintf(buf, "%s%s = int(buf.ReadInt64(err))\n", indent, varName) + case types.String: + fmt.Fprintf(buf, "%s%s = ctx.ReadString()\n", indent, varName) + default: + return fmt.Errorf("unsupported map value type: %v", valueType) + } + return nil + } + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", indent, varName) + return nil +} diff --git a/go/fory/codegen/encoder.go b/go/fory/codegen/encoder.go index 28c866c7be..81996f4cba 100644 --- a/go/fory/codegen/encoder.go +++ b/go/fory/codegen/encoder.go @@ -143,14 +143,17 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := elemType.(*types.Interface); ok && iface.Empty() { // For []interface{}, we need to manually implement the serialization // because WriteValue produces incorrect length encoding. - // Slices are nullable in Go, so write null flag first. + // In xlang mode, slices are NOT nullable by default. + // In native Go mode, slices can be nil and need null flags. fmt.Fprintf(buf, "\t// Dynamic slice []interface{} handling - manual serialization\n") fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tsliceLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tsliceLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") fmt.Fprintf(buf, "\t\t\tif sliceLen > 0 {\n") fmt.Fprintf(buf, "\t\t\t\t// WriteData collection flags for dynamic slice []interface{}\n") @@ -161,6 +164,24 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem))\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") + fmt.Fprintf(buf, "\t\t\t\tif sliceLen > 0 {\n") + fmt.Fprintf(buf, "\t\t\t\t\t// WriteData collection flags for dynamic slice []interface{}\n") + fmt.Fprintf(buf, "\t\t\t\t\t// Only CollectionTrackingRef is set (no declared type, may have different types)\n") + fmt.Fprintf(buf, "\t\t\t\t\tbuf.WriteInt8(1) // CollectionTrackingRef only\n") + fmt.Fprintf(buf, "\t\t\t\t\t// WriteData each element using WriteValue\n") + fmt.Fprintf(buf, "\t\t\t\t\tfor _, elem := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem))\n") + fmt.Fprintf(buf, "\t\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t}\n") fmt.Fprintf(buf, "\t}\n") return nil @@ -274,28 +295,23 @@ func generateSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAc // Check if element type is referencable (needs ref tracking) elemIsReferencable := isReferencableType(elemType) - // Slices are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. - // RefMode is either RefModeTracking (if trackRef && nullable) or RefModeNullOnly (if nullable only). - // Since codegen always writes null flag for slices, we match the RefModeNullOnly behavior. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // WriteData slice with null flag - use block scope to avoid variable name conflicts + // WriteData slice with conditional null flag - use block scope to avoid variable name conflicts fmt.Fprintf(buf, "\t{\n") - // Write null flag for slices (nullable=true) - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tsliceLen := len(%s)\n", fieldAccess) + // Check if xlang mode - in xlang mode, slices are not nullable by default + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tsliceLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tsliceLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") - - // WriteData collection header and elements for non-empty slice + // Write elements in xlang mode fmt.Fprintf(buf, "\t\t\tif sliceLen > 0 {\n") - - // For codegen, follow reflection's behavior for struct fields: - // Set both CollectionIsSameType and CollectionIsDeclElementType - // Add CollectionTrackingRef when ref tracking is enabled AND element is referencable - // This matches sliceConcreteValueSerializer.WriteData which adds CollectionTrackingRef for referencable elements fmt.Fprintf(buf, "\t\t\t\tcollectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType\n") if elemIsReferencable { fmt.Fprintf(buf, "\t\t\t\tif ctx.TrackRef() {\n") @@ -303,26 +319,57 @@ func generateSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice, fieldAc fmt.Fprintf(buf, "\t\t\t\t}\n") } fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(int8(collectFlag))\n") - - // Element type ID is NOT written when CollectionIsDeclElementType is set - // The reader knows the element type from the field type - - // WriteData elements - with ref flags if element is referencable and tracking is enabled fmt.Fprintf(buf, "\t\t\t\tfor _, elem := range %s {\n", fieldAccess) if elemIsReferencable { - // For referencable elements (like strings), need to write ref flag when tracking fmt.Fprintf(buf, "\t\t\t\t\tif ctx.TrackRef() {\n") fmt.Fprintf(buf, "\t\t\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag for element\n") fmt.Fprintf(buf, "\t\t\t\t\t}\n") } - if err := generateSliceElementWriteInline(buf, elemType, "elem"); err != nil { + if err := generateSliceElementWriteInlineIndented(buf, elemType, "elem", "\t\t\t\t\t"); err != nil { return err } - fmt.Fprintf(buf, "\t\t\t\t}\n") // end for loop fmt.Fprintf(buf, "\t\t\t}\n") // end if sliceLen > 0 - fmt.Fprintf(buf, "\t\t}\n") // end else (not nil) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + // Write null flag for slices in native mode + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tsliceLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(sliceLen))\n") + + // WriteData collection header and elements for non-empty slice in native mode + fmt.Fprintf(buf, "\t\t\t\tif sliceLen > 0 {\n") + + // For codegen, follow reflection's behavior for struct fields: + // Set both CollectionIsSameType and CollectionIsDeclElementType + // Add CollectionTrackingRef when ref tracking is enabled AND element is referencable + fmt.Fprintf(buf, "\t\t\t\t\tcollectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType\n") + if elemIsReferencable { + fmt.Fprintf(buf, "\t\t\t\t\tif ctx.TrackRef() {\n") + fmt.Fprintf(buf, "\t\t\t\t\t\tcollectFlag |= 1 // CollectionTrackingRef for referencable element type\n") + fmt.Fprintf(buf, "\t\t\t\t\t}\n") + } + fmt.Fprintf(buf, "\t\t\t\t\tbuf.WriteInt8(int8(collectFlag))\n") + + // WriteData elements - with ref flags if element is referencable and tracking is enabled + fmt.Fprintf(buf, "\t\t\t\t\tfor _, elem := range %s {\n", fieldAccess) + if elemIsReferencable { + fmt.Fprintf(buf, "\t\t\t\t\t\tif ctx.TrackRef() {\n") + fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag for element\n") + fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") + } + if err := generateSliceElementWriteInlineIndented(buf, elemType, "elem", "\t\t\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t\t\t}\n") // end for loop + fmt.Fprintf(buf, "\t\t\t\t}\n") // end if sliceLen > 0 + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) in native mode + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope return nil } @@ -344,41 +391,63 @@ func generatePrimitiveSliceWriteInline(buf *bytes.Buffer, sliceType *types.Slice elemType := sliceType.Elem() basic := elemType.Underlying().(*types.Basic) - // Slices are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where slices have nullable=true. - // Write null flag first, then call the helper function for the actual data. + // In xlang mode, slices are NOT nullable by default (only pointer types are nullable). + // In native Go mode, slices can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - fmt.Fprintf(buf, "\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t} else {\n") - fmt.Fprintf(buf, "\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t{\n") + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: slices are not nullable, write directly without null flag\n") - // Call the exported helper function for each primitive type + // Write primitive slice directly in xlang mode + if err := writePrimitiveSliceCall(buf, basic, fieldAccess, "\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: slices are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + + // Write primitive slice in native mode + if err := writePrimitiveSliceCall(buf, basic, fieldAccess, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + return nil +} + +// writePrimitiveSliceCall writes the helper function call for a primitive slice type +func writePrimitiveSliceCall(buf *bytes.Buffer, basic *types.Basic, fieldAccess string, indent string) error { switch basic.Kind() { case types.Bool: - fmt.Fprintf(buf, "\t\tfory.WriteBoolSlice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteBoolSlice(buf, %s)\n", indent, fieldAccess) case types.Int8: - fmt.Fprintf(buf, "\t\tfory.WriteInt8Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt8Slice(buf, %s)\n", indent, fieldAccess) case types.Uint8: - fmt.Fprintf(buf, "\t\tbuf.WriteLength(len(%s))\n", fieldAccess) - fmt.Fprintf(buf, "\t\tif len(%s) > 0 {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteBinary(%s)\n", fieldAccess) - fmt.Fprintf(buf, "\t\t}\n") + fmt.Fprintf(buf, "%sbuf.WriteLength(len(%s))\n", indent, fieldAccess) + fmt.Fprintf(buf, "%sif len(%s) > 0 {\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\tbuf.WriteBinary(%s)\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s}\n", indent) case types.Int16: - fmt.Fprintf(buf, "\t\tfory.WriteInt16Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt16Slice(buf, %s)\n", indent, fieldAccess) case types.Int32: - fmt.Fprintf(buf, "\t\tfory.WriteInt32Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt32Slice(buf, %s)\n", indent, fieldAccess) case types.Int64: - fmt.Fprintf(buf, "\t\tfory.WriteInt64Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteInt64Slice(buf, %s)\n", indent, fieldAccess) case types.Float32: - fmt.Fprintf(buf, "\t\tfory.WriteFloat32Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteFloat32Slice(buf, %s)\n", indent, fieldAccess) case types.Float64: - fmt.Fprintf(buf, "\t\tfory.WriteFloat64Slice(buf, %s)\n", fieldAccess) + fmt.Fprintf(buf, "%sfory.WriteFloat64Slice(buf, %s)\n", indent, fieldAccess) default: return fmt.Errorf("unsupported primitive type for ARRAY protocol: %s", basic.String()) } - - fmt.Fprintf(buf, "\t}\n") return nil } @@ -397,104 +466,133 @@ func generateMapWriteInline(buf *bytes.Buffer, mapType *types.Map, fieldAccess s valueIsInterface = true } - // Maps are nullable in Go (can be nil), so we need to write a null flag. - // This matches reflection behavior in struct.go where maps have nullable=true. + // In xlang mode, maps are NOT nullable by default (only pointer types are nullable). + // In native Go mode, maps can be nil and need null flags. + // Generate conditional code that respects the mode at runtime. - // WriteData map with null flag + // WriteData map with conditional null flag fmt.Fprintf(buf, "\t{\n") - fmt.Fprintf(buf, "\t\tif %s == nil {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-3) // NullFlag\n") - fmt.Fprintf(buf, "\t\t} else {\n") - fmt.Fprintf(buf, "\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") - fmt.Fprintf(buf, "\t\t\tmapLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\tisXlang := ctx.TypeResolver().IsXlang()\n") + fmt.Fprintf(buf, "\t\tif isXlang {\n") + fmt.Fprintf(buf, "\t\t\t// xlang mode: maps are not nullable, write directly without null flag\n") + fmt.Fprintf(buf, "\t\t\tmapLen := 0\n") + fmt.Fprintf(buf, "\t\t\tif %s != nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tmapLen = len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t\tbuf.WriteVaruint32(uint32(mapLen))\n") + // Write map chunks in xlang mode + if err := writeMapChunksCode(buf, keyType, valueType, fieldAccess, keyIsInterface, valueIsInterface, "\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t// Native Go mode: maps are nullable, write null flag\n") + fmt.Fprintf(buf, "\t\t\tif %s == nil {\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-3) // NullFlag\n") + fmt.Fprintf(buf, "\t\t\t} else {\n") + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(-1) // NotNullValueFlag\n") + fmt.Fprintf(buf, "\t\t\t\tmapLen := len(%s)\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\tbuf.WriteVaruint32(uint32(mapLen))\n") + + // Write map chunks in native mode + if err := writeMapChunksCode(buf, keyType, valueType, fieldAccess, keyIsInterface, valueIsInterface, "\t\t\t\t"); err != nil { + return err + } + + fmt.Fprintf(buf, "\t\t\t}\n") // end else (not nil) + fmt.Fprintf(buf, "\t\t}\n") // end else (native mode) + fmt.Fprintf(buf, "\t}\n") // end block scope + + return nil +} + +// writeMapChunksCode generates the map chunk writing code with specified indentation +func writeMapChunksCode(buf *bytes.Buffer, keyType, valueType types.Type, fieldAccess string, keyIsInterface, valueIsInterface bool, indent string) error { // WriteData chunks for non-empty map - fmt.Fprintf(buf, "\t\t\tif mapLen > 0 {\n") + fmt.Fprintf(buf, "%sif mapLen > 0 {\n", indent) // Calculate KV header based on types - fmt.Fprintf(buf, "\t\t\t\t// Calculate KV header flags\n") - fmt.Fprintf(buf, "\t\t\t\tkvHeader := uint8(0)\n") + fmt.Fprintf(buf, "%s\t// Calculate KV header flags\n", indent) + fmt.Fprintf(buf, "%s\tkvHeader := uint8(0)\n", indent) // Check if ref tracking is enabled - fmt.Fprintf(buf, "\t\t\t\tisRefTracking := ctx.TrackRef()\n") - fmt.Fprintf(buf, "\t\t\t\t_ = isRefTracking // Mark as used to avoid warning\n") + fmt.Fprintf(buf, "%s\tisRefTracking := ctx.TrackRef()\n", indent) + fmt.Fprintf(buf, "%s\t_ = isRefTracking // Mark as used to avoid warning\n", indent) // Set header flags based on type properties if !keyIsInterface { // For concrete key types, check if they're referencable if isReferencableType(keyType) { - fmt.Fprintf(buf, "\t\t\t\tif isRefTracking {\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader |= 0x1 // track key ref\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif isRefTracking {\n", indent) + fmt.Fprintf(buf, "%s\t\tkvHeader |= 0x1 // track key ref\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) } } else { // For interface{} keys, always set not declared type flag - fmt.Fprintf(buf, "\t\t\t\tkvHeader |= 0x4 // key type not declared\n") + fmt.Fprintf(buf, "%s\tkvHeader |= 0x4 // key type not declared\n", indent) } if !valueIsInterface { // For concrete value types, check if they're referencable if isReferencableType(valueType) { - fmt.Fprintf(buf, "\t\t\t\tif isRefTracking {\n") - fmt.Fprintf(buf, "\t\t\t\t\tkvHeader |= 0x8 // track value ref\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif isRefTracking {\n", indent) + fmt.Fprintf(buf, "%s\t\tkvHeader |= 0x8 // track value ref\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) } } else { // For interface{} values, always set not declared type flag - fmt.Fprintf(buf, "\t\t\t\tkvHeader |= 0x20 // value type not declared\n") + fmt.Fprintf(buf, "%s\tkvHeader |= 0x20 // value type not declared\n", indent) } // WriteData map elements in chunks - fmt.Fprintf(buf, "\t\t\t\tchunkSize := 0\n") - fmt.Fprintf(buf, "\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n") - fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n") - fmt.Fprintf(buf, "\t\t\t\tchunkSizeOffset := buf.WriterIndex()\n") - fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n") + fmt.Fprintf(buf, "%s\tchunkSize := 0\n", indent) + fmt.Fprintf(buf, "%s\t_ = buf.WriterIndex() // chunkHeaderOffset\n", indent) + fmt.Fprintf(buf, "%s\tbuf.WriteInt8(int8(kvHeader)) // KV header\n", indent) + fmt.Fprintf(buf, "%s\tchunkSizeOffset := buf.WriterIndex()\n", indent) + fmt.Fprintf(buf, "%s\tbuf.WriteInt8(0) // placeholder for chunk size\n", indent) - fmt.Fprintf(buf, "\t\t\t\tfor mapKey, mapValue := range %s {\n", fieldAccess) + fmt.Fprintf(buf, "%s\tfor mapKey, mapValue := range %s {\n", indent, fieldAccess) // WriteData key if keyIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(mapKey))\n") + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapKey))\n", indent) } else { - if err := generateMapKeyWrite(buf, keyType, "mapKey"); err != nil { + if err := generateMapKeyWriteIndented(buf, keyType, "mapKey", indent+"\t\t"); err != nil { return err } } // WriteData value if valueIsInterface { - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(mapValue))\n") + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapValue))\n", indent) } else { - if err := generateMapValueWrite(buf, valueType, "mapValue"); err != nil { + if err := generateMapValueWriteIndented(buf, valueType, "mapValue", indent+"\t\t"); err != nil { return err } } - fmt.Fprintf(buf, "\t\t\t\t\tchunkSize++\n") - fmt.Fprintf(buf, "\t\t\t\t\tif chunkSize >= 255 {\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t// WriteData chunk size and start new chunk\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n") - fmt.Fprintf(buf, "\t\t\t\t\t\tif len(%s) > chunkSize {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\t\t\tchunkSize = 0\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tchunkSizeOffset = buf.WriterIndex()\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n") - fmt.Fprintf(buf, "\t\t\t\t\t\t}\n") - fmt.Fprintf(buf, "\t\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\t\tchunkSize++\n", indent) + fmt.Fprintf(buf, "%s\t\tif chunkSize >= 255 {\n", indent) + fmt.Fprintf(buf, "%s\t\t\t// WriteData chunk size and start new chunk\n", indent) + fmt.Fprintf(buf, "%s\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n", indent) + fmt.Fprintf(buf, "%s\t\t\tif len(%s) > chunkSize {\n", indent, fieldAccess) + fmt.Fprintf(buf, "%s\t\t\t\tchunkSize = 0\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\t_ = buf.WriterIndex() // chunkHeaderOffset\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tbuf.WriteInt8(int8(kvHeader)) // KV header\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tchunkSizeOffset = buf.WriterIndex()\n", indent) + fmt.Fprintf(buf, "%s\t\t\t\tbuf.WriteInt8(0) // placeholder for chunk size\n", indent) + fmt.Fprintf(buf, "%s\t\t\t}\n", indent) + fmt.Fprintf(buf, "%s\t\t}\n", indent) - fmt.Fprintf(buf, "\t\t\t\t}\n") // end for loop + fmt.Fprintf(buf, "%s\t}\n", indent) // end for loop // WriteData final chunk size - fmt.Fprintf(buf, "\t\t\t\tif chunkSize > 0 {\n") - fmt.Fprintf(buf, "\t\t\t\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n") - fmt.Fprintf(buf, "\t\t\t\t}\n") + fmt.Fprintf(buf, "%s\tif chunkSize > 0 {\n", indent) + fmt.Fprintf(buf, "%s\t\tbuf.PutUint8(chunkSizeOffset, uint8(chunkSize))\n", indent) + fmt.Fprintf(buf, "%s\t}\n", indent) - fmt.Fprintf(buf, "\t\t\t}\n") // end if mapLen > 0 - fmt.Fprintf(buf, "\t\t}\n") // end else (not nil) - fmt.Fprintf(buf, "\t}\n") // end block scope + fmt.Fprintf(buf, "%s}\n", indent) // end if mapLen > 0 return nil } @@ -569,6 +667,50 @@ func generateMapValueWrite(buf *bytes.Buffer, valueType types.Type, varName stri return nil } +// generateMapKeyWriteIndented generates code to write a map key with custom indentation +func generateMapKeyWriteIndented(buf *bytes.Buffer, keyType types.Type, varName string, indent string) error { + // For basic types, match reflection's serializer behavior + if basic, ok := keyType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + // intSerializer uses WriteInt64, not WriteVarint64 + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, varName) + case types.String: + // stringSerializer.NeedWriteRef() = false, write directly + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, varName) + default: + return fmt.Errorf("unsupported map key type: %v", keyType) + } + return nil + } + + // For other types, use WriteValue + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, varName) + return nil +} + +// generateMapValueWriteIndented generates code to write a map value with custom indentation +func generateMapValueWriteIndented(buf *bytes.Buffer, valueType types.Type, varName string, indent string) error { + // For basic types, match reflection's serializer behavior + if basic, ok := valueType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Int: + // intSerializer uses WriteInt64, not WriteVarint64 + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, varName) + case types.String: + // stringSerializer.NeedWriteRef() = false, write directly + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, varName) + default: + return fmt.Errorf("unsupported map value type: %v", valueType) + } + return nil + } + + // For other types, use WriteValue + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, varName) + return nil +} + // generateElementTypeIDWriteInline generates element type ID write with specific indentation func generateElementTypeIDWriteInline(buf *bytes.Buffer, elemType types.Type) error { // Handle basic types @@ -652,3 +794,50 @@ func generateSliceElementWriteInline(buf *bytes.Buffer, elemType types.Type, ele return fmt.Errorf("unsupported element type for write: %s", elemType.String()) } + +// generateSliceElementWriteInlineIndented generates code to write a single slice element value with custom indentation +func generateSliceElementWriteInlineIndented(buf *bytes.Buffer, elemType types.Type, elemAccess string, indent string) error { + // Handle basic types - write the actual value without type info (type already written above) + if basic, ok := elemType.Underlying().(*types.Basic); ok { + switch basic.Kind() { + case types.Bool: + fmt.Fprintf(buf, "%sbuf.WriteBool(%s)\n", indent, elemAccess) + case types.Int8: + fmt.Fprintf(buf, "%sbuf.WriteInt8(%s)\n", indent, elemAccess) + case types.Int16: + fmt.Fprintf(buf, "%sbuf.WriteInt16(%s)\n", indent, elemAccess) + case types.Int32: + fmt.Fprintf(buf, "%sbuf.WriteVarint32(%s)\n", indent, elemAccess) + case types.Int, types.Int64: + fmt.Fprintf(buf, "%sbuf.WriteVarint64(%s)\n", indent, elemAccess) + case types.Uint8: + fmt.Fprintf(buf, "%sbuf.WriteByte_(%s)\n", indent, elemAccess) + case types.Uint16: + fmt.Fprintf(buf, "%sbuf.WriteInt16(int16(%s))\n", indent, elemAccess) + case types.Uint32: + fmt.Fprintf(buf, "%sbuf.WriteInt32(int32(%s))\n", indent, elemAccess) + case types.Uint, types.Uint64: + fmt.Fprintf(buf, "%sbuf.WriteInt64(int64(%s))\n", indent, elemAccess) + case types.Float32: + fmt.Fprintf(buf, "%sbuf.WriteFloat32(%s)\n", indent, elemAccess) + case types.Float64: + fmt.Fprintf(buf, "%sbuf.WriteFloat64(%s)\n", indent, elemAccess) + case types.String: + fmt.Fprintf(buf, "%sctx.WriteString(%s)\n", indent, elemAccess) + default: + return fmt.Errorf("unsupported basic type for element write: %s", basic.String()) + } + return nil + } + + // Handle interface types + if iface, ok := elemType.(*types.Interface); ok { + if iface.Empty() { + // For interface{} elements, use WriteValue for dynamic type handling + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, elemAccess) + return nil + } + } + + return fmt.Errorf("unsupported element type for write: %s", elemType.String()) +} diff --git a/go/fory/fory_test.go b/go/fory/fory_test.go index e60735916d..c74c6be6cf 100644 --- a/go/fory/fory_test.go +++ b/go/fory/fory_test.go @@ -222,7 +222,8 @@ func TestSerializeArray(t *testing.T) { func TestSerializeStructSimple(t *testing.T) { for _, referenceTracking := range []bool{false, true} { - fory := NewFory(WithRefTracking(referenceTracking)) + // Use WithXlang(false) for native Go mode where nil slices/maps are preserved + fory := NewFory(WithXlang(false), WithRefTracking(referenceTracking)) type A struct { F1 []string } diff --git a/go/fory/struct.go b/go/fory/struct.go index 6b3a6a7956..66add6b587 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -1092,17 +1092,14 @@ func (s *structSerializer) initFieldsFromTypeResolver(typeResolver *TypeResolver isEnum := internalId == ENUM || internalId == NAMED_ENUM // Determine nullable based on mode - // In both xlang and native mode, nil-able types (ptr, slice, map, interface) are nullable. - // This is necessary for correct null flag handling when serializing/deserializing. - // Note: In xlang mode, we don't set nullable for interface{} fields because the - // actual type determines nullability; for slices and maps, they can be nil. + // In xlang mode: only pointer types are nullable by default (per xlang spec) + // In native mode: Go's natural semantics - all nil-able types are nullable + // This ensures proper interoperability with Java/other languages in xlang mode. var nullableFlag bool if typeResolver.fory.config.IsXlang { - // xlang mode: pointers, slices, and maps are nullable - // (they can be nil and need null flag handling) - nullableFlag = fieldType.Kind() == reflect.Ptr || - fieldType.Kind() == reflect.Slice || - fieldType.Kind() == reflect.Map + // xlang mode: only pointer types are nullable by default per xlang spec + // Slices and maps are NOT nullable - they serialize as empty when nil + nullableFlag = fieldType.Kind() == reflect.Ptr } else { // Native mode: Go's natural semantics - all nil-able types are nullable nullableFlag = fieldType.Kind() == reflect.Ptr || diff --git a/go/fory/tests/generator_test.go b/go/fory/tests/generator_test.go index f5454bfad5..bbfbf5526e 100644 --- a/go/fory/tests/generator_test.go +++ b/go/fory/tests/generator_test.go @@ -135,12 +135,14 @@ func TestDynamicSliceDemo(t *testing.T) { func TestDynamicSliceDemoWithNilAndEmpty(t *testing.T) { // Test with nil and empty dynamic slices + // Use WithXlang(false) for native Go mode where nil slices are preserved original := &DynamicSliceDemo{ DynamicSlice: nil, // nil slice } // SerializeWithCallback using generated code - f := fory.NewFory(fory.WithRefTracking(true)) + // WithXlang(false) enables native Go mode where nil slices are preserved as nil + f := fory.NewFory(fory.WithXlang(false), fory.WithRefTracking(true)) data, err := f.Marshal(original) require.NoError(t, err, "Serialization should not fail") require.NotEmpty(t, data, "Serialized data should not be empty") @@ -174,6 +176,7 @@ func TestDynamicSliceDemoWithNilAndEmpty(t *testing.T) { // TestMapDemo tests basic map serialization and deserialization (including nil maps) func TestMapDemo(t *testing.T) { // Create test instance with various map types (including nil) + // Use WithXlang(false) for native Go mode where nil maps are preserved instance := &MapDemo{ StringMap: map[string]string{ "key1": "value1", @@ -188,7 +191,8 @@ func TestMapDemo(t *testing.T) { } // SerializeWithCallback with codegen - f := fory.NewFory(fory.WithRefTracking(true)) + // WithXlang(false) enables native Go mode where nil maps are preserved as nil + f := fory.NewFory(fory.WithXlang(false), fory.WithRefTracking(true)) data, err := f.Marshal(instance) require.NoError(t, err, "Serialization failed") diff --git a/go/fory/tests/structs_fory_gen.go b/go/fory/tests/structs_fory_gen.go index e2ede9369e..a636de57bc 100644 --- a/go/fory/tests/structs_fory_gen.go +++ b/go/fory/tests/structs_fory_gen.go @@ -1,6 +1,6 @@ // Code generated by forygen. DO NOT EDIT. -// source: structs.go -// generated at: 2026-01-01T05:48:22+08:00 +// source: /Users/chaokunyang/Desktop/dev/fory/go/fory/tests/structs.go +// generated at: 2026-01-03T14:03:09+08:00 package fory @@ -71,11 +71,13 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, // Field: DynamicSlice ([]interface{}) // Dynamic slice []interface{} handling - manual serialization { - if v.DynamicSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - sliceLen := len(v.DynamicSlice) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + sliceLen := 0 + if v.DynamicSlice != nil { + sliceLen = len(v.DynamicSlice) + } buf.WriteVaruint32(uint32(sliceLen)) if sliceLen > 0 { // WriteData collection flags for dynamic slice []interface{} @@ -86,6 +88,24 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, ctx.WriteValue(reflect.ValueOf(elem)) } } + } else { + // Native Go mode: slices are nullable, write null flag + if v.DynamicSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + sliceLen := len(v.DynamicSlice) + buf.WriteVaruint32(uint32(sliceLen)) + if sliceLen > 0 { + // WriteData collection flags for dynamic slice []interface{} + // Only CollectionTrackingRef is set (no declared type, may have different types) + buf.WriteInt8(1) // CollectionTrackingRef only + // WriteData each element using WriteValue + for _, elem := range v.DynamicSlice { + ctx.WriteValue(reflect.ValueOf(elem)) + } + } + } } } return nil @@ -157,10 +177,9 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v // Field: DynamicSlice ([]interface{}) // Dynamic slice []interface{} handling - manual deserialization { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.DynamicSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag sliceLen := int(buf.ReadVaruint32(err)) if sliceLen == 0 { v.DynamicSlice = make([]interface{}, 0) @@ -174,6 +193,26 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem()) } } + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.DynamicSlice = nil + } else { + sliceLen := int(buf.ReadVaruint32(err)) + if sliceLen == 0 { + v.DynamicSlice = make([]interface{}, 0) + } else { + // ReadData collection flags (ignore for now) + _ = buf.ReadInt8(err) + // Create slice with proper capacity + v.DynamicSlice = make([]interface{}, sliceLen) + // ReadData each element using ReadValue + for i := range v.DynamicSlice { + ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem()) + } + } + } } } @@ -264,11 +303,13 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem // WriteData fields in sorted order // Field: IntMap (map[int]int) { - if v.IntMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.IntMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.IntMap != nil { + mapLen = len(v.IntMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -300,15 +341,56 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.IntMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.IntMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.IntMap { + buf.WriteInt64(int64(mapKey)) + buf.WriteInt64(int64(mapValue)) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.IntMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } // Field: MixedMap (map[string]int) { - if v.MixedMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.MixedMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.MixedMap != nil { + mapLen = len(v.MixedMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -343,15 +425,59 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.MixedMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.MixedMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + if isRefTracking { + kvHeader |= 0x1 // track key ref + } + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.MixedMap { + ctx.WriteString(mapKey) + buf.WriteInt64(int64(mapValue)) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.MixedMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } // Field: StringMap (map[string]string) { - if v.StringMap == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - mapLen := len(v.StringMap) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, write directly without null flag + mapLen := 0 + if v.StringMap != nil { + mapLen = len(v.StringMap) + } buf.WriteVaruint32(uint32(mapLen)) if mapLen > 0 { // Calculate KV header flags @@ -389,6 +515,51 @@ func (g *MapDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *MapDem buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) } } + } else { + // Native Go mode: maps are nullable, write null flag + if v.StringMap == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + mapLen := len(v.StringMap) + buf.WriteVaruint32(uint32(mapLen)) + if mapLen > 0 { + // Calculate KV header flags + kvHeader := uint8(0) + isRefTracking := ctx.TrackRef() + _ = isRefTracking // Mark as used to avoid warning + if isRefTracking { + kvHeader |= 0x1 // track key ref + } + if isRefTracking { + kvHeader |= 0x8 // track value ref + } + chunkSize := 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset := buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + for mapKey, mapValue := range v.StringMap { + ctx.WriteString(mapKey) + ctx.WriteString(mapValue) + chunkSize++ + if chunkSize >= 255 { + // WriteData chunk size and start new chunk + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + if len(v.StringMap) > chunkSize { + chunkSize = 0 + _ = buf.WriterIndex() // chunkHeaderOffset + buf.WriteInt8(int8(kvHeader)) // KV header + chunkSizeOffset = buf.WriterIndex() + buf.WriteInt8(0) // placeholder for chunk size + } + } + } + if chunkSize > 0 { + buf.PutUint8(chunkSizeOffset, uint8(chunkSize)) + } + } + } } } return nil @@ -459,10 +630,9 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) // ReadData fields in same order as write // Field: IntMap (map[int]int) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.IntMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.IntMap = make(map[int]int) @@ -491,14 +661,48 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.IntMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.IntMap = make(map[int]int) + } else { + v.IntMap = make(map[int]int, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey int + mapKey = int(buf.ReadInt64(err)) + var mapValue int + mapValue = int(buf.ReadInt64(err)) + v.IntMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } // Field: MixedMap (map[string]int) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.MixedMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.MixedMap = make(map[string]int) @@ -527,14 +731,48 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.MixedMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.MixedMap = make(map[string]int) + } else { + v.MixedMap = make(map[string]int, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey string + mapKey = ctx.ReadString() + var mapValue int + mapValue = int(buf.ReadInt64(err)) + v.MixedMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } // Field: StringMap (map[string]string) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.StringMap = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: maps are not nullable, read directly without null flag mapLen := int(buf.ReadVaruint32(err)) if mapLen == 0 { v.StringMap = make(map[string]string) @@ -563,6 +801,41 @@ func (g *MapDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *MapDemo) mapSize -= chunkSize } } + } else { + // Native Go mode: maps are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.StringMap = nil + } else { + mapLen := int(buf.ReadVaruint32(err)) + if mapLen == 0 { + v.StringMap = make(map[string]string) + } else { + v.StringMap = make(map[string]string, mapLen) + mapSize := mapLen + for mapSize > 0 { + // ReadData KV header + kvHeader := buf.ReadByte(err) + chunkSize := int(buf.ReadByte(err)) + trackKeyRef := (kvHeader & 0x1) != 0 + keyNotDeclared := (kvHeader & 0x4) != 0 + trackValueRef := (kvHeader & 0x8) != 0 + valueNotDeclared := (kvHeader & 0x20) != 0 + _ = trackKeyRef + _ = keyNotDeclared + _ = trackValueRef + _ = valueNotDeclared + for i := 0; i < chunkSize; i++ { + var mapKey string + mapKey = ctx.ReadString() + var mapValue string + mapValue = ctx.ReadString() + v.StringMap[mapKey] = mapValue + } + mapSize -= chunkSize + } + } + } } } @@ -652,33 +925,62 @@ func (g *SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *Slic // WriteData fields in sorted order // Field: BoolSlice ([]bool) - if v.BoolSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteBoolSlice(buf, v.BoolSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteBoolSlice(buf, v.BoolSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.BoolSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteBoolSlice(buf, v.BoolSlice) + } + } } // Field: FloatSlice ([]float64) - if v.FloatSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteFloat64Slice(buf, v.FloatSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteFloat64Slice(buf, v.FloatSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.FloatSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteFloat64Slice(buf, v.FloatSlice) + } + } } // Field: IntSlice ([]int32) - if v.IntSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - fory.WriteInt32Slice(buf, v.IntSlice) + { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + fory.WriteInt32Slice(buf, v.IntSlice) + } else { + // Native Go mode: slices are nullable, write null flag + if v.IntSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + fory.WriteInt32Slice(buf, v.IntSlice) + } + } } // Field: StringSlice ([]string) { - if v.StringSlice == nil { - buf.WriteInt8(-3) // NullFlag - } else { - buf.WriteInt8(-1) // NotNullValueFlag - sliceLen := len(v.StringSlice) + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, write directly without null flag + sliceLen := 0 + if v.StringSlice != nil { + sliceLen = len(v.StringSlice) + } buf.WriteVaruint32(uint32(sliceLen)) if sliceLen > 0 { collectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType @@ -693,6 +995,28 @@ func (g *SliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, v *Slic ctx.WriteString(elem) } } + } else { + // Native Go mode: slices are nullable, write null flag + if v.StringSlice == nil { + buf.WriteInt8(-3) // NullFlag + } else { + buf.WriteInt8(-1) // NotNullValueFlag + sliceLen := len(v.StringSlice) + buf.WriteVaruint32(uint32(sliceLen)) + if sliceLen > 0 { + collectFlag := 12 // CollectionIsSameType | CollectionIsDeclElementType + if ctx.TrackRef() { + collectFlag |= 1 // CollectionTrackingRef for referencable element type + } + buf.WriteInt8(int8(collectFlag)) + for _, elem := range v.StringSlice { + if ctx.TrackRef() { + buf.WriteInt8(-1) // NotNullValueFlag for element + } + ctx.WriteString(elem) + } + } + } } } return nil @@ -763,37 +1087,57 @@ func (g *SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *SliceD // ReadData fields in same order as write // Field: BoolSlice ([]bool) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.BoolSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.BoolSlice = fory.ReadBoolSlice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.BoolSlice = nil + } else { + v.BoolSlice = fory.ReadBoolSlice(buf, err) + } } } // Field: FloatSlice ([]float64) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.FloatSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.FloatSlice = fory.ReadFloat64Slice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.FloatSlice = nil + } else { + v.FloatSlice = fory.ReadFloat64Slice(buf, err) + } } } // Field: IntSlice ([]int32) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.IntSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag v.IntSlice = fory.ReadInt32Slice(buf, err) + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.IntSlice = nil + } else { + v.IntSlice = fory.ReadInt32Slice(buf, err) + } } } // Field: StringSlice ([]string) { - nullFlag := buf.ReadInt8(err) - if nullFlag == -3 { - v.StringSlice = nil - } else { + isXlang := ctx.TypeResolver().IsXlang() + if isXlang { + // xlang mode: slices are not nullable, read directly without null flag sliceLen := int(buf.ReadVaruint32(err)) if sliceLen == 0 { v.StringSlice = make([]string, 0) @@ -826,6 +1170,45 @@ func (g *SliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v *SliceD } } } + } else { + // Native Go mode: slices are nullable, read null flag + nullFlag := buf.ReadInt8(err) + if nullFlag == -3 { + v.StringSlice = nil + } else { + sliceLen := int(buf.ReadVaruint32(err)) + if sliceLen == 0 { + v.StringSlice = make([]string, 0) + } else { + collectFlag := buf.ReadInt8(err) + // Check if CollectionIsDeclElementType is set (bit 2, value 4) + hasDeclType := (collectFlag & 4) != 0 + // Check if CollectionTrackingRef is set (bit 0, value 1) + trackRefs := (collectFlag & 1) != 0 + v.StringSlice = make([]string, sliceLen) + if hasDeclType { + // Elements are written directly without type IDs + for i := 0; i < sliceLen; i++ { + if trackRefs { + _ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag) + } + v.StringSlice[i] = ctx.ReadString() + } + } else { + // Need to read type ID once if CollectionIsSameType is set + if (collectFlag & 8) != 0 { + // ReadData element type ID once for all elements + _ = buf.ReadVaruint32(err) + } + for i := 0; i < sliceLen; i++ { + if trackRefs { + _ = buf.ReadInt8(err) // Read ref flag (NotNullValueFlag) + } + v.StringSlice[i] = ctx.ReadString() + } + } + } + } } } diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 98839af6a2..8fe7711d5c 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -441,17 +441,14 @@ func buildFieldDefs(fory *Fory, value reflect.Value) ([]FieldDef, error) { internalId := TypeId(typeId & 0xFF) isEnumField := internalId == ENUM || internalId == NAMED_ENUM // Determine nullable based on mode - // In both xlang and native mode, nil-able types (ptr, slice, map, interface) are nullable. - // This is necessary for correct null flag handling when serializing/deserializing. - // Note: In xlang mode, we don't set nullable for interface{} fields because the - // actual type determines nullability; for slices and maps, they can be nil. + // In xlang mode: only pointer types are nullable by default (per xlang spec) + // In native mode: Go's natural semantics - all nil-able types are nullable + // This ensures proper interoperability with Java/other languages in xlang mode. var nullableFlag bool if fory.config.IsXlang { - // xlang mode: pointers, slices, and maps are nullable - // (they can be nil and need null flag handling) - nullableFlag = field.Type.Kind() == reflect.Ptr || - field.Type.Kind() == reflect.Slice || - field.Type.Kind() == reflect.Map + // xlang mode: only pointer types are nullable by default per xlang spec + // Slices and maps are NOT nullable - they serialize as empty when nil + nullableFlag = field.Type.Kind() == reflect.Ptr } else { // Native mode: Go's natural semantics - all nil-able types are nullable nullableFlag = field.Type.Kind() == reflect.Ptr || diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 6b11b69b78..721548c4bb 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -299,6 +299,11 @@ func (r *TypeResolver) Compatible() bool { return r.fory.config.Compatible } +// IsXlang returns whether xlang (cross-language) mode is enabled +func (r *TypeResolver) IsXlang() bool { + return r.isXlang +} + func (r *TypeResolver) initialize() { serializers := []struct { reflect.Type diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index ce894ab38d..14af31306b 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -94,6 +94,7 @@ import org.apache.fory.codegen.Expression.ForEach; import org.apache.fory.codegen.Expression.ForLoop; import org.apache.fory.codegen.Expression.If; +import org.apache.fory.codegen.Expression.InstanceOf; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.ListExpression; import org.apache.fory.codegen.Expression.Literal; @@ -1781,9 +1782,31 @@ protected Expression deserializeForNotNull( } Expression obj; if (useCollectionSerialization(typeRef)) { - obj = deserializeForCollection(buffer, typeRef, serializer, invokeHint); + // When serializer is passed and its compile-time type is generic Serializer (not + // CollectionLikeSerializer), the runtime serializer could be different (e.g., + // JdkProxySerializer for proxy objects). We need a runtime instanceof check. + if (serializer != null && !COLLECTION_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { + Expression isCollectionSerializer = + new InstanceOf(serializer, COLLECTION_SERIALIZER_TYPE); + Expression collectionSerializer = cast(serializer, COLLECTION_SERIALIZER_TYPE); + Expression collectionPath = + deserializeForCollection(buffer, typeRef, collectionSerializer, invokeHint); + Expression genericPath = read(serializer, buffer, OBJECT_TYPE); + obj = new If(isCollectionSerializer, collectionPath, genericPath); + } else { + obj = deserializeForCollection(buffer, typeRef, serializer, invokeHint); + } } else if (useMapSerialization(typeRef)) { - obj = deserializeForMap(buffer, typeRef, serializer, invokeHint); + // Same logic as collection: check at runtime if serializer is actually a MapLikeSerializer + if (serializer != null && !MAP_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { + Expression isMapSerializer = new InstanceOf(serializer, MAP_SERIALIZER_TYPE); + Expression mapSerializer = cast(serializer, MAP_SERIALIZER_TYPE); + Expression mapPath = deserializeForMap(buffer, typeRef, mapSerializer, invokeHint); + Expression genericPath = read(serializer, buffer, OBJECT_TYPE); + obj = new If(isMapSerializer, mapPath, genericPath); + } else { + obj = deserializeForMap(buffer, typeRef, serializer, invokeHint); + } } else { if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); @@ -1862,9 +1885,31 @@ private Expression deserializeForNotNullForField( } Expression obj; if (useCollectionSerialization(typeRef)) { - obj = deserializeForCollection(buffer, typeRef, serializer, null); + // When serializer is passed and its compile-time type is generic Serializer (not + // CollectionLikeSerializer), the runtime serializer could be different (e.g., + // JdkProxySerializer for proxy objects). We need a runtime instanceof check. + if (serializer != null && !COLLECTION_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { + Expression isCollectionSerializer = + new InstanceOf(serializer, COLLECTION_SERIALIZER_TYPE); + Expression collectionSerializer = cast(serializer, COLLECTION_SERIALIZER_TYPE); + Expression collectionPath = + deserializeForCollection(buffer, typeRef, collectionSerializer, null); + Expression genericPath = read(serializer, buffer, OBJECT_TYPE); + obj = new If(isCollectionSerializer, collectionPath, genericPath); + } else { + obj = deserializeForCollection(buffer, typeRef, serializer, null); + } } else if (useMapSerialization(typeRef)) { - obj = deserializeForMap(buffer, typeRef, serializer, null); + // Same logic as collection: check at runtime if serializer is actually a MapLikeSerializer + if (serializer != null && !MAP_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { + Expression isMapSerializer = new InstanceOf(serializer, MAP_SERIALIZER_TYPE); + Expression mapSerializer = cast(serializer, MAP_SERIALIZER_TYPE); + Expression mapPath = deserializeForMap(buffer, typeRef, mapSerializer, null); + Expression genericPath = read(serializer, buffer, OBJECT_TYPE); + obj = new If(isMapSerializer, mapPath, genericPath); + } else { + obj = deserializeForMap(buffer, typeRef, serializer, null); + } } else { if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); @@ -2006,17 +2051,24 @@ protected Expression readCollectionCodegen( Expression isDeclType = eq(new BitAnd(flags, isDeclTypeFlag), isDeclTypeFlag); Invoke serializer = inlineInvoke(readClassInfo(elemClass, buffer), "getSerializer", SERIALIZER_TYPE); - TypeRef serializerType = getSerializerType(elementType); + // For non-final collection/map element types, the runtime serializer could be different + // (e.g., JdkProxySerializer for proxy objects). Use generic SERIALIZER_TYPE to allow + // runtime instanceof check in deserializeForNotNull. + boolean elemIsCollectionOrMap = + useCollectionSerialization(elementType) || useMapSerialization(elementType); + TypeRef serializerType = + elemIsCollectionOrMap ? SERIALIZER_TYPE : getSerializerType(elementType); Expression elemSerializer; // make it in scope of `if(sameElementClass)` boolean maybeDecl = typeResolver(r -> r.isSerializable(elemClass)); if (maybeDecl) { + TypeRef declSerializerType = getSerializerType(elementType); elemSerializer = new If( isDeclType, - cast(getOrCreateSerializer(elemClass), serializerType), + cast(getOrCreateSerializer(elemClass), declSerializerType), cast(serializer.inline(), serializerType), false, - serializerType); + SERIALIZER_TYPE); } else { elemSerializer = cast(serializer.inline(), serializerType); } diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 073c2ffdd7..0f5f10dcc6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2768,4 +2768,41 @@ public String toString() { return String.format("%s = %s", from, to); } } + + /** Expression for Java instanceof check. */ + class InstanceOf extends ValueExpression { + private Expression target; + private final TypeRef checkType; + + public InstanceOf(Expression target, TypeRef checkType) { + super(new Expression[] {target}); + this.target = target; + this.checkType = checkType; + this.inlineCall = true; + } + + @Override + public TypeRef type() { + return PRIMITIVE_BOOLEAN_TYPE; + } + + @Override + public ExprCode doGenCode(CodegenContext ctx) { + ExprCode targetCode = target.genCode(ctx); + String value = + String.format("(%s instanceof %s)", targetCode.value(), ctx.type(checkType)); + String code = StringUtils.isBlank(targetCode.code()) ? null : targetCode.code(); + return new ExprCode(code, FalseLiteral, Code.variable(boolean.class, value)); + } + + @Override + public boolean nullable() { + return false; + } + + @Override + public String toString() { + return String.format("%s instanceof %s", target, checkType); + } + } } From b02108b2d5d239d3c820359b761f81d391576115 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 14:53:27 +0800 Subject: [PATCH 17/35] style(java): apply spotless formatting to Expression.java --- .../org/apache/fory/codegen/Expression.java | 6 +- rust/tests/expanded.rs | 1584 ----------------- 2 files changed, 2 insertions(+), 1588 deletions(-) delete mode 100644 rust/tests/expanded.rs diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index 0f5f10dcc6..d84490b3b2 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -1871,8 +1871,7 @@ public ExprCode doGenCode(CodegenContext ctx) { codeBuilder.toString(), Code.isNullVariable(isNull), Code.variable(rawType, value)); } else { // When not nullable, return FalseLiteral instead of a variable that was never declared - return new ExprCode( - codeBuilder.toString(), FalseLiteral, Code.variable(rawType, value)); + return new ExprCode(codeBuilder.toString(), FalseLiteral, Code.variable(rawType, value)); } } else { String ifCode; @@ -2789,8 +2788,7 @@ public TypeRef type() { @Override public ExprCode doGenCode(CodegenContext ctx) { ExprCode targetCode = target.genCode(ctx); - String value = - String.format("(%s instanceof %s)", targetCode.value(), ctx.type(checkType)); + String value = String.format("(%s instanceof %s)", targetCode.value(), ctx.type(checkType)); String code = StringUtils.isBlank(targetCode.code()) ? null : targetCode.code(); return new ExprCode(code, FalseLiteral, Code.variable(boolean.class, value)); } diff --git a/rust/tests/expanded.rs b/rust/tests/expanded.rs deleted file mode 100644 index db2be92537..0000000000 --- a/rust/tests/expanded.rs +++ /dev/null @@ -1,1584 +0,0 @@ -#![feature(prelude_import)] -#[macro_use] -extern crate std; -#[prelude_import] -use std::prelude::rust_2021::*; -use fory_core::fory::Fory; -use fory_derive::ForyObject; -use std::collections::HashMap; -extern crate test; -#[rustc_test_marker = "test_simple"] -#[doc(hidden)] -pub const test_simple: test::TestDescAndFn = test::TestDescAndFn { - desc: test::TestDesc { - name: test::StaticTestName("test_simple"), - ignore: false, - ignore_message: ::core::option::Option::None, - source_file: "tests/tests/test_one_struct.rs", - start_line: 23usize, - start_col: 4usize, - end_line: 23usize, - end_col: 15usize, - compile_fail: false, - no_run: false, - should_panic: test::ShouldPanic::No, - test_type: test::TestType::IntegrationTest, - }, - testfn: test::StaticTestFn( - #[coverage(off)] - || test::assert_test_result(test_simple()), - ), -}; -fn test_simple() { - #[fory_debug] - struct Animal1 { - f1: HashMap>, - f2: String, - f3: Vec, - f5: String, - f6: Vec, - f7: i8, - last: i8, - } - use fory_core::ForyDefault as _; - impl fory_core::ForyDefault for Animal1 { - fn fory_default() -> Self { - Self { - f7: ::fory_default(), - last: ::fory_default(), - f2: ::fory_default(), - f5: ::fory_default(), - f3: as fory_core::ForyDefault>::fory_default(), - f6: as fory_core::ForyDefault>::fory_default(), - f1: > as fory_core::ForyDefault>::fory_default(), - } - } - } - impl std::default::Default for Animal1 { - fn default() -> Self { - Self::fory_default() - } - } - impl fory_core::StructSerializer for Animal1 { - #[inline(always)] - fn fory_type_index() -> u32 { - 0u32 - } - #[inline(always)] - fn fory_actual_type_id( - type_id: u32, - register_by_name: bool, - compatible: bool, - ) -> u32 { - fory_core::serializer::struct_::actual_type_id( - type_id, - register_by_name, - compatible, - ) - } - fn fory_get_sorted_field_names() -> &'static [&'static str] { - &["f7", "last", "f2", "f5", "f3", "f6", "f1"] - } - fn fory_fields_info( - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result, fory_core::error::Error> { - let field_infos: Vec = <[_]>::into_vec( - ::alloc::boxed::box_new([ - fory_core::meta::FieldInfo::new( - "f7", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "last", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f2", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f5", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f3", - fory_core::meta::FieldType::new( - 31u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f6", - fory_core::meta::FieldType::new( - 31u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f1", - fory_core::meta::FieldType::new( - , - > as fory_core::serializer::Serializer>::fory_get_type_id( - type_resolver, - )?, - true, - <[_]>::into_vec( - ::alloc::boxed::box_new([ - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - fory_core::meta::FieldType::new( - 31u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ]), - ) as Vec, - ), - ), - ]), - ); - Ok(field_infos) - } - #[inline] - fn fory_read_compatible( - context: &mut fory_core::resolver::context::ReadContext, - type_info: std::rc::Rc, - ) -> Result { - let fields = type_info.get_type_meta().get_field_infos().clone(); - let mut _f7: i8 = 0 as i8; - let mut _last: i8 = 0 as i8; - let mut _f2: Option = None; - let mut _f5: Option = None; - let mut _f3: Option> = None; - let mut _f6: Option> = None; - let mut _f1: Option>> = None; - let meta = context - .get_type_info(&std::any::TypeId::of::())? - .get_type_meta(); - let local_type_hash = meta.get_hash(); - let remote_type_hash = type_info.get_type_meta().get_hash(); - if remote_type_hash == local_type_hash { - ::fory_read_data(context) - } else { - for _field in fields.iter() { - match _field.field_id { - 0i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f7", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f7 = ::fory_read( - context, - true, - false, - )?; - } else { - _f7 = ::fory_read_data( - context, - )?; - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f7", - (&_f7) as &dyn std::any::Any, - context, - ); - } - 1i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "last", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _last = ::fory_read( - context, - true, - false, - )?; - } else { - _last = ::fory_read_data( - context, - )?; - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "last", - (&_last) as &dyn std::any::Any, - context, - ); - } - 2i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f2", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f2 = Some( - ::fory_read( - context, - true, - false, - )?, - ); - } else { - _f2 = Some( - ::fory_read_data(context)?, - ); - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f2", - (&_f2) as &dyn std::any::Any, - context, - ); - } - 3i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f5", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f5 = Some( - ::fory_read( - context, - true, - false, - )?, - ); - } else { - _f5 = Some( - ::fory_read_data(context)?, - ); - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f5", - (&_f5) as &dyn std::any::Any, - context, - ); - } - 4i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f3", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f3 = Some( - as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f3 = Some( - as fory_core::Serializer>::fory_read_data(context)?, - ); - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f3", - (&_f3) as &dyn std::any::Any, - context, - ); - } - 5i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f6", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f6 = Some( - as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f6 = Some( - as fory_core::Serializer>::fory_read_data(context)?, - ); - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f6", - (&_f6) as &dyn std::any::Any, - context, - ); - } - 6i16 => { - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f1", - context, - ); - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f1 = Some( - , - > as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f1 = Some( - , - > as fory_core::Serializer>::fory_read_data(context)?, - ); - } - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f1", - (&_f1) as &dyn std::any::Any, - context, - ); - } - _ => { - let field_type = &_field.field_type; - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - field_type.type_id, - field_type.nullable, - ); - let field_name = _field.field_name.as_str(); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - field_name, - context, - ); - fory_core::serializer::skip::skip_field_value( - context, - &field_type, - read_ref_flag, - )?; - let placeholder: &dyn std::any::Any = &(); - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - field_name, - placeholder, - context, - ); - } - } - } - Ok(Self { - f7: _f7, - last: _last, - f2: _f2.unwrap_or_default(), - f5: _f5.unwrap_or_default(), - f3: _f3.unwrap_or_default(), - f6: _f6.unwrap_or_default(), - f1: _f1.unwrap_or_default(), - }) - } - } - } - impl fory_core::Serializer for Animal1 { - #[inline(always)] - fn fory_get_type_id( - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result { - type_resolver.get_type_id(&std::any::TypeId::of::(), 0u32) - } - #[inline(always)] - fn fory_type_id_dyn( - &self, - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result { - Self::fory_get_type_id(type_resolver) - } - #[inline(always)] - fn as_any(&self) -> &dyn std::any::Any { - self - } - #[inline(always)] - fn FORY_STATIC_TYPE_ID -> fory_core::TypeId - where - Self: Sized, - { - fory_core::TypeId::STRUCT - } - #[inline(always)] - fn fory_reserved_space() -> usize { - ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + > as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - } - #[inline(always)] - fn fory_write( - &self, - context: &mut fory_core::resolver::context::WriteContext, - write_ref_info: bool, - write_type_info: bool, - _: bool, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::write::< - Self, - >(self, context, write_ref_info, write_type_info) - } - #[inline] - fn fory_write_data( - &self, - context: &mut fory_core::resolver::context::WriteContext, - ) -> Result<(), fory_core::error::Error> { - if context.is_check_struct_version() { - context.writer.write_i32(-362304397i32); - } - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f7", - (&self.f7) as &dyn std::any::Any, - context, - ); - ::fory_write_data(&self.f7, context)?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f7", - (&self.f7) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "last", - (&self.last) as &dyn std::any::Any, - context, - ); - ::fory_write_data(&self.last, context)?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "last", - (&self.last) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f2", - (&self.f2) as &dyn std::any::Any, - context, - ); - ::fory_write( - &self.f2, - context, - true, - false, - false, - )?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f2", - (&self.f2) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f5", - (&self.f5) as &dyn std::any::Any, - context, - ); - ::fory_write( - &self.f5, - context, - true, - false, - false, - )?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f5", - (&self.f5) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f3", - (&self.f3) as &dyn std::any::Any, - context, - ); - as fory_core::Serializer>::fory_write( - &self.f3, - context, - true, - false, - false, - )?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f3", - (&self.f3) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f6", - (&self.f6) as &dyn std::any::Any, - context, - ); - as fory_core::Serializer>::fory_write( - &self.f6, - context, - true, - false, - false, - )?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f6", - (&self.f6) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_write_field( - "Animal1", - "f1", - (&self.f1) as &dyn std::any::Any, - context, - ); - , - > as fory_core::Serializer>::fory_write( - &self.f1, - context, - true, - false, - true, - )?; - fory_core::serializer::struct_::struct_after_write_field( - "Animal1", - "f1", - (&self.f1) as &dyn std::any::Any, - context, - ); - Ok(()) - } - #[inline(always)] - fn fory_write_type_info( - context: &mut fory_core::resolver::context::WriteContext, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::write_type_info::(context) - } - #[inline(always)] - fn fory_read( - context: &mut fory_core::resolver::context::ReadContext, - read_ref_info: bool, - read_type_info: bool, - ) -> Result { - let ref_flag = if read_ref_info { - context.reader.read_i8()? - } else { - fory_core::RefFlag::NotNullValue as i8 - }; - if ref_flag == (fory_core::RefFlag::NotNullValue as i8) - || ref_flag == (fory_core::RefFlag::RefValue as i8) - { - if context.is_compatible() { - let type_info = if read_type_info { - context.read_any_typeinfo()? - } else { - let rs_type_id = std::any::TypeId::of::(); - context.get_type_info(&rs_type_id)? - }; - ::fory_read_compatible( - context, - type_info, - ) - } else { - if read_type_info { - ::fory_read_type_info(context)?; - } - ::fory_read_data(context) - } - } else if ref_flag == (fory_core::RefFlag::Null as i8) { - Ok(::fory_default()) - } else { - Err( - fory_core::error::Error::invalid_ref( - ::alloc::__export::must_use({ - ::alloc::fmt::format( - format_args!("Unknown ref flag, value:{0}", ref_flag), - ) - }), - ), - ) - } - } - #[inline(always)] - fn fory_read_with_type_info( - context: &mut fory_core::resolver::context::ReadContext, - read_ref_info: bool, - type_info: std::rc::Rc, - ) -> Result { - let ref_flag = if read_ref_info { - context.reader.read_i8()? - } else { - fory_core::RefFlag::NotNullValue as i8 - }; - if ref_flag == (fory_core::RefFlag::NotNullValue as i8) - || ref_flag == (fory_core::RefFlag::RefValue as i8) - { - if context.is_compatible() { - ::fory_read_compatible( - context, - type_info, - ) - } else { - ::fory_read_data(context) - } - } else if ref_flag == (fory_core::RefFlag::Null as i8) { - Ok(::fory_default()) - } else { - Err( - fory_core::error::Error::invalid_ref( - ::alloc::__export::must_use({ - ::alloc::fmt::format( - format_args!("Unknown ref flag, value:{0}", ref_flag), - ) - }), - ), - ) - } - } - #[inline] - fn fory_read_data( - context: &mut fory_core::resolver::context::ReadContext, - ) -> Result { - if context.is_check_struct_version() { - let read_version = context.reader.read_i32()?; - let type_name = std::any::type_name::(); - fory_core::meta::TypeMeta::check_struct_version( - read_version, - -362304397i32, - type_name, - )?; - } - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f7", - context, - ); - let _f7 = ::fory_read_data(context)?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f7", - (&_f7) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "last", - context, - ); - let _last = ::fory_read_data(context)?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "last", - (&_last) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f2", - context, - ); - let _f2 = ::fory_read( - context, - true, - false, - )?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f2", - (&_f2) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f5", - context, - ); - let _f5 = ::fory_read( - context, - true, - false, - )?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f5", - (&_f5) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f3", - context, - ); - let _f3 = as fory_core::Serializer>::fory_read(context, true, false)?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f3", - (&_f3) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f6", - context, - ); - let _f6 = as fory_core::Serializer>::fory_read(context, true, false)?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f6", - (&_f6) as &dyn std::any::Any, - context, - ); - fory_core::serializer::struct_::struct_before_read_field( - "Animal1", - "f1", - context, - ); - let _f1 = , - > as fory_core::Serializer>::fory_read(context, true, false)?; - fory_core::serializer::struct_::struct_after_read_field( - "Animal1", - "f1", - (&_f1) as &dyn std::any::Any, - context, - ); - Ok(Self { - f7: _f7, - last: _last, - f2: _f2, - f5: _f5, - f3: _f3, - f6: _f6, - f1: _f1, - }) - } - #[inline(always)] - fn fory_read_type_info( - context: &mut fory_core::resolver::context::ReadContext, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::read_type_info::(context) - } - } - #[automatically_derived] - impl ::core::fmt::Debug for Animal1 { - #[inline] - fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - let names: &'static _ = &["f1", "f2", "f3", "f5", "f6", "f7", "last"]; - let values: &[&dyn ::core::fmt::Debug] = &[ - &self.f1, - &self.f2, - &self.f3, - &self.f5, - &self.f6, - &self.f7, - &&self.last, - ]; - ::core::fmt::Formatter::debug_struct_fields_finish( - f, - "Animal1", - names, - values, - ) - } - } - struct Animal2 { - f1: HashMap>, - f3: Vec, - f4: String, - f5: i8, - f6: Vec, - f7: i16, - last: i8, - } - use fory_core::ForyDefault as _; - impl fory_core::ForyDefault for Animal2 { - fn fory_default() -> Self { - Self { - f7: ::fory_default(), - f5: ::fory_default(), - last: ::fory_default(), - f4: ::fory_default(), - f3: as fory_core::ForyDefault>::fory_default(), - f6: as fory_core::ForyDefault>::fory_default(), - f1: > as fory_core::ForyDefault>::fory_default(), - } - } - } - impl std::default::Default for Animal2 { - fn default() -> Self { - Self::fory_default() - } - } - impl fory_core::StructSerializer for Animal2 { - #[inline(always)] - fn fory_type_index() -> u32 { - 1u32 - } - #[inline(always)] - fn fory_actual_type_id( - type_id: u32, - register_by_name: bool, - compatible: bool, - ) -> u32 { - fory_core::serializer::struct_::actual_type_id( - type_id, - register_by_name, - compatible, - ) - } - fn fory_get_sorted_field_names() -> &'static [&'static str] { - &["f7", "f5", "last", "f4", "f3", "f6", "f1"] - } - fn fory_fields_info( - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result, fory_core::error::Error> { - let field_infos: Vec = <[_]>::into_vec( - ::alloc::boxed::box_new([ - fory_core::meta::FieldInfo::new( - "f7", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f5", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "last", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f4", - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f3", - fory_core::meta::FieldType::new( - 31u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f6", - fory_core::meta::FieldType::new( - 32u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ), - fory_core::meta::FieldInfo::new( - "f1", - fory_core::meta::FieldType::new( - , - > as fory_core::serializer::Serializer>::fory_get_type_id( - type_resolver, - )?, - true, - <[_]>::into_vec( - ::alloc::boxed::box_new([ - fory_core::meta::FieldType::new( - ::fory_get_type_id( - type_resolver, - )?, - false, - ::alloc::vec::Vec::new() as Vec, - ), - fory_core::meta::FieldType::new( - 31u32, - true, - ::alloc::vec::Vec::new() as Vec, - ), - ]), - ) as Vec, - ), - ), - ]), - ); - Ok(field_infos) - } - #[inline] - fn fory_read_compatible( - context: &mut fory_core::resolver::context::ReadContext, - type_info: std::rc::Rc, - ) -> Result { - let fields = type_info.get_type_meta().get_field_infos().clone(); - let mut _f7: i16 = 0 as i16; - let mut _f5: i8 = 0 as i8; - let mut _last: i8 = 0 as i8; - let mut _f4: Option = None; - let mut _f3: Option> = None; - let mut _f6: Option> = None; - let mut _f1: Option>> = None; - let meta = context - .get_type_info(&std::any::TypeId::of::())? - .get_type_meta(); - let local_type_hash = meta.get_hash(); - let remote_type_hash = type_info.get_type_meta().get_hash(); - if remote_type_hash == local_type_hash { - ::fory_read_data(context) - } else { - for _field in fields.iter() { - match _field.field_id { - 0i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f7 = ::fory_read( - context, - true, - false, - )?; - } else { - _f7 = ::fory_read_data( - context, - )?; - } - } - 1i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f5 = ::fory_read( - context, - true, - false, - )?; - } else { - _f5 = ::fory_read_data( - context, - )?; - } - } - 2i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _last = ::fory_read( - context, - true, - false, - )?; - } else { - _last = ::fory_read_data( - context, - )?; - } - } - 3i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f4 = Some( - ::fory_read( - context, - true, - false, - )?, - ); - } else { - _f4 = Some( - ::fory_read_data(context)?, - ); - } - } - 4i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f3 = Some( - as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f3 = Some( - as fory_core::Serializer>::fory_read_data(context)?, - ); - } - } - 5i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f6 = Some( - as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f6 = Some( - as fory_core::Serializer>::fory_read_data(context)?, - ); - } - } - 6i16 => { - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - _field.field_type.type_id, - _field.field_type.nullable, - ); - if read_ref_flag { - _f1 = Some( - , - > as fory_core::Serializer>::fory_read( - context, - true, - false, - )?, - ); - } else { - _f1 = Some( - , - > as fory_core::Serializer>::fory_read_data(context)?, - ); - } - } - _ => { - let field_type = &_field.field_type; - let read_ref_flag = fory_core::serializer::util::field_requires_ref_flag( - field_type.type_id, - field_type.nullable, - ); - fory_core::serializer::skip::skip_field_value( - context, - field_type, - read_ref_flag, - )?; - } - } - } - Ok(Self { - f7: _f7, - f5: _f5, - last: _last, - f4: _f4.unwrap_or_default(), - f3: _f3.unwrap_or_default(), - f6: _f6.unwrap_or_default(), - f1: _f1.unwrap_or_default(), - }) - } - } - } - impl fory_core::Serializer for Animal2 { - #[inline(always)] - fn fory_get_type_id( - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result { - type_resolver.get_type_id(&std::any::TypeId::of::(), 1u32) - } - #[inline(always)] - fn fory_type_id_dyn( - &self, - type_resolver: &fory_core::resolver::type_resolver::TypeResolver, - ) -> Result { - Self::fory_get_type_id(type_resolver) - } - #[inline(always)] - fn as_any(&self) -> &dyn std::any::Any { - self - } - #[inline(always)] - fn FORY_STATIC_TYPE_ID -> fory_core::TypeId - where - Self: Sized, - { - fory_core::TypeId::STRUCT - } - #[inline(always)] - fn fory_reserved_space() -> usize { - ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + ::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - + > as fory_core::Serializer>::fory_reserved_space() - + fory_core::types::SIZE_OF_REF_AND_TYPE - } - #[inline(always)] - fn fory_write( - &self, - context: &mut fory_core::resolver::context::WriteContext, - write_ref_info: bool, - write_type_info: bool, - _: bool, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::write::< - Self, - >(self, context, write_ref_info, write_type_info) - } - #[inline] - fn fory_write_data( - &self, - context: &mut fory_core::resolver::context::WriteContext, - ) -> Result<(), fory_core::error::Error> { - if context.is_check_struct_version() { - context.writer.write_i32(-1615591914i32); - } - ::fory_write_data(&self.f7, context)?; - ::fory_write_data(&self.f5, context)?; - ::fory_write_data(&self.last, context)?; - ::fory_write( - &self.f4, - context, - true, - false, - false, - )?; - as fory_core::Serializer>::fory_write( - &self.f3, - context, - true, - false, - false, - )?; - as fory_core::Serializer>::fory_write( - &self.f6, - context, - true, - false, - false, - )?; - , - > as fory_core::Serializer>::fory_write( - &self.f1, - context, - true, - false, - true, - )?; - Ok(()) - } - #[inline(always)] - fn fory_write_type_info( - context: &mut fory_core::resolver::context::WriteContext, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::write_type_info::(context) - } - #[inline(always)] - fn fory_read( - context: &mut fory_core::resolver::context::ReadContext, - read_ref_info: bool, - read_type_info: bool, - ) -> Result { - let ref_flag = if read_ref_info { - context.reader.read_i8()? - } else { - fory_core::RefFlag::NotNullValue as i8 - }; - if ref_flag == (fory_core::RefFlag::NotNullValue as i8) - || ref_flag == (fory_core::RefFlag::RefValue as i8) - { - if context.is_compatible() { - let type_info = if read_type_info { - context.read_any_typeinfo()? - } else { - let rs_type_id = std::any::TypeId::of::(); - context.get_type_info(&rs_type_id)? - }; - ::fory_read_compatible( - context, - type_info, - ) - } else { - if read_type_info { - ::fory_read_type_info(context)?; - } - ::fory_read_data(context) - } - } else if ref_flag == (fory_core::RefFlag::Null as i8) { - Ok(::fory_default()) - } else { - Err( - fory_core::error::Error::invalid_ref( - ::alloc::__export::must_use({ - ::alloc::fmt::format( - format_args!("Unknown ref flag, value:{0}", ref_flag), - ) - }), - ), - ) - } - } - #[inline(always)] - fn fory_read_with_type_info( - context: &mut fory_core::resolver::context::ReadContext, - read_ref_info: bool, - type_info: std::rc::Rc, - ) -> Result { - let ref_flag = if read_ref_info { - context.reader.read_i8()? - } else { - fory_core::RefFlag::NotNullValue as i8 - }; - if ref_flag == (fory_core::RefFlag::NotNullValue as i8) - || ref_flag == (fory_core::RefFlag::RefValue as i8) - { - if context.is_compatible() { - ::fory_read_compatible( - context, - type_info, - ) - } else { - ::fory_read_data(context) - } - } else if ref_flag == (fory_core::RefFlag::Null as i8) { - Ok(::fory_default()) - } else { - Err( - fory_core::error::Error::invalid_ref( - ::alloc::__export::must_use({ - ::alloc::fmt::format( - format_args!("Unknown ref flag, value:{0}", ref_flag), - ) - }), - ), - ) - } - } - #[inline] - fn fory_read_data( - context: &mut fory_core::resolver::context::ReadContext, - ) -> Result { - if context.is_check_struct_version() { - let read_version = context.reader.read_i32()?; - let type_name = std::any::type_name::(); - fory_core::meta::TypeMeta::check_struct_version( - read_version, - -1615591914i32, - type_name, - )?; - } - let _f7 = ::fory_read_data(context)?; - let _f5 = ::fory_read_data(context)?; - let _last = ::fory_read_data(context)?; - let _f4 = ::fory_read( - context, - true, - false, - )?; - let _f3 = as fory_core::Serializer>::fory_read(context, true, false)?; - let _f6 = as fory_core::Serializer>::fory_read(context, true, false)?; - let _f1 = , - > as fory_core::Serializer>::fory_read(context, true, false)?; - Ok(Self { - f7: _f7, - f5: _f5, - last: _last, - f4: _f4, - f3: _f3, - f6: _f6, - f1: _f1, - }) - } - #[inline(always)] - fn fory_read_type_info( - context: &mut fory_core::resolver::context::ReadContext, - ) -> Result<(), fory_core::error::Error> { - fory_core::serializer::struct_::read_type_info::(context) - } - } - #[automatically_derived] - impl ::core::fmt::Debug for Animal2 { - #[inline] - fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { - let names: &'static _ = &["f1", "f3", "f4", "f5", "f6", "f7", "last"]; - let values: &[&dyn ::core::fmt::Debug] = &[ - &self.f1, - &self.f3, - &self.f4, - &self.f5, - &self.f6, - &self.f7, - &&self.last, - ]; - ::core::fmt::Formatter::debug_struct_fields_finish( - f, - "Animal2", - names, - values, - ) - } - } - let mut fory1 = Fory::default().compatible(true); - let mut fory2 = Fory::default().compatible(true); - fory1.register::(999).unwrap(); - fory2.register::(999).unwrap(); - let animal: Animal1 = Animal1 { - f1: HashMap::from([(1, <[_]>::into_vec(::alloc::boxed::box_new([2])))]), - f2: String::from("hello"), - f3: <[_]>::into_vec(::alloc::boxed::box_new([1, 2, 3])), - f5: String::from("f5"), - f6: <[_]>::into_vec(::alloc::boxed::box_new([42])), - f7: 43, - last: 44, - }; - let bin = fory1.serialize(&animal).unwrap(); - let obj: Animal2 = fory2.deserialize(&bin).unwrap(); - match (&animal.f1, &obj.f1) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&animal.f3, &obj.f3) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&obj.f4, &String::default()) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&obj.f5, &i8::default()) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&obj.f6, &Vec::::default()) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&obj.f7, &i16::default()) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; - match (&animal.last, &obj.last) { - (left_val, right_val) => { - if !(*left_val == *right_val) { - let kind = ::core::panicking::AssertKind::Eq; - ::core::panicking::assert_failed( - kind, - &*left_val, - &*right_val, - ::core::option::Option::None, - ); - } - } - }; -} -#[rustc_main] -#[coverage(off)] -#[doc(hidden)] -pub fn main() -> () { - extern crate test; - test::test_main_static(&[&test_simple]) -} From 3d2db30c806d27b2223e851bfb68d569044f3a79 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 15:47:42 +0800 Subject: [PATCH 18/35] fix java codegen handle morphic --- .../fory/builder/BaseObjectCodecBuilder.java | 197 ++++++------------ .../fory/builder/ObjectCodecBuilder.java | 6 +- .../fory/builder/ObjectCodecOptimizer.java | 8 +- 3 files changed, 69 insertions(+), 142 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index 14af31306b..ee583683ef 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -94,7 +94,6 @@ import org.apache.fory.codegen.Expression.ForEach; import org.apache.fory.codegen.Expression.ForLoop; import org.apache.fory.codegen.Expression.If; -import org.apache.fory.codegen.Expression.InstanceOf; import org.apache.fory.codegen.Expression.Invoke; import org.apache.fory.codegen.Expression.ListExpression; import org.apache.fory.codegen.Expression.Literal; @@ -620,14 +619,14 @@ protected boolean isMonomorphic(Class clz) { return typeResolver(r -> r.isMonomorphic(clz)); } - protected boolean isMonomorphic(Descriptor descriptor) { - return typeResolver(r -> r.isMonomorphic(descriptor)); - } - protected boolean isMonomorphic(TypeRef typeRef) { return isMonomorphic(typeRef.getRawType()); } + protected boolean isMonomorphic(Descriptor descriptor) { + return typeResolver(r -> r.isMonomorphic(descriptor)); + } + protected Expression serializeForNotNullObject( Expression inputObject, Expression buffer, TypeRef typeRef, Expression serializer) { Class clz = getRawType(typeRef); @@ -1782,31 +1781,9 @@ protected Expression deserializeForNotNull( } Expression obj; if (useCollectionSerialization(typeRef)) { - // When serializer is passed and its compile-time type is generic Serializer (not - // CollectionLikeSerializer), the runtime serializer could be different (e.g., - // JdkProxySerializer for proxy objects). We need a runtime instanceof check. - if (serializer != null && !COLLECTION_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { - Expression isCollectionSerializer = - new InstanceOf(serializer, COLLECTION_SERIALIZER_TYPE); - Expression collectionSerializer = cast(serializer, COLLECTION_SERIALIZER_TYPE); - Expression collectionPath = - deserializeForCollection(buffer, typeRef, collectionSerializer, invokeHint); - Expression genericPath = read(serializer, buffer, OBJECT_TYPE); - obj = new If(isCollectionSerializer, collectionPath, genericPath); - } else { - obj = deserializeForCollection(buffer, typeRef, serializer, invokeHint); - } + obj = deserializeForCollection(buffer, typeRef, serializer, invokeHint); } else if (useMapSerialization(typeRef)) { - // Same logic as collection: check at runtime if serializer is actually a MapLikeSerializer - if (serializer != null && !MAP_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { - Expression isMapSerializer = new InstanceOf(serializer, MAP_SERIALIZER_TYPE); - Expression mapSerializer = cast(serializer, MAP_SERIALIZER_TYPE); - Expression mapPath = deserializeForMap(buffer, typeRef, mapSerializer, invokeHint); - Expression genericPath = read(serializer, buffer, OBJECT_TYPE); - obj = new If(isMapSerializer, mapPath, genericPath); - } else { - obj = deserializeForMap(buffer, typeRef, serializer, invokeHint); - } + obj = deserializeForMap(buffer, typeRef, serializer, invokeHint); } else { if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); @@ -1834,8 +1811,7 @@ protected Expression deserializeField( boolean typeNeedsRef = needWriteRef(typeRef); if (useRefTracking) { - return readRef( - buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); + return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (!nullable) { Expression value = deserializeForNotNullForField(buffer, descriptor, null); @@ -1885,31 +1861,9 @@ private Expression deserializeForNotNullForField( } Expression obj; if (useCollectionSerialization(typeRef)) { - // When serializer is passed and its compile-time type is generic Serializer (not - // CollectionLikeSerializer), the runtime serializer could be different (e.g., - // JdkProxySerializer for proxy objects). We need a runtime instanceof check. - if (serializer != null && !COLLECTION_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { - Expression isCollectionSerializer = - new InstanceOf(serializer, COLLECTION_SERIALIZER_TYPE); - Expression collectionSerializer = cast(serializer, COLLECTION_SERIALIZER_TYPE); - Expression collectionPath = - deserializeForCollection(buffer, typeRef, collectionSerializer, null); - Expression genericPath = read(serializer, buffer, OBJECT_TYPE); - obj = new If(isCollectionSerializer, collectionPath, genericPath); - } else { - obj = deserializeForCollection(buffer, typeRef, serializer, null); - } + obj = deserializeForCollection(buffer, typeRef, serializer, null); } else if (useMapSerialization(typeRef)) { - // Same logic as collection: check at runtime if serializer is actually a MapLikeSerializer - if (serializer != null && !MAP_SERIALIZER_TYPE.isSupertypeOf(serializer.type())) { - Expression isMapSerializer = new InstanceOf(serializer, MAP_SERIALIZER_TYPE); - Expression mapSerializer = cast(serializer, MAP_SERIALIZER_TYPE); - Expression mapPath = deserializeForMap(buffer, typeRef, mapSerializer, null); - Expression genericPath = read(serializer, buffer, OBJECT_TYPE); - obj = new If(isMapSerializer, mapPath, genericPath); - } else { - obj = deserializeForMap(buffer, typeRef, serializer, null); - } + obj = deserializeForMap(buffer, typeRef, serializer, null); } else { if (serializer != null) { return read(serializer, buffer, OBJECT_TYPE); @@ -2021,27 +1975,22 @@ protected Expression deserializeForCollection( protected Expression readCollectionCodegen( Expression buffer, Expression collection, Expression size, TypeRef elementType) { ListExpression builder = new ListExpression(); - Invoke flags = new Invoke(buffer, "readUnsignedByte", "flags", PRIMITIVE_INT_TYPE, false); + Invoke flags = new Invoke(buffer, "readByte", "flags", PRIMITIVE_INT_TYPE, false); builder.add(flags); Class elemClass = TypeUtils.getRawType(elementType); walkPath.add(elementType.toString()); boolean finalType = isMonomorphic(elemClass); - // Read TRACKING_REF flag from bitmap at runtime for xlang compatibility. - // This ensures Java correctly reads data serialized by other languages (C++, Go, Rust, etc.) - // that may set TRACKING_REF based on element type (e.g., shared_ptr) rather than global config. - Literal trackingRefFlag = ofInt(CollectionFlags.TRACKING_REF); - Expression trackingRef = eq(new BitAnd(flags, trackingRefFlag), trackingRefFlag, "trackingRef"); - Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); - // Don't use flags.inline() - it mutates the flags object to prevent variable generation - Expression hasNull = eq(new BitAnd(flags, hasNullFlag), hasNullFlag, "hasNull"); + boolean trackingRef = typeResolver(resolver -> resolver.needToWriteRef(elementType)); if (finalType) { - builder.add(hasNull); - // Use runtime trackingRef flag to determine if ref flags are present - Expression trackingRefRead = - readContainerElements(elementType, true, null, null, buffer, collection, size); - Expression noTrackingRefRead = - readContainerElements(elementType, false, null, hasNull, buffer, collection, size); - builder.add(new If(trackingRef, trackingRefRead, noTrackingRefRead)); + if (trackingRef) { + builder.add(readContainerElements(elementType, true, null, null, buffer, collection, size)); + } else { + Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); + Expression hasNull = eq(new BitAnd(flags.inline(), hasNullFlag), hasNullFlag, "hasNull"); + builder.add( + hasNull, + readContainerElements(elementType, false, null, hasNull, buffer, collection, size)); + } } else { Literal isSameTypeFlag = ofInt(CollectionFlags.IS_SAME_TYPE); Expression sameElementClass = @@ -2051,82 +2000,60 @@ protected Expression readCollectionCodegen( Expression isDeclType = eq(new BitAnd(flags, isDeclTypeFlag), isDeclTypeFlag); Invoke serializer = inlineInvoke(readClassInfo(elemClass, buffer), "getSerializer", SERIALIZER_TYPE); - // For non-final collection/map element types, the runtime serializer could be different - // (e.g., JdkProxySerializer for proxy objects). Use generic SERIALIZER_TYPE to allow - // runtime instanceof check in deserializeForNotNull. - boolean elemIsCollectionOrMap = - useCollectionSerialization(elementType) || useMapSerialization(elementType); - TypeRef serializerType = - elemIsCollectionOrMap ? SERIALIZER_TYPE : getSerializerType(elementType); + TypeRef serializerType = getSerializerType(elementType); Expression elemSerializer; // make it in scope of `if(sameElementClass)` boolean maybeDecl = typeResolver(r -> r.isSerializable(elemClass)); if (maybeDecl) { - TypeRef declSerializerType = getSerializerType(elementType); elemSerializer = new If( isDeclType, - cast(getOrCreateSerializer(elemClass), declSerializerType), + cast(getOrCreateSerializer(elemClass), serializerType), cast(serializer.inline(), serializerType), false, - SERIALIZER_TYPE); + serializerType); } else { elemSerializer = cast(serializer.inline(), serializerType); } elemSerializer = uninline(elemSerializer); - // For xlang compatibility, we must read the TRACKING_REF flag from the serialized data - // because different languages may set this flag differently (e.g., C++ sets it for - // shared_ptr). - // The elemSerializer (which contains readClassInfo) must be conditionally evaluated: - // - It should only be evaluated when sameElementClass == true - // - It should be in scope for both tracking ref branches - // Wrap elemSerializer in a conditional that only evaluates when sameElementClass is true - Expression elemSerializerInit = - new If(sameElementClass, elemSerializer, nullValue(serializerType)); - builder.add(sameElementClass, hasNull, elemSerializerInit); - // Build both tracking and non-tracking paths, then branch at runtime based on flags byte - // Tracking ref path - same element class - // Pass elemSerializerInit (not elemSerializer) so that we reference the variable - // defined by the If expression, not regenerate the readClassInfo call - ListExpression trackingRefSameBuilder = new ListExpression(); - trackingRefSameBuilder.add( - readContainerElements( - elementType, true, elemSerializerInit, null, buffer, collection, size)); - // Tracking ref path - different element class - Set cutPoint1 = ofHashSet(buffer, collection, size); - Expression trackingRefDifferent = - invokeGenerated( - ctx, - cutPoint1, - readContainerElements(elementType, true, null, null, buffer, collection, size), - "differentTypeElemsReadTrackingRef", - false); - Expression trackingRefAction = - new If(sameElementClass, trackingRefSameBuilder, trackingRefDifferent); - - // No tracking ref path - same element class - // Pass elemSerializerInit to reference the same variable as the tracking ref path - ListExpression noTrackingRefSameBuilder = new ListExpression(); - noTrackingRefSameBuilder.add( - readContainerElements( - elementType, false, elemSerializerInit, hasNull, buffer, collection, size)); - // No tracking ref path - different element class - Set cutPoint2 = ofHashSet(buffer, collection, size, hasNull); - Expression noTrackingRefDifferent = - invokeGenerated( - ctx, - cutPoint2, - readContainerElements(elementType, false, null, hasNull, buffer, collection, size), - "differentTypeElemsRead", - false); - Expression noTrackingRefAction = - new If(sameElementClass, noTrackingRefSameBuilder, noTrackingRefDifferent); - - // Use neq to check: if (flags & TRACKING_REF) != 0 - // This is the standard way to check if a bit flag is set - Expression trackingRefCheck = - neq(new BitAnd(flags, trackingRefFlag), ofInt(0), "isTrackingRef"); - builder.add(trackingRefCheck); - builder.add(new If(trackingRefCheck, trackingRefAction, noTrackingRefAction)); + builder.add(sameElementClass); + Expression action; + if (trackingRef) { + // Same element class read start + ListExpression readBuilder = new ListExpression(elemSerializer); + readBuilder.add( + readContainerElements( + elementType, true, elemSerializer, null, buffer, collection, size)); + // Same element class read end + Set cutPoint = ofHashSet(buffer, collection, size); + Expression differentElemTypeRead = + invokeGenerated( + ctx, + cutPoint, + readContainerElements(elementType, true, null, null, buffer, collection, size), + "differentTypeElemsRead", + false); + action = new If(sameElementClass, readBuilder, differentElemTypeRead); + } else { + Literal hasNullFlag = ofInt(CollectionFlags.HAS_NULL); + Expression hasNull = eq(new BitAnd(flags, hasNullFlag), hasNullFlag, "hasNull"); + builder.add(hasNull); + // Same element class read start + ListExpression readBuilder = new ListExpression(elemSerializer); + readBuilder.add( + readContainerElements( + elementType, false, elemSerializer, hasNull, buffer, collection, size)); + // Same element class read end + Set cutPoint = ofHashSet(buffer, collection, size, hasNull); + Expression differentTypeElemsRead = + invokeGenerated( + ctx, + cutPoint, + readContainerElements(elementType, false, null, hasNull, buffer, collection, size), + "differentTypeElemsRead", + false); + action = new If(sameElementClass, readBuilder, differentTypeElemsRead); + } + builder.add(action); } walkPath.removeLast(); // place newCollection as last as expr value diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index 13b317f2fa..1e528094b4 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -173,7 +173,7 @@ public Expression buildEncodeExpression() { addGroupExpressions( objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer); addGroupExpressions( - objectCodecOptimizer.finalWriteGroups, numGroups, expressions, bean, buffer); + objectCodecOptimizer.buildInWriteGroups, numGroups, expressions, bean, buffer); for (Descriptor descriptor : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(serializeGroup(Collections.singletonList(descriptor), bean, buffer, false)); @@ -203,7 +203,7 @@ private void addGroupExpressions( private int getNumGroups(ObjectCodecOptimizer objectCodecOptimizer) { return objectCodecOptimizer.boxedWriteGroups.size() - + objectCodecOptimizer.finalWriteGroups.size() + + objectCodecOptimizer.buildInWriteGroups.size() + objectCodecOptimizer.otherWriteGroups.size() + objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors().size() + objectCodecOptimizer.descriptorGrouper.getMapDescriptors().size(); @@ -466,7 +466,7 @@ public Expression buildDecodeExpression() { deserializeReadGroup( objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer); deserializeReadGroup( - objectCodecOptimizer.finalReadGroups, numGroups, expressions, bean, buffer); + objectCodecOptimizer.buildInReadGroups, numGroups, expressions, bean, buffer); for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) { expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java index 01264fffa9..95c5efe7a8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecOptimizer.java @@ -70,8 +70,8 @@ public class ObjectCodecOptimizer extends ExpressionOptimizer { final List> primitiveGroups = new ArrayList<>(); final List> boxedWriteGroups = new ArrayList<>(); final List> boxedReadGroups = new ArrayList<>(); - final List> finalWriteGroups = new ArrayList<>(); - final List> finalReadGroups = new ArrayList<>(); + final List> buildInWriteGroups = new ArrayList<>(); + final List> buildInReadGroups = new ArrayList<>(); final List> otherWriteGroups = new ArrayList<>(); final List> otherReadGroups = new ArrayList<>(); @@ -117,9 +117,9 @@ private void buildGroups() { boxedReadWeight, boxedReadGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 9, finalWriteGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 9, buildInWriteGroups), MutableTuple3.of( - new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 5, finalReadGroups), + new ArrayList<>(descriptorGrouper.getBuildInDescriptors()), 5, buildInReadGroups), MutableTuple3.of( new ArrayList<>(descriptorGrouper.getOtherDescriptors()), 4, otherReadGroups), MutableTuple3.of( From 00f49dba2b27535bddb764277efd415a8eb4ba74 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 15:49:07 +0800 Subject: [PATCH 19/35] format java code --- .../java/org/apache/fory/builder/BaseObjectCodecBuilder.java | 3 ++- .../src/test/java/org/apache/fory/xlang/RustXlangTest.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java index ee583683ef..7b7a04dd4e 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/BaseObjectCodecBuilder.java @@ -1811,7 +1811,8 @@ protected Expression deserializeField( boolean typeNeedsRef = needWriteRef(typeRef); if (useRefTracking) { - return readRef(buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); + return readRef( + buffer, callback, () -> deserializeForNotNullForField(buffer, descriptor, null)); } else { if (!nullable) { Expression value = deserializeForNotNullForField(buffer, descriptor, null); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index f322d56f33..583bb3b588 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -53,7 +53,7 @@ public class RustXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_RUST_JAVA_CI"); if (!"1".equals(enabled)) { - // throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); + throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); } boolean rustInstalled = true; try { From 48af6f1c4e10a5344aa1738bee21a788cb41b7f3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 16:08:31 +0800 Subject: [PATCH 20/35] fix rust tests --- rust/fory-core/src/config.rs | 8 ++++---- rust/fory-core/src/fory.rs | 16 ++++++++-------- rust/fory-core/src/resolver/context.rs | 8 ++++++++ rust/fory-core/src/serializer/weak.rs | 20 ++++++++++++-------- rust/fory-core/src/types.rs | 6 +++--- rust/fory/src/lib.rs | 4 ++-- rust/tests/tests/test_cross_language.rs | 7 ++----- rust/tests/tests/test_weak.rs | 18 +++++++++--------- 8 files changed, 48 insertions(+), 39 deletions(-) diff --git a/rust/fory-core/src/config.rs b/rust/fory-core/src/config.rs index 077a47c564..991db1d724 100644 --- a/rust/fory-core/src/config.rs +++ b/rust/fory-core/src/config.rs @@ -37,7 +37,7 @@ pub struct Config { /// Whether reference tracking is enabled. /// When enabled, shared references and circular references are tracked /// and preserved during serialization/deserialization. - pub ref_tracking: bool, + pub track_ref: bool, } impl Default for Config { @@ -49,7 +49,7 @@ impl Default for Config { compress_string: false, max_dyn_depth: 5, check_struct_version: false, - ref_tracking: false, + track_ref: false, } } } @@ -98,7 +98,7 @@ impl Config { /// Check if reference tracking is enabled. #[inline(always)] - pub fn is_ref_tracking(&self) -> bool { - self.ref_tracking + pub fn is_track_ref(&self) -> bool { + self.track_ref } } diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs index a9a3ce948a..165de4bfc4 100644 --- a/rust/fory-core/src/fory.rs +++ b/rust/fory-core/src/fory.rs @@ -264,7 +264,7 @@ impl Fory { /// /// # Arguments /// - /// * `ref_tracking` - If `true`, enables reference tracking which allows + /// * `track_ref` - If `true`, enables reference tracking which allows /// preserving shared object references and circular references during /// serialization/deserialization. /// @@ -281,10 +281,10 @@ impl Fory { /// ```rust /// use fory_core::Fory; /// - /// let fory = Fory::default().ref_tracking(true); + /// let fory = Fory::default().track_ref(true); /// ``` - pub fn ref_tracking(mut self, ref_tracking: bool) -> Self { - self.config.ref_tracking = ref_tracking; + pub fn track_ref(mut self, track_ref: bool) -> Self { + self.config.track_ref = track_ref; self } @@ -619,9 +619,9 @@ impl Fory { context.writer.write_i32(-1); }; // Use RefMode based on config: - // - If ref_tracking is enabled, use RefMode::Tracking for the root object + // - If track_ref is enabled, use RefMode::Tracking for the root object // - Otherwise, use RefMode::NullOnly which writes NOT_NULL_VALUE_FLAG - let ref_mode = if self.config.ref_tracking { + let ref_mode = if self.config.track_ref { RefMode::Tracking } else { RefMode::NullOnly @@ -1014,9 +1014,9 @@ impl Fory { } } // Use RefMode based on config: - // - If ref_tracking is enabled, use RefMode::Tracking for the root object + // - If track_ref is enabled, use RefMode::Tracking for the root object // - Otherwise, use RefMode::NullOnly - let ref_mode = if self.config.ref_tracking { + let ref_mode = if self.config.track_ref { RefMode::Tracking } else { RefMode::NullOnly diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs index 615c7ac703..701fb75c06 100644 --- a/rust/fory-core/src/resolver/context.rs +++ b/rust/fory-core/src/resolver/context.rs @@ -120,6 +120,7 @@ pub struct WriteContext<'a> { compress_string: bool, xlang: bool, check_struct_version: bool, + track_ref: bool, // Context-specific fields default_writer: Option>, @@ -139,6 +140,7 @@ impl<'a> WriteContext<'a> { compress_string: config.compress_string, xlang: config.xlang, check_struct_version: config.check_struct_version, + track_ref: config.track_ref, default_writer: None, writer: Writer::from_buffer(Self::get_leak_buffer()), meta_resolver: MetaWriterResolver::default(), @@ -205,6 +207,12 @@ impl<'a> WriteContext<'a> { self.check_struct_version } + /// Check if reference tracking is enabled + #[inline(always)] + pub fn is_track_ref(&self) -> bool { + self.track_ref + } + #[inline(always)] pub fn empty(&mut self) -> bool { self.meta_resolver.empty() diff --git a/rust/fory-core/src/serializer/weak.rs b/rust/fory-core/src/serializer/weak.rs index 09181ea15d..c4bd3f4ee7 100644 --- a/rust/fory-core/src/serializer/weak.rs +++ b/rust/fory-core/src/serializer/weak.rs @@ -319,6 +319,12 @@ impl Serializer for RcWeak { write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { + // Weak pointers require track_ref to be enabled on the Fory instance + if !context.is_track_ref() { + return Err(Error::invalid_ref( + "RcWeak requires track_ref to be enabled. Use Fory::default().track_ref(true)", + )); + } // Weak MUST use ref tracking - otherwise read value will be lost if ref_mode != RefMode::Tracking { return Err(Error::invalid_ref( @@ -330,10 +336,6 @@ impl Serializer for RcWeak { .ref_writer .try_write_rc_ref(&mut context.writer, &rc) { - // Target not previously registered - serialize its data. - // Note: For circular references, ref_tracking should be enabled - // on the Fory instance to ensure proper reference resolution. - // Use `Fory::default().ref_tracking(true)` for circular references. if write_type_info { T::fory_write_type_info(context)?; } @@ -483,6 +485,12 @@ impl Serializer for ArcWeak write_type_info: bool, has_generics: bool, ) -> Result<(), Error> { + // Weak pointers require track_ref to be enabled on the Fory instance + if !context.is_track_ref() { + return Err(Error::invalid_ref( + "ArcWeak requires track_ref to be enabled. Use Fory::default().track_ref(true)", + )); + } // Weak MUST use ref tracking - otherwise read value will be lost if ref_mode != RefMode::Tracking { return Err(Error::invalid_ref( @@ -494,10 +502,6 @@ impl Serializer for ArcWeak .ref_writer .try_write_arc_ref(&mut context.writer, &arc) { - // Target not previously registered - serialize its data. - // Note: For circular references with Mutex/RwLock, this may cause deadlock - // if ref_tracking is not enabled on the Fory instance. - // Use `Fory::default().ref_tracking(true)` for circular references. if write_type_info { T::fory_write_type_info(context)?; } diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs index eb2d72a1c7..ec2c28b598 100644 --- a/rust/fory-core/src/types.rs +++ b/rust/fory-core/src/types.rs @@ -65,10 +65,10 @@ pub enum RefMode { } impl RefMode { - /// Create RefMode from nullable and ref_tracking flags. + /// Create RefMode from nullable and track_ref flags. #[inline] - pub const fn from_flags(nullable: bool, ref_tracking: bool) -> Self { - match (nullable, ref_tracking) { + pub const fn from_flags(nullable: bool, track_ref: bool) -> Self { + match (nullable, track_ref) { (false, false) => RefMode::None, (true, false) => RefMode::NullOnly, (_, true) => RefMode::Tracking, diff --git a/rust/fory/src/lib.rs b/rust/fory/src/lib.rs index 41f641ff19..15a4ccde65 100644 --- a/rust/fory/src/lib.rs +++ b/rust/fory/src/lib.rs @@ -251,7 +251,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default(); +//! let mut fory = Fory::default().track_ref(true); //! fory.register::(2000); //! //! let parent = Rc::new(RefCell::new(Node { @@ -293,7 +293,7 @@ //! } //! //! # fn main() -> Result<(), Error> { -//! let mut fory = Fory::default(); +//! let mut fory = Fory::default().track_ref(true); //! fory.register::(6000); //! //! let parent = Arc::new(Mutex::new(Node { diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs index 532b75b639..f62a894dce 100644 --- a/rust/tests/tests/test_cross_language.rs +++ b/rust/tests/tests/test_cross_language.rs @@ -1703,7 +1703,7 @@ fn test_ref_schema_consistent() { let mut fory = Fory::default() .compatible(false) .xlang(true) - .ref_tracking(true); + .track_ref(true); fory.register::(501).unwrap(); fory.register::(502).unwrap(); @@ -1747,10 +1747,7 @@ fn test_ref_compatible() { let data_file_path = get_data_file(); let bytes = fs::read(&data_file_path).unwrap(); - let mut fory = Fory::default() - .compatible(true) - .xlang(true) - .ref_tracking(true); + let mut fory = Fory::default().compatible(true).xlang(true).track_ref(true); fory.register::(503).unwrap(); fory.register::(504).unwrap(); diff --git a/rust/tests/tests/test_weak.rs b/rust/tests/tests/test_weak.rs index e3b561b990..ce3f53ea19 100644 --- a/rust/tests/tests/test_weak.rs +++ b/rust/tests/tests/test_weak.rs @@ -25,7 +25,7 @@ use std::sync::Mutex; #[test] fn test_rc_weak_null_serialization() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let weak: RcWeak = RcWeak::new(); @@ -37,7 +37,7 @@ fn test_rc_weak_null_serialization() { #[test] fn test_arc_weak_null_serialization() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let weak: ArcWeak = ArcWeak::new(); @@ -49,7 +49,7 @@ fn test_arc_weak_null_serialization() { #[test] fn test_rc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let weak = { let rc = Rc::new(42i32); @@ -69,7 +69,7 @@ fn test_rc_weak_dead_pointer_serializes_as_null() { #[test] fn test_arc_weak_dead_pointer_serializes_as_null() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let weak = { let arc = Arc::new(String::from("test")); @@ -89,7 +89,7 @@ fn test_arc_weak_dead_pointer_serializes_as_null() { #[test] fn test_rc_weak_in_vec_circular_reference() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let data1 = Rc::new(42i32); let data2 = Rc::new(100i32); @@ -107,7 +107,7 @@ fn test_rc_weak_in_vec_circular_reference() { #[test] fn test_arc_weak_in_vec_circular_reference() { - let fory = Fory::default().ref_tracking(true); + let fory = Fory::default().track_ref(true); let data1 = Arc::new(String::from("hello")); let data2 = Arc::new(String::from("world")); @@ -133,7 +133,7 @@ fn test_rc_weak_field_in_struct() { weak_ref: RcWeak, } - let mut fory = Fory::default().ref_tracking(true); + let mut fory = Fory::default().track_ref(true); fory.register::(1000).unwrap(); let data = Rc::new(42i32); @@ -160,7 +160,7 @@ struct Node { #[test] fn test_node_circular_reference_with_parent_children() { // Register the Node type with Fory - let mut fory = Fory::default().ref_tracking(true); + let mut fory = Fory::default().track_ref(true); fory.register::(2000).unwrap(); // Create parent @@ -218,7 +218,7 @@ fn test_arc_mutex_circular_reference() { children: Vec>>, } - let mut fory = Fory::default().ref_tracking(true); + let mut fory = Fory::default().track_ref(true); fory.register::(6000).unwrap(); let parent = Arc::new(Mutex::new(Node { From 0e577c64656a28a6e3e5f75a3ea6ea2a67654f43 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 16:15:49 +0800 Subject: [PATCH 21/35] fix(cpp): fix MSVC compilation and code style issues - Replace constexpr lambdas with helper template functions in type_resolver.h to fix MSVC C3493 error about implicit capture - Fix code style issue in type_info.h (remove extra space before &) --- cpp/fory/serialization/type_info.h | 6 +-- cpp/fory/serialization/type_resolver.h | 51 +++++++++++++++----------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/cpp/fory/serialization/type_info.h b/cpp/fory/serialization/type_info.h index 9334611223..12a8110299 100644 --- a/cpp/fory/serialization/type_info.h +++ b/cpp/fory/serialization/type_info.h @@ -56,12 +56,12 @@ struct Harness { using WriteFn = void (*)(const void *value, WriteContext &ctx, RefMode ref_mode, bool write_type_info, bool has_generics); - using ReadFn = void *(*)(ReadContext & ctx, RefMode ref_mode, + using ReadFn = void *(*)(ReadContext &ctx, RefMode ref_mode, bool read_type_info); using WriteDataFn = void (*)(const void *value, WriteContext &ctx, bool has_generics); - using ReadDataFn = void *(*)(ReadContext & ctx); - using ReadCompatibleFn = void *(*)(ReadContext & ctx, + using ReadDataFn = void *(*)(ReadContext &ctx); + using ReadCompatibleFn = void *(*)(ReadContext &ctx, const struct TypeInfo *type_info); using SortedFieldInfosFn = Result, Error> (*)(TypeResolver &); diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h index 29380ad7e1..8baa42042b 100644 --- a/cpp/fory/serialization/type_resolver.h +++ b/cpp/fory/serialization/type_resolver.h @@ -482,6 +482,33 @@ struct FieldTypeBuilder< } }; +// Helper template functions to compute is_nullable and track_ref at compile +// time. These replace constexpr lambdas which have issues on MSVC. +template +constexpr bool compute_is_nullable() { + if constexpr (is_fory_field_v) { + return ActualFieldType::is_nullable; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::is_nullable; + } else { + // Default: nullable if std::optional or std::shared_ptr + return is_optional_v || + is_shared_ptr_v; + } +} + +template +constexpr bool compute_track_ref() { + if constexpr (is_fory_field_v) { + return ActualFieldType::track_ref; + } else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::track_ref; + } else { + return false; + } +} + template struct FieldInfoBuilder { static FieldInfo build() { const auto meta = ForyFieldInfo(T{}); @@ -499,27 +526,9 @@ template struct FieldInfoBuilder { // Get nullable and track_ref from field tags (FORY_FIELD_TAGS or // fory::field<>) - constexpr bool is_nullable = []() { - if constexpr (is_fory_field_v) { - return ActualFieldType::is_nullable; - } else if constexpr (::fory::detail::has_field_tags_v) { - return ::fory::detail::GetFieldTagEntry::is_nullable; - } else { - // Default: nullable if std::optional or std::shared_ptr - return is_optional_v || - is_shared_ptr_v; - } - }(); - - constexpr bool track_ref = []() { - if constexpr (is_fory_field_v) { - return ActualFieldType::track_ref; - } else if constexpr (::fory::detail::has_field_tags_v) { - return ::fory::detail::GetFieldTagEntry::track_ref; - } else { - return false; - } - }(); + constexpr bool is_nullable = + compute_is_nullable(); + constexpr bool track_ref = compute_track_ref(); FieldType field_type = FieldTypeBuilder::build(false); // Override nullable and ref_tracking from field-level metadata From 678ab210984c4ab2f8ff80711278bb623189e2e3 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 17:36:45 +0800 Subject: [PATCH 22/35] fix(cpp): fix collection serialization for xlang compatibility - Fix MSVC C3493 error by replacing constexpr lambdas with helper template functions in type_resolver.h - Fix code style issue in type_info.h (remove extra space before &) - Fix collection ref flag encoding: only set COLL_TRACKING_REF when global ref tracking is enabled (ctx.track_ref()) - Fix heterogeneous collection element writing: use RefMode::None when has_null=false to avoid writing unexpected null flags - Fix same-type shared_ref collection writing: use write_data instead of RefMode::NullOnly when no nulls are present - Remove unused is_first_occurrence variable in smart_ptr_serializers --- .../serialization/collection_serializer.h | 57 +++++++++---------- .../serialization/smart_ptr_serializers.h | 2 - 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index eec5f7a0c1..ff6770deef 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -269,8 +269,12 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, if (is_same_type) { bitmap |= COLL_IS_SAME_TYPE; } + // Only set TRACKING_REF if element is shared ref AND global ref tracking is + // enabled if constexpr (elem_is_shared_ref) { - bitmap |= COLL_TRACKING_REF; + if (ctx.track_ref()) { + bitmap |= COLL_TRACKING_REF; + } } // Write header @@ -291,29 +295,24 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, if (is_same_type) { // All elements have same type - type info written once above if (!has_null) { - if constexpr (elem_is_shared_ref) { - // Write with ref flag, without type - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::NullOnly, false, - has_generics); - } - } else { - // Write data directly - for (const auto &elem : coll) { - if constexpr (is_nullable_v) { - using Inner = nullable_element_t; - Serializer::write_data(deref_nullable(elem), ctx); + // No nulls - write data directly without null flag + for (const auto &elem : coll) { + if constexpr (is_nullable_v) { + using Inner = nullable_element_t; + Serializer::write_data(deref_nullable(elem), ctx); + } else if constexpr (elem_is_shared_ref) { + // For shared_ptr, use write_data which handles polymorphic types + Serializer::write_data(elem, ctx); + } else { + if constexpr (is_generic_type_v) { + Serializer::write_data_generic(elem, ctx, has_generics); } else { - if constexpr (is_generic_type_v) { - Serializer::write_data_generic(elem, ctx, has_generics); - } else { - Serializer::write_data(elem, ctx); - } + Serializer::write_data(elem, ctx); } } } } else { - // Has null elements - write with ref flag for null tracking + // Has null elements - write with null flag for (const auto &elem : coll) { Serializer::write(elem, ctx, RefMode::NullOnly, false, has_generics); } @@ -321,18 +320,12 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, } else { // Heterogeneous types - write type info per element if (!has_null) { - if constexpr (elem_is_shared_ref) { - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::NullOnly, true, - has_generics); - } - } else { - for (const auto &elem : coll) { - Serializer::write(elem, ctx, RefMode::None, true, has_generics); - } + // No nulls - write without null flag (RefMode::None) + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::None, true, has_generics); } } else { - // Has null elements + // Has null elements - write with null flag (RefMode::NullOnly) for (const auto &elem : coll) { Serializer::write(elem, ctx, RefMode::NullOnly, true, has_generics); } @@ -1609,8 +1602,12 @@ struct Serializer> { if (is_same_type) { bitmap |= COLL_IS_SAME_TYPE; } + // Only set TRACKING_REF if element is shared ref AND global ref tracking + // is enabled if constexpr (elem_is_shared_ref) { - bitmap |= COLL_TRACKING_REF; + if (ctx.track_ref()) { + bitmap |= COLL_TRACKING_REF; + } } // Write header diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index 23229f04e4..cb8b7f9374 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -323,12 +323,10 @@ template struct Serializer> { return; } - bool is_first_occurrence = false; if (ctx.track_ref()) { if (ctx.ref_writer().try_write_shared_ref(ctx, ptr)) { return; } - is_first_occurrence = true; } else { ctx.write_int8(NOT_NULL_VALUE_FLAG); } From ee6b89a1d3eb93bfaa678495b407bd09b76b169a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 17:42:03 +0800 Subject: [PATCH 23/35] add ref tracking header check in java --- .../serializer/collection/CollectionLikeSerializer.java | 1 + .../fory/serializer/collection/MapLikeSerializer.java | 9 +++++++++ .../main/java/org/apache/fory/util/Preconditions.java | 6 ++++++ .../test/java/org/apache/fory/xlang/RustXlangTest.java | 2 +- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java index 0a0e18880a..4589065265 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/CollectionLikeSerializer.java @@ -607,6 +607,7 @@ private void readSameTypeElements( private void readDifferentTypeElements( Fory fory, MemoryBuffer buffer, int flags, T collection, int numElements) { if ((flags & CollectionFlags.TRACKING_REF) == CollectionFlags.TRACKING_REF) { + Preconditions.checkState(fory.trackingRef(), "Reference tracking is not enabled"); for (int i = 0; i < numElements; i++) { collection.add(binding.readRef(buffer)); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java index e1ff58fd7a..79b984e375 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/collection/MapLikeSerializer.java @@ -52,6 +52,7 @@ import org.apache.fory.type.GenericType; import org.apache.fory.type.Generics; import org.apache.fory.type.TypeUtils; +import org.apache.fory.util.Preconditions; /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) @@ -80,6 +81,7 @@ public abstract class MapLikeSerializer extends Serializer { private int numElements; private final TypeResolver typeResolver; protected final SerializationBinding binding; + private boolean trackRef; public MapLikeSerializer(Fory fory, Class cls) { this(fory, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); @@ -92,6 +94,7 @@ public MapLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook) { public MapLikeSerializer(Fory fory, Class cls, boolean supportCodegenHook, boolean immutable) { super(fory, cls, immutable); this.typeResolver = fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); + trackRef = fory.trackingRef(); this.supportCodegenHook = supportCodegenHook; keyClassInfoWriteCache = typeResolver.nilClassInfoHolder(); keyClassInfoReadCache = typeResolver.nilClassInfoHolder(); @@ -773,6 +776,9 @@ private long readJavaChunk( // noinspection Duplicates boolean trackKeyRef = (chunkHeader & TRACKING_KEY_REF) != 0; boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0; + if (trackKeyRef || trackValueRef) { + Preconditions.checkState(trackRef, "Ref tracking is not enabled"); + } boolean keyIsDeclaredType = (chunkHeader & KEY_DECL_TYPE) != 0; boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0; int chunkSize = buffer.readUnsignedByte(); @@ -818,6 +824,9 @@ private long readJavaChunkGeneric( // noinspection Duplicates boolean trackKeyRef = (chunkHeader & TRACKING_KEY_REF) != 0; boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0; + if (trackKeyRef || trackValueRef) { + Preconditions.checkState(trackRef, "Ref tracking is not enabled"); + } boolean keyIsDeclaredType = (chunkHeader & KEY_DECL_TYPE) != 0; boolean valueIsDeclaredType = (chunkHeader & VALUE_DECL_TYPE) != 0; int chunkSize = buffer.readUnsignedByte(); diff --git a/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java b/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java index 1a6cfaead0..1f0e133b58 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/Preconditions.java @@ -35,6 +35,12 @@ public static T checkNotNull(T o, String errorMessage) { return o; } + public static void checkState(boolean expression, String errorMessage) { + if (!expression) { + throw new IllegalStateException(errorMessage); + } + } + public static void checkState(boolean expression) { if (!expression) { throw new IllegalStateException(); diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java index 583bb3b588..43e91072df 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/RustXlangTest.java @@ -53,7 +53,7 @@ public class RustXlangTest extends XlangTestBase { protected void ensurePeerReady() { String enabled = System.getenv("FORY_RUST_JAVA_CI"); if (!"1".equals(enabled)) { - throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); + throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1"); } boolean rustInstalled = true; try { From 7737c7a5d8fc87eaa56d46797d8c7e7f8acbe49a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:04:30 +0800 Subject: [PATCH 24/35] simplify shared_ptr value type --- .../serialization/smart_ptr_serializers.h | 116 +++++++++--------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index cb8b7f9374..c739e016d3 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -253,7 +253,17 @@ template struct SharedPtrTypeIdHelper { /// Supports reference tracking for shared and circular references. /// When reference tracking is enabled, identical shared_ptr instances /// will serialize only once and use reference IDs for subsequent occurrences. +/// +/// Note: The element type T must be a value type (struct/class or primitive). +/// Raw pointers and nullable wrappers are not allowed as they would require +/// nested ref metadata handling, which complicates the protocol. template struct Serializer> { + static_assert(!std::is_pointer_v, + "shared_ptr of raw pointer types is not supported"); + static_assert(!requires_ref_metadata_v, + "shared_ptr of nullable types (optional/shared_ptr/unique_ptr) " + "is not supported. Use the wrapper type directly instead."); + static constexpr TypeId type_id = SharedPtrTypeIdHelper>::value; @@ -279,7 +289,6 @@ template struct Serializer> { static inline void write(const std::shared_ptr &ptr, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -310,9 +319,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } return; } @@ -359,10 +367,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - // Non-polymorphic path - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } } @@ -416,7 +422,6 @@ template struct Serializer> { static inline std::shared_ptr read(ReadContext &ctx, RefMode ref_mode, bool read_type) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -437,9 +442,9 @@ template struct Serializer> { // Now use read_with_type_info with the concrete type info return read_with_type_info(ctx, ref_mode, *type_info); } else { - constexpr RefMode inner_ref_mode = - inner_requires_ref ? RefMode::NullOnly : RefMode::None; - T value = Serializer::read(ctx, inner_ref_mode, read_type); + // T is guaranteed to be a value type (not pointer or nullable wrapper) + // by static_assert, so no inner ref metadata needed. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -453,7 +458,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::shared_ptr(nullptr); + return nullptr; } const bool tracking_refs = ctx.track_ref(); if (flag == REF_FLAG) { @@ -491,15 +496,9 @@ template struct Serializer> { reserved_ref_id = ctx.ref_reader().reserve_ref_id(); } - // In compatible mode with ref tracking, first occurrence (RefValue) - // has type info written after the ref flag by Java/Go. - // So we must read type info for first occurrence. - const bool should_read_type = - ctx.is_compatible() && is_first_occurrence ? true : read_type; - // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!should_read_type) { + if (!read_type) { ctx.set_error(Error::type_error( "Cannot deserialize polymorphic std::shared_ptr " "without type info (read_type=false)")); @@ -532,10 +531,9 @@ template struct Serializer> { } return result; } else { - // Non-polymorphic path - T value = Serializer::read( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - should_read_type); + // Non-polymorphic path: T is guaranteed to be a value type (not pointer + // or nullable wrapper) by static_assert, so no inner ref metadata needed. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -550,7 +548,6 @@ template struct Serializer> { static inline std::shared_ptr read_with_type_info(ReadContext &ctx, RefMode ref_mode, const TypeInfo &type_info) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -564,10 +561,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::shared_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -582,7 +578,7 @@ template struct Serializer> { } if (flag == NULL_FLAG) { - return std::shared_ptr(nullptr); + return nullptr; } const bool tracking_refs = ctx.track_ref(); if (flag == REF_FLAG) { @@ -641,10 +637,9 @@ template struct Serializer> { } return result; } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -652,7 +647,6 @@ template struct Serializer> { if (flag == REF_VALUE_FLAG) { ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); } - return result; } } @@ -685,14 +679,23 @@ template struct UniquePtrTypeIdHelper { /// Note: unique_ptr does not support reference tracking since /// it represents exclusive ownership. Each unique_ptr is serialized /// independently. +/// +/// Note: The element type T must be a value type (struct/class or primitive). +/// Raw pointers and nullable wrappers are not allowed as they would require +/// nested ref metadata handling, which complicates the protocol. template struct Serializer> { + static_assert(!std::is_pointer_v, + "unique_ptr of raw pointer types is not supported"); + static_assert(!requires_ref_metadata_v, + "unique_ptr of nullable types (optional/shared_ptr/unique_ptr) " + "is not supported. Use the wrapper type directly instead."); + static constexpr TypeId type_id = UniquePtrTypeIdHelper>::value; static inline void write(const std::unique_ptr &ptr, WriteContext &ctx, RefMode ref_mode, bool write_type, bool has_generics = false) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -723,9 +726,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } return; } @@ -758,9 +760,8 @@ template struct Serializer> { const void *value_ptr = ptr.get(); type_info->harness.write_data_fn(value_ptr, ctx, has_generics); } else { - Serializer::write( - *ptr, ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - write_type); + // T is guaranteed to be a value type by static_assert. + Serializer::write(*ptr, ctx, RefMode::None, write_type); } } @@ -812,7 +813,6 @@ template struct Serializer> { static inline std::unique_ptr read(ReadContext &ctx, RefMode ref_mode, bool read_type) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -833,9 +833,8 @@ template struct Serializer> { // Now use read_with_type_info with the concrete type info return read_with_type_info(ctx, ref_mode, *type_info); } else { - constexpr RefMode inner_ref_mode = - inner_requires_ref ? RefMode::NullOnly : RefMode::None; - T value = Serializer::read(ctx, inner_ref_mode, read_type); + // T is guaranteed to be a value type by static_assert. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -849,7 +848,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::unique_ptr(nullptr); + return nullptr; } if (flag != NOT_NULL_VALUE_FLAG) { ctx.set_error( @@ -889,10 +888,8 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - read_type); + // T is guaranteed to be a value type by static_assert. + T value = Serializer::read(ctx, RefMode::None, read_type); if (ctx.has_error()) { return nullptr; } @@ -903,7 +900,6 @@ template struct Serializer> { static inline std::unique_ptr read_with_type_info(ReadContext &ctx, RefMode ref_mode, const TypeInfo &type_info) { - constexpr bool inner_requires_ref = requires_ref_metadata_v; constexpr bool is_polymorphic = std::is_polymorphic_v; // Handle ref_mode == RefMode::None case (similar to Rust) @@ -917,10 +913,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } @@ -934,7 +929,7 @@ template struct Serializer> { return nullptr; } if (flag == NULL_FLAG) { - return std::unique_ptr(nullptr); + return nullptr; } if (flag != NOT_NULL_VALUE_FLAG) { ctx.set_error( @@ -961,10 +956,9 @@ template struct Serializer> { T *obj_ptr = static_cast(raw_ptr); return std::unique_ptr(obj_ptr); } else { - // Non-polymorphic path - T value = Serializer::read_with_type_info( - ctx, inner_requires_ref ? RefMode::NullOnly : RefMode::None, - type_info); + // T is guaranteed to be a value type by static_assert. + T value = + Serializer::read_with_type_info(ctx, RefMode::None, type_info); if (ctx.has_error()) { return nullptr; } From 8a87f501ffc020c1b5cc038a42336f3bbcb1cb61 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:08:12 +0800 Subject: [PATCH 25/35] fix c++ collection tests --- .../serialization/collection_serializer.h | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cpp/fory/serialization/collection_serializer.h b/cpp/fory/serialization/collection_serializer.h index ff6770deef..1000fdb31d 100644 --- a/cpp/fory/serialization/collection_serializer.h +++ b/cpp/fory/serialization/collection_serializer.h @@ -291,11 +291,19 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, } } + // Determine if we're actually tracking refs for this collection + const bool tracking_refs = (bitmap & COLL_TRACKING_REF) != 0; + // Write elements if (is_same_type) { // All elements have same type - type info written once above - if (!has_null) { - // No nulls - write data directly without null flag + if (tracking_refs) { + // Track refs - write ref flag per element per xlang spec + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::Tracking, false, has_generics); + } + } else if (!has_null) { + // No nulls, no ref tracking - write data directly without null flag for (const auto &elem : coll) { if constexpr (is_nullable_v) { using Inner = nullable_element_t; @@ -319,7 +327,12 @@ inline void write_collection_data_slow(const Container &coll, WriteContext &ctx, } } else { // Heterogeneous types - write type info per element - if (!has_null) { + if (tracking_refs) { + // Track refs - write ref flag + type info per element + for (const auto &elem : coll) { + Serializer::write(elem, ctx, RefMode::Tracking, true, has_generics); + } + } else if (!has_null) { // No nulls - write without null flag (RefMode::None) for (const auto &elem : coll) { Serializer::write(elem, ctx, RefMode::None, true, has_generics); @@ -409,7 +422,8 @@ inline Container read_collection_data_slow(ReadContext &ctx, uint32_t length) { return result; } if constexpr (elem_is_polymorphic) { - auto elem = Serializer::read_with_type_info(ctx, RefMode::NullOnly, + // Use RefMode::Tracking to read ref flag per element + auto elem = Serializer::read_with_type_info(ctx, RefMode::Tracking, *elem_type_info); collection_insert(result, std::move(elem)); } else { From 99cc7fb7afdab54ffa7c498b668907b108a1a198 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:19:18 +0800 Subject: [PATCH 26/35] fix map serialization header in c++ --- cpp/fory/serialization/map_serializer.h | 54 +++++++++++++++---------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/cpp/fory/serialization/map_serializer.h b/cpp/fory/serialization/map_serializer.h index 767726fa65..8947b2773f 100644 --- a/cpp/fory/serialization/map_serializer.h +++ b/cpp/fory/serialization/map_serializer.h @@ -206,8 +206,6 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, constexpr bool val_is_polymorphic = is_polymorphic_v; constexpr bool key_is_shared_ref = is_shared_ref_v; constexpr bool val_is_shared_ref = is_shared_ref_v; - constexpr bool key_needs_ref = requires_ref_metadata_v; - constexpr bool val_needs_ref = requires_ref_metadata_v; const bool is_key_declared = has_generics && !need_to_write_type_for_field(); @@ -251,7 +249,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, // Non-null key, null value // Java writes: chunk_header, then ref_flag, then type_info, then data uint8_t chunk_header = VALUE_NULL; - bool write_ref = key_is_shared_ref || key_needs_ref; + // Only track refs for shared_ptr types when global ref tracking enabled + bool write_ref = key_is_shared_ref && ctx.track_ref(); if (write_ref) { chunk_header |= TRACKING_KEY_REF; } @@ -297,7 +296,8 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, // key_is_none // Java writes: chunk_header, then ref_flag, then type_info, then data uint8_t chunk_header = KEY_NULL; - bool write_ref = val_is_shared_ref || val_needs_ref; + // Only track refs for shared_ptr types when global ref tracking enabled + bool write_ref = val_is_shared_ref && ctx.track_ref(); if (write_ref) { chunk_header |= TRACKING_VALUE_REF; } @@ -385,16 +385,18 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, ctx.write_uint16(0); // Placeholder for header and chunk size uint8_t chunk_header = 0; - // Set key flags - if (key_is_shared_ref || key_needs_ref) { + // Set key flags - only track refs for shared_ptr when global ref tracking + // enabled + if (key_is_shared_ref && ctx.track_ref()) { chunk_header |= TRACKING_KEY_REF; } if (is_key_declared && !key_is_polymorphic) { chunk_header |= DECL_KEY_TYPE; } - // Set value flags - if (val_is_shared_ref || val_needs_ref) { + // Set value flags - only track refs for shared_ptr when global ref + // tracking enabled + if (val_is_shared_ref && ctx.track_ref()) { chunk_header |= TRACKING_VALUE_REF; } if (is_val_declared && !val_is_polymorphic) { @@ -443,12 +445,15 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } // Write key-value pair - // For polymorphic types, we've already written type info above, - // so we write ref flag + data directly using the serializer + // For shared_ptr with ref tracking: write ref flag + data + // For other types: null cases already handled via KEY_NULL/VALUE_NULL, + // so just write data directly if constexpr (key_is_shared_ref) { - Serializer::write(key, ctx, RefMode::NullOnly, false, has_generics); - } else if constexpr (key_needs_ref) { - Serializer::write(key, ctx, RefMode::NullOnly, false); + if (ctx.track_ref()) { + Serializer::write(key, ctx, RefMode::Tracking, false, has_generics); + } else { + Serializer::write_data(key, ctx); + } } else { if (has_generics && is_generic_type_v) { Serializer::write_data_generic(key, ctx, has_generics); @@ -458,9 +463,12 @@ inline void write_map_data_slow(const MapType &map, WriteContext &ctx, } if constexpr (val_is_shared_ref) { - Serializer::write(value, ctx, RefMode::NullOnly, false, has_generics); - } else if constexpr (val_needs_ref) { - Serializer::write(value, ctx, RefMode::NullOnly, false); + if (ctx.track_ref()) { + Serializer::write(value, ctx, RefMode::Tracking, false, + has_generics); + } else { + Serializer::write_data(value, ctx); + } } else { if (has_generics && is_generic_type_v) { Serializer::write_data_generic(value, ctx, has_generics); @@ -822,15 +830,16 @@ inline MapType read_map_data_slow(ReadContext &ctx, uint32_t length) { // Read key - use type info if available (polymorphic case) K key; if constexpr (key_is_polymorphic) { + // TRACKING_KEY_REF means full ref tracking for shared_ptr key = Serializer::read_with_type_info( - ctx, key_read_ref ? RefMode::NullOnly : RefMode::None, + ctx, key_read_ref ? RefMode::Tracking : RefMode::None, *key_type_info); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } } else if (key_read_ref) { - key = - Serializer::read(ctx, make_ref_mode(false, key_read_ref), false); + // TRACKING_KEY_REF means full ref tracking for shared_ptr + key = Serializer::read(ctx, RefMode::Tracking, false); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } @@ -845,15 +854,16 @@ inline MapType read_map_data_slow(ReadContext &ctx, uint32_t length) { // Read value - use type info if available (polymorphic case) V value; if constexpr (val_is_polymorphic) { + // TRACKING_VALUE_REF means full ref tracking for shared_ptr value = Serializer::read_with_type_info( - ctx, val_read_ref ? RefMode::NullOnly : RefMode::None, + ctx, val_read_ref ? RefMode::Tracking : RefMode::None, *value_type_info); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } } else if (val_read_ref) { - value = - Serializer::read(ctx, make_ref_mode(false, val_read_ref), false); + // TRACKING_VALUE_REF means full ref tracking for shared_ptr + value = Serializer::read(ctx, RefMode::Tracking, false); if (FORY_PREDICT_FALSE(ctx.has_error())) { return MapType{}; } From 3a9149f689223805806aacfda6efab16c0652231 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:21:07 +0800 Subject: [PATCH 27/35] fix python xlang collection serde --- python/pyfory/_serializer.py | 2 +- python/pyfory/collection.pxi | 2 +- python/pyfory/collection.py | 2 +- python/pyfory/serialization.pyx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pyfory/_serializer.py b/python/pyfory/_serializer.py index e1a8231985..33bbe6e003 100644 --- a/python/pyfory/_serializer.py +++ b/python/pyfory/_serializer.py @@ -39,7 +39,7 @@ class Serializer(ABC): def __init__(self, fory, type_: type): self.fory = fory self.type_: type = type_ - self.need_to_write_ref = not is_primitive_type(type_) + self.need_to_write_ref = fory.ref_tracking and not is_primitive_type(type_) def write(self, buffer, value): raise NotImplementedError diff --git a/python/pyfory/collection.pxi b/python/pyfory/collection.pxi index e800f59300..2e9769fbea 100644 --- a/python/pyfory/collection.pxi +++ b/python/pyfory/collection.pxi @@ -638,7 +638,7 @@ cdef int32_t NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF =KEY_HAS_NULL | VALUE_DECL_TY # Value is null, key type is declared type, and ref tracking for key is disabled. cdef int32_t NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE # Value is null, key type is declared type, and ref tracking for key is enabled. -cdef int32_t NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_VALUE_REF +cdef int32_t NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF @cython.final diff --git a/python/pyfory/collection.py b/python/pyfory/collection.py index 451d05b4d1..f8a026ddc7 100644 --- a/python/pyfory/collection.py +++ b/python/pyfory/collection.py @@ -342,7 +342,7 @@ def get_next_element(buffer, ref_resolver, type_resolver, is_py): # Value is null, key type is declared type, and ref tracking for key is disabled. NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE # Value is null, key type is declared type, and ref tracking for key is enabled. -NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_VALUE_REF +NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF class MapSerializer(Serializer): diff --git a/python/pyfory/serialization.pyx b/python/pyfory/serialization.pyx index a687b64a70..592de476ea 100644 --- a/python/pyfory/serialization.pyx +++ b/python/pyfory/serialization.pyx @@ -1694,7 +1694,7 @@ cdef class Serializer: def __init__(self, fory, type_: Union[type, TypeVar]): self.fory = fory self.type_ = type_ - self.need_to_write_ref = not is_primitive_type(type_) + self.need_to_write_ref = fory.ref_tracking and not is_primitive_type(type_) cpdef write(self, Buffer buffer, value): raise NotImplementedError(f"write method not implemented in {type(self)}") From fa2d909e948d6c3f1a3d2a8fd5431439b3d1017d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:30:09 +0800 Subject: [PATCH 28/35] fix c++ struct field type info --- cpp/fory/serialization/struct_serializer.h | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 786fc30069..31d9edaf9a 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -1456,9 +1456,16 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { // `Serializer::read` can dispatch to `read_compatible` with the correct // remote schema. constexpr bool field_requires_ref = requires_ref_metadata_v; - constexpr bool is_struct_field = is_fory_serializable_v; - constexpr bool is_polymorphic_field = - Serializer::type_id == TypeId::UNKNOWN; + constexpr TypeId field_type_id = Serializer::type_id; + // Check if field is a struct type - use type_id to handle shared_ptr + constexpr bool is_struct_field = + field_type_id == TypeId::STRUCT || + field_type_id == TypeId::COMPATIBLE_STRUCT || + field_type_id == TypeId::NAMED_STRUCT || + field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; + constexpr bool is_ext_field = + field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; bool read_type = is_polymorphic_field; // Get field metadata from fory::field<> or FORY_FIELD_TAGS or defaults @@ -1470,14 +1477,14 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { // `Serializer::read` can dispatch to `read_compatible` with the correct // remote TypeMeta instead of treating the bytes as part of the first field // value. - if (!is_polymorphic_field && is_struct_field && ctx.is_compatible()) { + if (!is_polymorphic_field && (is_struct_field || is_ext_field) && + ctx.is_compatible()) { read_type = true; } // Per xlang spec, all non-primitive fields have ref flags. // Primitive types: bool, int8-64, var_int32/64, sli_int64, float16/32/64 // Non-primitives include: string, list, set, map, struct, enum, etc. - constexpr TypeId field_type_id = Serializer::type_id; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); // Compute RefMode based on field metadata @@ -1534,10 +1541,16 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, // Unwrap fory::field<> to get the actual type for deserialization using FieldType = unwrap_field_t; - constexpr bool is_struct_field = is_fory_serializable_v; - constexpr bool is_polymorphic_field = - Serializer::type_id == TypeId::UNKNOWN; constexpr TypeId field_type_id = Serializer::type_id; + // Check if field is a struct type - use type_id to handle shared_ptr + constexpr bool is_struct_field = + field_type_id == TypeId::STRUCT || + field_type_id == TypeId::COMPATIBLE_STRUCT || + field_type_id == TypeId::NAMED_STRUCT || + field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; + constexpr bool is_ext_field = + field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); bool read_type = is_polymorphic_field; @@ -1547,7 +1560,8 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, // `Serializer::read` can dispatch to `read_compatible` with the correct // remote TypeMeta instead of treating the bytes as part of the first field // value. - if (!is_polymorphic_field && is_struct_field && ctx.is_compatible()) { + if (!is_polymorphic_field && (is_struct_field || is_ext_field) && + ctx.is_compatible()) { read_type = true; } From 15047d8f2f1b8b63a9983fa34e49e6dfe408527f Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:40:05 +0800 Subject: [PATCH 29/35] fix go serialization for any value --- go/fory/codegen/decoder.go | 30 ++++---- go/fory/codegen/encoder.go | 28 +++---- go/fory/fory.go | 14 ++-- go/fory/map.go | 56 ++++++++++++-- go/fory/pointer.go | 10 +-- go/fory/reader.go | 117 +++++++++++++++++++----------- go/fory/struct.go | 8 +- go/fory/tests/structs_fory_gen.go | 8 +- go/fory/writer.go | 10 ++- 9 files changed, 179 insertions(+), 102 deletions(-) diff --git a/go/fory/codegen/decoder.go b/go/fory/codegen/decoder.go index 1eadc41b04..9542f499ae 100644 --- a/go/fory/codegen/decoder.go +++ b/go/fory/codegen/decoder.go @@ -99,7 +99,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { case "time.Time", "github.com/apache/fory/go/fory.Date": // These types are "other internal types" in the new spec // They use: | null flag | value data | format - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } } @@ -107,7 +107,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { // Handle pointer types if _, ok := field.Type.(*types.Pointer); ok { // For pointer types, use ReadValue - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -173,7 +173,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t%s = make([]interface{}, sliceLen)\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t// ReadData each element using ReadValue\n") fmt.Fprintf(buf, "\t\t\t\tfor i := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem(), fory.RefModeTracking, true)\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t} else {\n") @@ -192,7 +192,7 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t\t%s = make([]interface{}, sliceLen)\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t\t// ReadData each element using ReadValue\n") fmt.Fprintf(buf, "\t\t\t\t\tfor i := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s[i]).Elem(), fory.RefModeTracking, true)\n", fieldAccess) fmt.Fprintf(buf, "\t\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") @@ -220,14 +220,14 @@ func generateFieldReadTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := field.Type.(*types.Interface); ok { if iface.Empty() { // For interface{}, use ReadValue for dynamic type handling - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } } // Handle struct types if _, ok := field.Type.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -343,14 +343,14 @@ func generateSliceElementRead(buf *bytes.Buffer, elemType types.Type, elemAccess } // Check if it's a struct if _, ok := named.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } } // Handle struct types if _, ok := elemType.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } @@ -611,7 +611,7 @@ func generateSliceElementReadInline(buf *bytes.Buffer, elemType types.Type, elem if iface, ok := elemType.(*types.Interface); ok { if iface.Empty() { // For interface{} elements, use ReadValue for dynamic type handling - fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", elemAccess) return nil } } @@ -781,7 +781,7 @@ func writeMapReadChunks(buf *bytes.Buffer, mapType *types.Map, fieldAccess strin // ReadData key if keyIsInterface { fmt.Fprintf(buf, "%s\t\tvar mapKey interface{}\n", indent) - fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem())\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapKey).Elem(), fory.RefModeTracking, true)\n", indent) } else { keyVarType := getGoTypeString(keyType) fmt.Fprintf(buf, "%s\t\tvar mapKey %s\n", indent, keyVarType) @@ -793,7 +793,7 @@ func writeMapReadChunks(buf *bytes.Buffer, mapType *types.Map, fieldAccess strin // ReadData value if valueIsInterface { fmt.Fprintf(buf, "%s\t\tvar mapValue interface{}\n", indent) - fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem())\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.ReadValue(reflect.ValueOf(&mapValue).Elem(), fory.RefModeTracking, true)\n", indent) } else { valueVarType := getGoTypeString(valueType) fmt.Fprintf(buf, "%s\t\tvar mapValue %s\n", indent, valueVarType) @@ -847,7 +847,7 @@ func generateMapKeyRead(buf *bytes.Buffer, keyType types.Type, varName string) e } // For other types, use ReadValue - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", varName) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", varName) return nil } @@ -870,7 +870,7 @@ func generateMapValueRead(buf *bytes.Buffer, valueType types.Type, varName strin } // For other types, use ReadValue - fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", varName) + fmt.Fprintf(buf, "\t\t\t\t\tctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", varName) return nil } @@ -887,7 +887,7 @@ func generateMapKeyReadIndented(buf *bytes.Buffer, keyType types.Type, varName s } return nil } - fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", indent, varName) + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", indent, varName) return nil } @@ -904,6 +904,6 @@ func generateMapValueReadIndented(buf *bytes.Buffer, valueType types.Type, varNa } return nil } - fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem())\n", indent, varName) + fmt.Fprintf(buf, "%sctx.ReadValue(reflect.ValueOf(&%s).Elem(), fory.RefModeTracking, true)\n", indent, varName) return nil } diff --git a/go/fory/codegen/encoder.go b/go/fory/codegen/encoder.go index 81996f4cba..b4a0d141a3 100644 --- a/go/fory/codegen/encoder.go +++ b/go/fory/codegen/encoder.go @@ -85,7 +85,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { case "time.Time", "github.com/apache/fory/go/fory.Date": // These types are "other internal types" in the new spec // They use: | null flag | value data | format - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } } @@ -93,7 +93,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { // Handle pointer types if _, ok := field.Type.(*types.Pointer); ok { // For all pointer types, use WriteValue - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -161,7 +161,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\tbuf.WriteInt8(1) // CollectionTrackingRef only\n") fmt.Fprintf(buf, "\t\t\t\t// WriteData each element using WriteValue\n") fmt.Fprintf(buf, "\t\t\t\tfor _, elem := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem))\n") + fmt.Fprintf(buf, "\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true)\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") fmt.Fprintf(buf, "\t\t} else {\n") @@ -178,7 +178,7 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { fmt.Fprintf(buf, "\t\t\t\t\tbuf.WriteInt8(1) // CollectionTrackingRef only\n") fmt.Fprintf(buf, "\t\t\t\t\t// WriteData each element using WriteValue\n") fmt.Fprintf(buf, "\t\t\t\t\tfor _, elem := range %s {\n", fieldAccess) - fmt.Fprintf(buf, "\t\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem))\n") + fmt.Fprintf(buf, "\t\t\t\t\t\tctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true)\n") fmt.Fprintf(buf, "\t\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t\t}\n") fmt.Fprintf(buf, "\t\t\t}\n") @@ -206,14 +206,14 @@ func generateFieldWriteTyped(buf *bytes.Buffer, field *FieldInfo) error { if iface, ok := field.Type.(*types.Interface); ok { if iface.Empty() { // For interface{}, use WriteValue for dynamic type handling - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } } // Handle struct types if _, ok := field.Type.Underlying().(*types.Struct); ok { - fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s))\n", fieldAccess) + fmt.Fprintf(buf, "\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", fieldAccess) return nil } @@ -556,7 +556,7 @@ func writeMapChunksCode(buf *bytes.Buffer, keyType, valueType types.Type, fieldA // WriteData key if keyIsInterface { - fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapKey))\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapKey), fory.RefModeTracking, true)\n", indent) } else { if err := generateMapKeyWriteIndented(buf, keyType, "mapKey", indent+"\t\t"); err != nil { return err @@ -565,7 +565,7 @@ func writeMapChunksCode(buf *bytes.Buffer, keyType, valueType types.Type, fieldA // WriteData value if valueIsInterface { - fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapValue))\n", indent) + fmt.Fprintf(buf, "%s\t\tctx.WriteValue(reflect.ValueOf(mapValue), fory.RefModeTracking, true)\n", indent) } else { if err := generateMapValueWriteIndented(buf, valueType, "mapValue", indent+"\t\t"); err != nil { return err @@ -641,7 +641,7 @@ func generateMapKeyWrite(buf *bytes.Buffer, keyType types.Type, varName string) } // For other types, use WriteValue - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", varName) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", varName) return nil } @@ -663,7 +663,7 @@ func generateMapValueWrite(buf *bytes.Buffer, valueType types.Type, varName stri } // For other types, use WriteValue - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", varName) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", varName) return nil } @@ -685,7 +685,7 @@ func generateMapKeyWriteIndented(buf *bytes.Buffer, keyType types.Type, varName } // For other types, use WriteValue - fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, varName) + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, varName) return nil } @@ -707,7 +707,7 @@ func generateMapValueWriteIndented(buf *bytes.Buffer, valueType types.Type, varN } // For other types, use WriteValue - fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, varName) + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, varName) return nil } @@ -787,7 +787,7 @@ func generateSliceElementWriteInline(buf *bytes.Buffer, elemType types.Type, ele if iface, ok := elemType.(*types.Interface); ok { if iface.Empty() { // For interface{} elements, use WriteValue for dynamic type handling - fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s))\n", elemAccess) + fmt.Fprintf(buf, "\t\t\t\tctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", elemAccess) return nil } } @@ -834,7 +834,7 @@ func generateSliceElementWriteInlineIndented(buf *bytes.Buffer, elemType types.T if iface, ok := elemType.(*types.Interface); ok { if iface.Empty() { // For interface{} elements, use WriteValue for dynamic type handling - fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s))\n", indent, elemAccess) + fmt.Fprintf(buf, "%sctx.WriteValue(reflect.ValueOf(%s), fory.RefModeTracking, true)\n", indent, elemAccess) return nil } } diff --git a/go/fory/fory.go b/go/fory/fory.go index 0dbaa2b3a8..244fbd45ca 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -418,7 +418,7 @@ func (f *Fory) Serialize(value any) ([]byte, error) { } // SerializeWithCallback the value - f.writeCtx.WriteValue(reflect.ValueOf(value)) + f.writeCtx.WriteValue(reflect.ValueOf(value), RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } @@ -477,7 +477,7 @@ func (f *Fory) Deserialize(data []byte, v interface{}) error { // Read directly into target value target := reflect.ValueOf(v).Elem() - f.readCtx.ReadValue(target) + f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } @@ -559,7 +559,7 @@ func (f *Fory) SerializeTo(buf *ByteBuffer, value interface{}) error { } // Standard path - f.writeCtx.WriteValue(rv) + f.writeCtx.WriteValue(rv, RefModeTracking, true) if f.writeCtx.HasError() { f.writeCtx.buffer = origBuffer return f.writeCtx.TakeError() @@ -633,7 +633,7 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v interface{}) error { // Read directly into target value target := reflect.ValueOf(v).Elem() - f.readCtx.ReadValue(target) + f.readCtx.ReadValue(target, RefModeTracking, true) if f.readCtx.HasError() { f.readCtx.buffer = origBuffer return f.readCtx.TakeError() @@ -710,7 +710,7 @@ func (f *Fory) SerializeWithCallback(buffer *ByteBuffer, v interface{}, callback } // SerializeWithCallback the value - f.writeCtx.WriteValue(reflect.ValueOf(v)) + f.writeCtx.WriteValue(reflect.ValueOf(v), RefModeTracking, true) if f.writeCtx.HasError() { return f.writeCtx.TakeError() } @@ -798,7 +798,7 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v interface{}, return fmt.Errorf("v must be a non-nil pointer") } // DeserializeWithCallbackBuffers directly into v - f.readCtx.ReadValue(rv.Elem()) + f.readCtx.ReadValue(rv.Elem(), RefModeTracking, true) if f.readCtx.HasError() { return f.readCtx.TakeError() } @@ -826,7 +826,7 @@ func (f *Fory) serializeReflectValue(value reflect.Value) ([]byte, error) { } // SerializeWithCallback the value - f.writeCtx.WriteValue(value) + f.writeCtx.WriteValue(value, RefModeTracking, true) if f.writeCtx.HasError() { return nil, f.writeCtx.TakeError() } diff --git a/go/fory/map.go b/go/fory/map.go index f910146e6e..3ce612e4fc 100644 --- a/go/fory/map.go +++ b/go/fory/map.go @@ -165,8 +165,27 @@ func (s mapSerializer) WriteData(ctx *WriteContext, value reflect.Value) { } } } else { - buf.WriteInt8(VALUE_HAS_NULL | TRACKING_KEY_REF) - ctx.WriteValue(entryKey) + // Polymorphic key with null value + keyTypeInfo, typeErr := getActualTypeInfo(entryKey, typeResolver) + if typeErr != nil { + ctx.SetError(FromError(typeErr)) + return + } + // Only set TRACKING_KEY_REF if ref tracking is enabled AND type needs it + chunkHeader := int8(VALUE_HAS_NULL) + trackKeyRef := ctx.TrackRef() && keyTypeInfo.NeedWriteRef + if trackKeyRef { + chunkHeader |= TRACKING_KEY_REF + } + buf.WriteInt8(chunkHeader) + // Write type info + typeResolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) + // Write value with appropriate ref mode + keyRefMode := RefModeNone + if trackKeyRef { + keyRefMode = RefModeTracking + } + keyTypeInfo.Serializer.Write(ctx, keyRefMode, false, false, entryKey) if ctx.HasError() { return } @@ -189,8 +208,27 @@ func (s mapSerializer) WriteData(ctx *WriteContext, value reflect.Value) { } } } else { - buf.WriteInt8(KEY_HAS_NULL | TRACKING_VALUE_REF) - ctx.WriteValue(entryVal) + // Polymorphic value with null key + valueTypeInfo, typeErr := getActualTypeInfo(entryVal, typeResolver) + if typeErr != nil { + ctx.SetError(FromError(typeErr)) + return + } + // Only set TRACKING_VALUE_REF if ref tracking is enabled AND type needs it + chunkHeader := int8(KEY_HAS_NULL) + trackValueRef := ctx.TrackRef() && valueTypeInfo.NeedWriteRef + if trackValueRef { + chunkHeader |= TRACKING_VALUE_REF + } + buf.WriteInt8(chunkHeader) + // Write type info + typeResolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) + // Write value with appropriate ref mode + valueRefMode := RefModeNone + if trackValueRef { + valueRefMode = RefModeTracking + } + valueTypeInfo.Serializer.Write(ctx, valueRefMode, false, false, entryVal) if ctx.HasError() { return } @@ -241,20 +279,22 @@ func (s mapSerializer) WriteData(ctx *WriteContext, value reflect.Value) { valueWriteRef = valueTypeInfo.NeedWriteRef } - if keyWriteRef { + // Only set TRACKING_KEY_REF/TRACKING_VALUE_REF if ref tracking is enabled + trackRef := ctx.TrackRef() + if keyWriteRef && trackRef { chunkHeader |= TRACKING_KEY_REF } - if valueWriteRef { + if valueWriteRef && trackRef { chunkHeader |= TRACKING_VALUE_REF } buf.PutUint8(chunkSizeOffset-2, uint8(chunkHeader)) chunkSize := 0 keyRefMode := RefModeNone - if keyWriteRef { + if keyWriteRef && trackRef { keyRefMode = RefModeTracking } valueRefMode := RefModeNone - if valueWriteRef { + if valueWriteRef && trackRef { valueRefMode = RefModeTracking } for chunkSize < MAX_CHUNK_SIZE { diff --git a/go/fory/pointer.go b/go/fory/pointer.go index 003287c356..7c3ef54818 100644 --- a/go/fory/pointer.go +++ b/go/fory/pointer.go @@ -178,8 +178,8 @@ func (s *ptrToInterfaceSerializer) WriteData(ctx *WriteContext, value reflect.Va // Get the concrete element that the interface pointer points to elemValue := value.Elem() - // Use WriteValue to handle the polymorphic interface value - ctx.WriteValue(elemValue) + // Use WriteValue to handle the polymorphic interface value with ref tracking and type info + ctx.WriteValue(elemValue, RefModeTracking, true) } func (s *ptrToInterfaceSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { @@ -206,7 +206,7 @@ func (s *ptrToInterfaceSerializer) Write(ctx *WriteContext, refMode RefMode, wri } // For interface pointers, we don't write type info here - // WriteValue will handle the type info for the concrete value + // WriteData will call WriteValue which handles the type info for the concrete value s.WriteData(ctx, value) } @@ -214,8 +214,8 @@ func (s *ptrToInterfaceSerializer) ReadData(ctx *ReadContext, type_ reflect.Type // Create a new interface pointer newVal := reflect.New(type_.Elem()) - // Use ReadValue to handle the polymorphic interface value - ctx.ReadValue(newVal.Elem()) + // Use ReadValue to handle the polymorphic interface value with ref tracking and type info + ctx.ReadValue(newVal.Elem(), RefModeTracking, true) if ctx.HasError() { return } diff --git a/go/fory/reader.go b/go/fory/reader.go index 8bd05a883c..951aebe333 100644 --- a/go/fory/reader.go +++ b/go/fory/reader.go @@ -548,8 +548,11 @@ func (c *ReadContext) decDepth() { c.depth-- } -// ReadValue reads a polymorphic value - queries serializer by type and deserializes -func (c *ReadContext) ReadValue(value reflect.Value) { +// ReadValue reads a polymorphic value with configurable reference tracking and type info reading. +// Parameters: +// - refMode: controls reference tracking behavior (RefModeNone, RefModeTracking, RefModeNullOnly) +// - readType: if true, reads type info from the buffer +func (c *ReadContext) ReadValue(value reflect.Value, refMode RefMode, readType bool) { if !value.IsValid() { c.SetError(DeserializationError("invalid reflect.Value")) return @@ -557,28 +560,41 @@ func (c *ReadContext) ReadValue(value reflect.Value) { // Handle array targets (arrays are serialized as slices) if value.Type().Kind() == reflect.Array { - c.readArrayValue(value) + c.readArrayValueWithMode(value, refMode, readType) return } // For interface{} types, we need to read the actual type from the buffer first if value.Type().Kind() == reflect.Interface { - // Read ref flag - refID, err := c.RefResolver().TryPreserveRefId(c.buffer) - if err != nil { - c.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found - obj := c.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) + // Handle ref tracking based on refMode + var refID int32 = int32(NotNullValueFlag) + if refMode == RefModeTracking { + var err error + refID, err = c.RefResolver().TryPreserveRefId(c.buffer) + if err != nil { + c.SetError(FromError(err)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found + obj := c.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return + } + } else if refMode == RefModeNullOnly { + flag := c.buffer.ReadInt8(c.Err()) + if flag == NullFlag { + return } - return } // Read type info to determine the actual type + if !readType { + c.SetError(DeserializationError("cannot read interface{} without type info")) + return + } ctxErr := c.Err() typeInfo := c.typeResolver.ReadTypeInfo(c.buffer, ctxErr) if ctxErr.HasError() { @@ -625,7 +641,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { // For named structs, register the pointer BEFORE reading data // This is critical for circular references to work correctly - if isNamedStruct && refID >= int32(NotNullValueFlag) { + if isNamedStruct && refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, newValue) } @@ -643,7 +659,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { } // Register reference after reading data for non-struct types - if !isNamedStruct && refID >= int32(NotNullValueFlag) { + if !isNamedStruct && refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, newValue) } @@ -652,15 +668,17 @@ func (c *ReadContext) ReadValue(value reflect.Value) { return } - // For struct types, use optimized ReadStruct path + // For struct types, use optimized ReadStruct path when using full ref tracking and type info valueType := value.Type() - if valueType.Kind() == reflect.Struct { - c.ReadStruct(value) - return - } - if valueType.Kind() == reflect.Ptr && valueType.Elem().Kind() == reflect.Struct { - c.ReadStruct(value) - return + if refMode == RefModeTracking && readType { + if valueType.Kind() == reflect.Struct { + c.ReadStruct(value) + return + } + if valueType.Kind() == reflect.Ptr && valueType.Elem().Kind() == reflect.Struct { + c.ReadStruct(value) + return + } } // Get serializer for the value's type @@ -671,7 +689,7 @@ func (c *ReadContext) ReadValue(value reflect.Value) { } // Read handles ref tracking and type info internally - serializer.Read(c, RefModeTracking, true, false, value) + serializer.Read(c, refMode, readType, false, value) } // ReadStruct reads a struct value with optimized type resolution. @@ -765,23 +783,40 @@ func (c *ReadContext) ReadInto(value reflect.Value, serializer Serializer, refMo // readArrayValue handles array targets when stream contains slice data // Arrays are serialized as slices in xlang protocol func (c *ReadContext) readArrayValue(target reflect.Value) { - // Read ref flag - refID, err := c.RefResolver().TryPreserveRefId(c.buffer) - if err != nil { - c.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := c.RefResolver().GetReadObject(refID) - if obj.IsValid() { - reflect.Copy(target, obj) + c.readArrayValueWithMode(target, RefModeTracking, true) +} + +// readArrayValueWithMode handles array targets with configurable ref mode and type reading +func (c *ReadContext) readArrayValueWithMode(target reflect.Value, refMode RefMode, readType bool) { + var refID int32 = int32(NotNullValueFlag) + + // Handle ref tracking based on refMode + if refMode == RefModeTracking { + var err error + refID, err = c.RefResolver().TryPreserveRefId(c.buffer) + if err != nil { + c.SetError(FromError(err)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference to existing object + obj := c.RefResolver().GetReadObject(refID) + if obj.IsValid() { + reflect.Copy(target, obj) + } + return + } + } else if refMode == RefModeNullOnly { + flag := c.buffer.ReadInt8(c.Err()) + if flag == NullFlag { + return } - return } - // Read type ID (will be slice type in stream) - c.buffer.ReadVaruint32Small7(c.Err()) + // Read type ID if requested (will be slice type in stream) + if readType { + c.buffer.ReadVaruint32Small7(c.Err()) + } // Get slice serializer to read the data sliceType := reflect.SliceOf(target.Type().Elem()) @@ -812,7 +847,7 @@ func (c *ReadContext) readArrayValue(target reflect.Value) { reflect.Copy(target, tempSlice) // Register for circular refs - if refID >= int32(NotNullValueFlag) { + if refMode == RefModeTracking && refID >= int32(NotNullValueFlag) { c.RefResolver().SetReadObject(refID, target) } } diff --git a/go/fory/struct.go b/go/fory/struct.go index 66add6b587..02d0307848 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -570,7 +570,7 @@ func (s *structSerializer) writeRemainingField(ctx *WriteContext, ptr unsafe.Poi if field.Serializer != nil { field.Serializer.Write(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) } else { - ctx.WriteValue(fieldValue) + ctx.WriteValue(fieldValue, RefModeTracking, true) } } @@ -894,7 +894,7 @@ func (s *structSerializer) readRemainingField(ctx *ReadContext, ptr unsafe.Point if field.Serializer != nil { field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) } else { - ctx.ReadValue(fieldValue) + ctx.ReadValue(fieldValue, RefModeTracking, true) } } @@ -992,7 +992,7 @@ func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Val // Use pre-computed RefMode and WriteType from field initialization field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) } else { - ctx.ReadValue(fieldValue) + ctx.ReadValue(fieldValue, RefModeTracking, true) } } } @@ -1018,7 +1018,7 @@ func (s *structSerializer) skipField(ctx *ReadContext, field *FieldInfo) { } field.Serializer.Read(ctx, refMode, readType, false, tempValue) } else { - ctx.ReadValue(tempValue) + ctx.ReadValue(tempValue, RefModeTracking, true) } } diff --git a/go/fory/tests/structs_fory_gen.go b/go/fory/tests/structs_fory_gen.go index a636de57bc..e3538214d9 100644 --- a/go/fory/tests/structs_fory_gen.go +++ b/go/fory/tests/structs_fory_gen.go @@ -85,7 +85,7 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, buf.WriteInt8(1) // CollectionTrackingRef only // WriteData each element using WriteValue for _, elem := range v.DynamicSlice { - ctx.WriteValue(reflect.ValueOf(elem)) + ctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true) } } } else { @@ -102,7 +102,7 @@ func (g *DynamicSliceDemo_ForyGenSerializer) WriteTyped(ctx *fory.WriteContext, buf.WriteInt8(1) // CollectionTrackingRef only // WriteData each element using WriteValue for _, elem := range v.DynamicSlice { - ctx.WriteValue(reflect.ValueOf(elem)) + ctx.WriteValue(reflect.ValueOf(elem), fory.RefModeTracking, true) } } } @@ -190,7 +190,7 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v v.DynamicSlice = make([]interface{}, sliceLen) // ReadData each element using ReadValue for i := range v.DynamicSlice { - ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem()) + ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem(), fory.RefModeTracking, true) } } } else { @@ -209,7 +209,7 @@ func (g *DynamicSliceDemo_ForyGenSerializer) ReadTyped(ctx *fory.ReadContext, v v.DynamicSlice = make([]interface{}, sliceLen) // ReadData each element using ReadValue for i := range v.DynamicSlice { - ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem()) + ctx.ReadValue(reflect.ValueOf(&v.DynamicSlice[i]).Elem(), fory.RefModeTracking, true) } } } diff --git a/go/fory/writer.go b/go/fory/writer.go index 08a7bfc496..b518d177fd 100644 --- a/go/fory/writer.go +++ b/go/fory/writer.go @@ -559,10 +559,12 @@ func (c *WriteContext) WriteBufferObject(bufferObject BufferObject) { // If out-of-band, we just write false (already done above) and the data is handled externally } -// WriteValue writes a polymorphic value with reference tracking and type info. +// WriteValue writes a polymorphic value with configurable reference tracking and type info. // This is used when the concrete type is not known at compile time. -// Each serializer's Write method handles reference tracking internally. -func (c *WriteContext) WriteValue(value reflect.Value) { +// Parameters: +// - refMode: controls reference tracking behavior (RefModeNone, RefModeTracking, RefModeNullOnly) +// - writeType: if true, writes type info before the value +func (c *WriteContext) WriteValue(value reflect.Value, refMode RefMode, writeType bool) { // Handle interface values by getting their concrete element if value.Kind() == reflect.Interface { if !value.IsValid() || value.IsNil() { @@ -604,5 +606,5 @@ func (c *WriteContext) WriteValue(value reflect.Value) { } // Use serializer's Write method which handles ref tracking and type info internally - typeInfo.Serializer.Write(c, RefModeTracking, true, false, value) + typeInfo.Serializer.Write(c, refMode, writeType, false, value) } From b92060d7f8a3d2e5d12f6e38d8891dd5efec8148 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 18:58:21 +0800 Subject: [PATCH 30/35] support mono for not pure virtual type field --- cpp/fory/meta/field.h | 62 +++++- .../smart_ptr_serializer_test.cc | 72 ++++++ .../serialization/smart_ptr_serializers.h | 206 +++++++++++------- cpp/fory/serialization/struct_serializer.h | 48 +++- cpp/fory/serialization/type_resolver.cc | 4 +- 5 files changed, 300 insertions(+), 92 deletions(-) diff --git a/cpp/fory/meta/field.h b/cpp/fory/meta/field.h index a01bb8c646..7ec8835208 100644 --- a/cpp/fory/meta/field.h +++ b/cpp/fory/meta/field.h @@ -50,6 +50,12 @@ struct not_null {}; /// tracking). struct ref {}; +/// Tag to mark a polymorphic shared_ptr/unique_ptr field as monomorphic. +/// Use this when the field type has virtual methods but you know the actual +/// runtime type will always be exactly T (not a derived type). +/// This avoids dynamic type dispatch overhead during serialization. +struct monomorphic {}; + namespace detail { // ============================================================================ @@ -96,10 +102,12 @@ inline constexpr bool has_option_v = (std::is_same_v || ...); // ============================================================================ /// Compile-time field tag metadata entry -template struct FieldTagEntry { +template +struct FieldTagEntry { static constexpr int16_t id = Id; static constexpr bool is_nullable = Nullable; static constexpr bool track_ref = Ref; + static constexpr bool is_monomorphic = Monomorphic; }; /// Default: no field tags defined for type T @@ -163,6 +171,11 @@ template class field { "fory::ref is only valid for shared_ptr " "(reference tracking requires shared ownership)."); + // Validate: monomorphic only for smart pointers + static_assert(!detail::has_option_v || + detail::is_smart_ptr_v, + "fory::monomorphic is only valid for shared_ptr/unique_ptr."); + // Validate: no options for optional (inherently nullable) static_assert(!detail::is_optional_v || sizeof...(Options) == 0, "std::optional is inherently nullable. No options allowed."); @@ -189,6 +202,13 @@ template class field { static constexpr bool track_ref = detail::is_shared_ptr_v && detail::has_option_v; + /// Monomorphic serialization is enabled if: + /// - It's std::shared_ptr or std::unique_ptr with fory::monomorphic option + /// - When true, the serializer will not use dynamic type dispatch + static constexpr bool is_monomorphic = + detail::is_smart_ptr_v && + detail::has_option_v; + T value{}; // Default constructor @@ -309,6 +329,19 @@ struct field_track_ref> { template inline constexpr bool field_track_ref_v = field_track_ref::value; +/// Get is_monomorphic from field type +template struct field_is_monomorphic { + static constexpr bool value = false; +}; + +template +struct field_is_monomorphic> { + static constexpr bool value = field::is_monomorphic; +}; + +template +inline constexpr bool field_is_monomorphic_v = field_is_monomorphic::value; + // ============================================================================ // FORY_FIELD_TAGS Macro Support // ============================================================================ @@ -317,7 +350,7 @@ namespace detail { // Helper to parse field tag entry from macro arguments // Supports: (field, id), (field, id, nullable), (field, id, ref), -// (field, id, nullable, ref) +// (field, id, nullable, ref), (field, id, monomorphic), etc. template struct ParseFieldTagEntry { static constexpr bool is_nullable = @@ -327,6 +360,9 @@ struct ParseFieldTagEntry { static constexpr bool track_ref = is_shared_ptr_v && has_option_v; + static constexpr bool is_monomorphic = + is_smart_ptr_v && has_option_v; + // Compile-time validation static_assert(!has_option_v || is_smart_ptr_v, @@ -335,7 +371,11 @@ struct ParseFieldTagEntry { static_assert(!has_option_v || is_shared_ptr_v, "fory::ref is only valid for shared_ptr"); - using type = FieldTagEntry; + static_assert(!has_option_v || + is_smart_ptr_v, + "fory::monomorphic is only valid for shared_ptr/unique_ptr"); + + using type = FieldTagEntry; }; /// Get field tag entry by index from ForyFieldTagsImpl @@ -343,6 +383,7 @@ template struct GetFieldTagEntry { static constexpr int16_t id = -1; static constexpr bool is_nullable = false; static constexpr bool track_ref = false; + static constexpr bool is_monomorphic = false; }; template @@ -355,6 +396,7 @@ struct GetFieldTagEntry< static constexpr int16_t id = Entry::id; static constexpr bool is_nullable = Entry::is_nullable; static constexpr bool track_ref = Entry::track_ref; + static constexpr bool is_monomorphic = Entry::is_monomorphic; }; } // namespace detail @@ -377,12 +419,14 @@ struct GetFieldTagEntry< #define FORY_FT_GET_OPT1_IMPL(f, i, o1, ...) o1 #define FORY_FT_GET_OPT2(tuple) FORY_FT_GET_OPT2_IMPL tuple #define FORY_FT_GET_OPT2_IMPL(f, i, o1, o2, ...) o2 +#define FORY_FT_GET_OPT3(tuple) FORY_FT_GET_OPT3_IMPL tuple +#define FORY_FT_GET_OPT3_IMPL(f, i, o1, o2, o3, ...) o3 -// Detect number of elements in tuple: 2, 3, or 4 +// Detect number of elements in tuple: 2, 3, 4, or 5 #define FORY_FT_TUPLE_SIZE(tuple) FORY_FT_TUPLE_SIZE_IMPL tuple #define FORY_FT_TUPLE_SIZE_IMPL(...) \ - FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 4, 3, 2, 1, 0) -#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, N, ...) N + FORY_FT_TUPLE_SIZE_SELECT(__VA_ARGS__, 5, 4, 3, 2, 1, 0) +#define FORY_FT_TUPLE_SIZE_SELECT(_1, _2, _3, _4, _5, N, ...) N // Create FieldTagEntry based on tuple size using indirect call pattern // This pattern ensures the concatenated macro name is properly rescanned @@ -408,6 +452,12 @@ struct GetFieldTagEntry< decltype(std::declval().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple), \ ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple)>::type +#define FORY_FT_MAKE_ENTRY_5(Type, tuple) \ + typename ::fory::detail::ParseFieldTagEntry< \ + decltype(std::declval().FORY_FT_FIELD(tuple)), FORY_FT_ID(tuple), \ + ::fory::FORY_FT_GET_OPT1(tuple), ::fory::FORY_FT_GET_OPT2(tuple), \ + ::fory::FORY_FT_GET_OPT3(tuple)>::type + // Main macro: FORY_FIELD_TAGS(Type, (field1, id1), (field2, id2, nullable),...) // Note: Uses fory::detail:: instead of ::fory::detail:: for GCC compatibility #define FORY_FIELD_TAGS(Type, ...) \ diff --git a/cpp/fory/serialization/smart_ptr_serializer_test.cc b/cpp/fory/serialization/smart_ptr_serializer_test.cc index 7cea160da8..63f20ff335 100644 --- a/cpp/fory/serialization/smart_ptr_serializer_test.cc +++ b/cpp/fory/serialization/smart_ptr_serializer_test.cc @@ -441,5 +441,77 @@ TEST(SmartPtrSerializerTest, MaxDynDepthDefault) { } } // namespace + +// ============================================================================ +// Monomorphic field tests (fory::field<> style) +// ============================================================================ +namespace { + +// A polymorphic base class (has virtual methods) +struct PolymorphicBaseForMono { + virtual ~PolymorphicBaseForMono() = default; + virtual std::string name() const { return "PolymorphicBaseForMono"; } + int32_t value = 0; + std::string data; +}; +FORY_STRUCT(PolymorphicBaseForMono, value, data); + +// Holder with monomorphic field using fory::field<> +struct MonomorphicFieldHolder { + // Field marked as monomorphic - no dynamic type dispatch, always + // PolymorphicBaseForMono + fory::field, 0, fory::nullable, + fory::monomorphic> + ptr; +}; +FORY_STRUCT(MonomorphicFieldHolder, ptr); + +TEST(SmartPtrSerializerTest, MonomorphicFieldWithForyField) { + MonomorphicFieldHolder original; + original.ptr.value = std::make_shared(); + original.ptr.value->value = 42; + original.ptr.value->data = "test data"; + + auto fory = Fory::builder().track_ref(false).build(); + fory.register_struct(400); + fory.register_struct(401); + + auto bytes_result = fory.serialize(original); + ASSERT_TRUE(bytes_result.ok()) << bytes_result.error().to_string(); + + auto deserialize_result = fory.deserialize( + bytes_result->data(), bytes_result->size()); + ASSERT_TRUE(deserialize_result.ok()) + << deserialize_result.error().to_string(); + + auto deserialized = std::move(deserialize_result).value(); + ASSERT_TRUE(deserialized.ptr.value); + EXPECT_EQ(deserialized.ptr.value->value, 42); + EXPECT_EQ(deserialized.ptr.value->data, "test data"); + EXPECT_EQ(deserialized.ptr.value->name(), "PolymorphicBaseForMono"); +} + +TEST(SmartPtrSerializerTest, MonomorphicFieldNullValue) { + MonomorphicFieldHolder original; + original.ptr.value = nullptr; + + auto fory = Fory::builder().track_ref(false).build(); + fory.register_struct(404); + fory.register_struct(405); + + auto bytes_result = fory.serialize(original); + ASSERT_TRUE(bytes_result.ok()) << bytes_result.error().to_string(); + + auto deserialize_result = fory.deserialize( + bytes_result->data(), bytes_result->size()); + ASSERT_TRUE(deserialize_result.ok()) + << deserialize_result.error().to_string(); + + auto deserialized = std::move(deserialize_result).value(); + EXPECT_FALSE(deserialized.ptr.value); +} + +} // namespace + } // namespace serialization } // namespace fory \ No newline at end of file diff --git a/cpp/fory/serialization/smart_ptr_serializers.h b/cpp/fory/serialization/smart_ptr_serializers.h index c739e016d3..0ca9cfc26b 100644 --- a/cpp/fory/serialization/smart_ptr_serializers.h +++ b/cpp/fory/serialization/smart_ptr_serializers.h @@ -427,20 +427,30 @@ template struct Serializer> { // Handle ref_mode == RefMode::None case (similar to Rust) if (ref_mode == RefMode::None) { if constexpr (is_polymorphic) { - // For polymorphic types, we must read type info when read_type=true - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::shared_ptr " - "without type info (read_type=false)")); - return nullptr; - } - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; + if (read_type) { + // Polymorphic path: read type info from stream to get concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } + // Now use read_with_type_info with the concrete type info + return read_with_type_info(ctx, ref_mode, *type_info); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_shared(std::move(value)); + } } - // Now use read_with_type_info with the concrete type info - return read_with_type_info(ctx, ref_mode, *type_info); } else { // T is guaranteed to be a value type (not pointer or nullable wrapper) // by static_assert, so no inner ref metadata needed. @@ -498,38 +508,53 @@ template struct Serializer> { // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::shared_ptr " - "without type info (read_type=false)")); - return nullptr; - } - - // Check and increase dynamic depth for polymorphic deserialization - auto depth_res = ctx.increase_dyn_depth(); - if (!depth_res.ok()) { - ctx.set_error(std::move(depth_res).error()); - return nullptr; - } - DynDepthGuard dyn_depth_guard(ctx); + if (read_type) { + // Polymorphic path: read type info and use harness for deserialization + // Check and increase dynamic depth for polymorphic deserialization + auto depth_res = ctx.increase_dyn_depth(); + if (!depth_res.ok()) { + ctx.set_error(std::move(depth_res).error()); + return nullptr; + } + DynDepthGuard dyn_depth_guard(ctx); - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; - } + // Read type info from stream to get the concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } - // Use the harness to deserialize the concrete type - void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return nullptr; - } - T *obj_ptr = static_cast(raw_ptr); - auto result = std::shared_ptr(obj_ptr); - if (is_first_occurrence) { - ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + // Use the harness to deserialize the concrete type + void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return nullptr; + } + T *obj_ptr = static_cast(raw_ptr); + auto result = std::shared_ptr(obj_ptr); + if (is_first_occurrence) { + ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + } + return result; + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + auto result = std::make_shared(std::move(value)); + if (is_first_occurrence) { + ctx.ref_reader().store_shared_ref_at(reserved_ref_id, result); + } + return result; + } } - return result; } else { // Non-polymorphic path: T is guaranteed to be a value type (not pointer // or nullable wrapper) by static_assert, so no inner ref metadata needed. @@ -818,20 +843,30 @@ template struct Serializer> { // Handle ref_mode == RefMode::None case (similar to Rust) if (ref_mode == RefMode::None) { if constexpr (is_polymorphic) { - // For polymorphic types, we must read type info when read_type=true - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::unique_ptr " - "without type info (read_type=false)")); - return nullptr; - } - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; + if (read_type) { + // Polymorphic path: read type info from stream to get concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } + // Now use read_with_type_info with the concrete type info + return read_with_type_info(ctx, ref_mode, *type_info); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_unique(std::move(value)); + } } - // Now use read_with_type_info with the concrete type info - return read_with_type_info(ctx, ref_mode, *type_info); } else { // T is guaranteed to be a value type by static_assert. T value = Serializer::read(ctx, RefMode::None, read_type); @@ -859,34 +894,45 @@ template struct Serializer> { // For polymorphic types, read type info AFTER handling ref flags if constexpr (is_polymorphic) { - if (!read_type) { - ctx.set_error(Error::type_error( - "Cannot deserialize polymorphic std::unique_ptr " - "without type info (read_type=false)")); - return nullptr; - } - - // Check and increase dynamic depth for polymorphic deserialization - auto depth_res = ctx.increase_dyn_depth(); - if (!depth_res.ok()) { - ctx.set_error(std::move(depth_res).error()); - return nullptr; - } - DynDepthGuard dyn_depth_guard(ctx); + if (read_type) { + // Polymorphic path: read type info and use harness for deserialization + // Check and increase dynamic depth for polymorphic deserialization + auto depth_res = ctx.increase_dyn_depth(); + if (!depth_res.ok()) { + ctx.set_error(std::move(depth_res).error()); + return nullptr; + } + DynDepthGuard dyn_depth_guard(ctx); - // Read type info from stream to get the concrete type - const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); - if (ctx.has_error()) { - return nullptr; - } + // Read type info from stream to get the concrete type + const TypeInfo *type_info = ctx.read_any_typeinfo(ctx.error()); + if (ctx.has_error()) { + return nullptr; + } - // Use the harness to deserialize the concrete type - void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); - if (FORY_PREDICT_FALSE(ctx.has_error())) { - return nullptr; + // Use the harness to deserialize the concrete type + void *raw_ptr = read_polymorphic_harness_data(ctx, type_info); + if (FORY_PREDICT_FALSE(ctx.has_error())) { + return nullptr; + } + T *obj_ptr = static_cast(raw_ptr); + return std::unique_ptr(obj_ptr); + } else { + // Monomorphic path: read_type=false means field is marked monomorphic + // Use Serializer::read directly without dynamic type dispatch + // Note: abstract types cannot use monomorphic path + if constexpr (std::is_abstract_v) { + ctx.set_error(Error::unsupported( + "Cannot use monomorphic deserialization for abstract type")); + return nullptr; + } else { + T value = Serializer::read(ctx, RefMode::None, false); + if (ctx.has_error()) { + return nullptr; + } + return std::make_unique(std::move(value)); + } } - T *obj_ptr = static_cast(raw_ptr); - return std::unique_ptr(obj_ptr); } else { // T is guaranteed to be a value type by static_assert. T value = Serializer::read(ctx, RefMode::None, read_type); diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 31d9edaf9a..8cf43c099f 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -358,6 +358,31 @@ template struct CompileTimeFieldHelpers { } } + /// Returns true if the field at Index is marked as monomorphic. + /// Use this for shared_ptr/unique_ptr fields with polymorphic inner types + /// when you know the actual runtime type will always be exactly T. + template static constexpr bool field_monomorphic() { + if constexpr (FieldCount == 0) { + return false; + } else { + using PtrT = std::tuple_element_t; + using RawFieldType = meta::RemoveMemberPointerCVRefT; + + // If it's a fory::field<> wrapper, use its is_monomorphic metadata + if constexpr (is_fory_field_v) { + return RawFieldType::is_monomorphic; + } + // Else if FORY_FIELD_TAGS is defined, use that metadata + else if constexpr (::fory::detail::has_field_tags_v) { + return ::fory::detail::GetFieldTagEntry::is_monomorphic; + } + // Default: not monomorphic (polymorphic types use dynamic dispatch) + else { + return false; + } + } + } + /// Get the underlying field type (unwraps fory::field<> if present) template struct UnwrappedFieldTypeHelper { using PtrT = std::tuple_element_t; @@ -1300,10 +1325,13 @@ void write_single_field(const T &obj, WriteContext &ctx, field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; constexpr bool is_polymorphic = field_type_id == TypeId::UNKNOWN; + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + // Per C++ read logic: struct fields need type info only in compatible mode - // Polymorphic types always need type info - bool write_type = - is_polymorphic || ((is_struct || is_ext) && ctx.is_compatible()); + // Polymorphic types always need type info, UNLESS marked as monomorphic + bool write_type = (is_polymorphic && !is_monomorphic) || + ((is_struct || is_ext) && ctx.is_compatible()); Serializer::write(field_value, ctx, field_ref_mode, write_type); } @@ -1466,7 +1494,12 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { constexpr bool is_ext_field = field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; - bool read_type = is_polymorphic_field; + + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + + // Polymorphic types need type info, UNLESS marked as monomorphic + bool read_type = is_polymorphic_field && !is_monomorphic; // Get field metadata from fory::field<> or FORY_FIELD_TAGS or defaults constexpr bool is_nullable = Helpers::template field_nullable(); @@ -1533,6 +1566,7 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { template void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, RefMode remote_ref_mode) { + using Helpers = CompileTimeFieldHelpers; const auto field_info = ForyFieldInfo(obj); const auto field_ptrs = decltype(field_info)::Ptrs; const auto field_ptr = std::get(field_ptrs); @@ -1553,7 +1587,11 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); - bool read_type = is_polymorphic_field; + // Check if field is marked as monomorphic (skip dynamic type dispatch) + constexpr bool is_monomorphic = Helpers::template field_monomorphic(); + + // Polymorphic types need type info, UNLESS marked as monomorphic + bool read_type = is_polymorphic_field && !is_monomorphic; // In compatible mode, nested struct fields always carry type metadata // (xtypeId + meta index). We must read this metadata so that diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index 319eadad26..c5045cbc70 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1007,11 +1007,13 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { // Use the low 64 bits and then keep low 32 bits as i32. uint64_t low = static_cast(hash_out[0]); uint32_t version = static_cast(low & 0xFFFF'FFFFu); +#ifdef FORY_DEBUG // DEBUG: Print fingerprint for debugging version mismatch std::cerr << "[xlang][debug] struct_version type_name=" << meta.type_name << ", fingerprint=\"" << fingerprint << "\" version=" << static_cast(version) << std::endl; - return static_cast(version); +#endif + return static_cast(version); } // ============================================================================ From 4012a9b9702a644cc7fe14fe469798984ce0b49b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 19:00:15 +0800 Subject: [PATCH 31/35] simplify is_struct_type --- cpp/fory/serialization/struct_serializer.h | 26 +++++----------------- cpp/fory/type/type.h | 15 +++++++++++++ 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/cpp/fory/serialization/struct_serializer.h b/cpp/fory/serialization/struct_serializer.h index 8cf43c099f..cd27a5cffb 100644 --- a/cpp/fory/serialization/struct_serializer.h +++ b/cpp/fory/serialization/struct_serializer.h @@ -1317,12 +1317,8 @@ void write_single_field(const T &obj, WriteContext &ctx, // Enums: false (per Rust util.rs:58-59) // Structs/EXT: true ONLY in compatible mode (per C++ read logic) // Others: false - constexpr bool is_struct = field_type_id == TypeId::STRUCT || - field_type_id == TypeId::COMPATIBLE_STRUCT || - field_type_id == TypeId::NAMED_STRUCT || - field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; - constexpr bool is_ext = - field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_struct = is_struct_type(field_type_id); + constexpr bool is_ext = is_ext_type(field_type_id); constexpr bool is_polymorphic = field_type_id == TypeId::UNKNOWN; // Check if field is marked as monomorphic (skip dynamic type dispatch) @@ -1486,13 +1482,8 @@ void read_single_field_by_index(T &obj, ReadContext &ctx) { constexpr bool field_requires_ref = requires_ref_metadata_v; constexpr TypeId field_type_id = Serializer::type_id; // Check if field is a struct type - use type_id to handle shared_ptr - constexpr bool is_struct_field = - field_type_id == TypeId::STRUCT || - field_type_id == TypeId::COMPATIBLE_STRUCT || - field_type_id == TypeId::NAMED_STRUCT || - field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; - constexpr bool is_ext_field = - field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_struct_field = is_struct_type(field_type_id); + constexpr bool is_ext_field = is_ext_type(field_type_id); constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; // Check if field is marked as monomorphic (skip dynamic type dispatch) @@ -1577,13 +1568,8 @@ void read_single_field_by_index_compatible(T &obj, ReadContext &ctx, constexpr TypeId field_type_id = Serializer::type_id; // Check if field is a struct type - use type_id to handle shared_ptr - constexpr bool is_struct_field = - field_type_id == TypeId::STRUCT || - field_type_id == TypeId::COMPATIBLE_STRUCT || - field_type_id == TypeId::NAMED_STRUCT || - field_type_id == TypeId::NAMED_COMPATIBLE_STRUCT; - constexpr bool is_ext_field = - field_type_id == TypeId::EXT || field_type_id == TypeId::NAMED_EXT; + constexpr bool is_struct_field = is_struct_type(field_type_id); + constexpr bool is_ext_field = is_ext_type(field_type_id); constexpr bool is_polymorphic_field = field_type_id == TypeId::UNKNOWN; constexpr bool is_primitive_field = is_primitive_type_id(field_type_id); diff --git a/cpp/fory/type/type.h b/cpp/fory/type/type.h index 75c622f842..540a6ba366 100644 --- a/cpp/fory/type/type.h +++ b/cpp/fory/type/type.h @@ -175,6 +175,21 @@ inline bool IsTypeShareMeta(int32_t type_id) { } } +/// Check if type_id represents a struct type. +/// Struct types include STRUCT, COMPATIBLE_STRUCT, NAMED_STRUCT, and +/// NAMED_COMPATIBLE_STRUCT. +inline constexpr bool is_struct_type(TypeId type_id) { + return type_id == TypeId::STRUCT || type_id == TypeId::COMPATIBLE_STRUCT || + type_id == TypeId::NAMED_STRUCT || + type_id == TypeId::NAMED_COMPATIBLE_STRUCT; +} + +/// Check if type_id represents an ext type. +/// Ext types include EXT and NAMED_EXT. +inline constexpr bool is_ext_type(TypeId type_id) { + return type_id == TypeId::EXT || type_id == TypeId::NAMED_EXT; +} + /// Check if type_id represents an internal (built-in) type. /// Internal types are all types except user-defined types (ENUM, STRUCT, EXT). /// UNKNOWN is excluded because it's a marker for polymorphic types, not a From 3180182c9d71b4718b6743ba798dcaf09f3aaa75 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 19:19:27 +0800 Subject: [PATCH 32/35] refine go track ref --- go/fory/fory.go | 2 ++ go/fory/reader.go | 19 ++++++++++--------- go/fory/ref_resolver.go | 27 +++++++++++++++++++++++++++ go/fory/slice.go | 4 ++-- go/fory/type_resolver.go | 26 ++++++++------------------ go/fory/writer.go | 6 ++++++ 6 files changed, 55 insertions(+), 29 deletions(-) diff --git a/go/fory/fory.go b/go/fory/fory.go index 244fbd45ca..a6948c84df 100644 --- a/go/fory/fory.go +++ b/go/fory/fory.go @@ -169,11 +169,13 @@ func New(opts ...Option) *Fory { f.writeCtx.typeResolver = f.typeResolver f.writeCtx.refResolver = f.refResolver f.writeCtx.compatible = f.config.Compatible + f.writeCtx.xlang = f.config.IsXlang f.readCtx = NewReadContext(f.config.TrackRef) f.readCtx.typeResolver = f.typeResolver f.readCtx.refResolver = f.refResolver f.readCtx.compatible = f.config.Compatible + f.readCtx.xlang = f.config.IsXlang return f } diff --git a/go/fory/reader.go b/go/fory/reader.go index 951aebe333..2248947510 100644 --- a/go/fory/reader.go +++ b/go/fory/reader.go @@ -32,6 +32,7 @@ type ReadContext struct { buffer *ByteBuffer refReader *RefReader trackRef bool // Cached flag to avoid indirection + xlang bool // Cross-language serialization mode compatible bool // Schema evolution compatibility mode typeResolver *TypeResolver // For complex type deserialization refResolver *RefResolver // For reference tracking (legacy) @@ -42,6 +43,11 @@ type ReadContext struct { err Error // Accumulated error state for deferred checking } +// IsXlang returns whether cross-language serialization mode is enabled +func (c *ReadContext) IsXlang() bool { + return c.xlang +} + // NewReadContext creates a new read context func NewReadContext(trackRef bool) *ReadContext { return &ReadContext{ @@ -560,7 +566,7 @@ func (c *ReadContext) ReadValue(value reflect.Value, refMode RefMode, readType b // Handle array targets (arrays are serialized as slices) if value.Type().Kind() == reflect.Array { - c.readArrayValueWithMode(value, refMode, readType) + c.ReadArrayValue(value, refMode, readType) return } @@ -780,14 +786,9 @@ func (c *ReadContext) ReadInto(value reflect.Value, serializer Serializer, refMo serializer.Read(c, refMode, readTypeInfo, false, value) } -// readArrayValue handles array targets when stream contains slice data -// Arrays are serialized as slices in xlang protocol -func (c *ReadContext) readArrayValue(target reflect.Value) { - c.readArrayValueWithMode(target, RefModeTracking, true) -} - -// readArrayValueWithMode handles array targets with configurable ref mode and type reading -func (c *ReadContext) readArrayValueWithMode(target reflect.Value, refMode RefMode, readType bool) { +// ReadArrayValue handles array targets with configurable ref mode and type reading. +// Arrays are serialized as slices in xlang protocol. +func (c *ReadContext) ReadArrayValue(target reflect.Value, refMode RefMode, readType bool) { var refID int32 = int32(NotNullValueFlag) // Handle ref tracking based on refMode diff --git a/go/fory/ref_resolver.go b/go/fory/ref_resolver.go index ba5f108a2b..fdc9ced81f 100644 --- a/go/fory/ref_resolver.go +++ b/go/fory/ref_resolver.go @@ -88,6 +88,33 @@ func isReferencable(t reflect.Type) bool { } } +// isRefType determines if a type should track references. +// For xlang mode, only pointer to struct/ext, interface, slice, and map track refs. +// Arrays do NOT track refs in xlang mode (they are value types). +// For native Go mode, uses standard Go reference semantics. +// +// Note: Struct fields can provide tags to override ref tracking behavior for specific +// field types. However, if the value is not a struct field (e.g., top-level value, +// collection element), then this function's result is always used. +func isRefType(t reflect.Type, xlang bool) bool { + kind := t.Kind() + if xlang { + switch kind { + case reflect.Ptr: + // Only pointer to struct tracks ref in xlang + elemKind := t.Elem().Kind() + return elemKind == reflect.Struct + case reflect.Interface, reflect.Slice, reflect.Map: + return true + default: + // Arrays and other types don't track refs in xlang + return false + } + } + // Native Go mode: pointers, maps, slices, and interfaces track refs + return isReferencable(t) +} + func newRefResolver(refTracking bool) *RefResolver { refResolver := &RefResolver{ refTracking: refTracking, diff --git a/go/fory/slice.go b/go/fory/slice.go index e085c6acb3..539f329b14 100644 --- a/go/fory/slice.go +++ b/go/fory/slice.go @@ -119,7 +119,7 @@ type sliceConcreteValueSerializer struct { // It returns an error if the element type is an interface, pointer to interface, or a primitive type. // Primitive numeric types (bool, int8, int16, int32, int64, uint8, float32, float64) must use // dedicated primitive slice serializers that use ARRAY protocol (binary size + binary). -func newSliceConcreteValueSerializer(type_ reflect.Type, elemSerializer Serializer) (*sliceConcreteValueSerializer, error) { +func newSliceConcreteValueSerializer(type_ reflect.Type, elemSerializer Serializer, xlang bool) (*sliceConcreteValueSerializer, error) { elem := type_.Elem() if elem.Kind() == reflect.Interface { return nil, fmt.Errorf("sliceConcreteValueSerializer does not support interface element type: %v", type_) @@ -136,7 +136,7 @@ func newSliceConcreteValueSerializer(type_ reflect.Type, elemSerializer Serializ return &sliceConcreteValueSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(elem), + referencable: isRefType(elem, xlang), }, nil } diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 721548c4bb..4dca046856 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -970,7 +970,7 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, serializer = &arrayConcreteValueSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(type_.Elem()), + referencable: isRefType(type_.Elem(), r.isXlang), } } } @@ -1353,7 +1353,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } // Always use xlang mode (LIST typeId) for non-primitive slices - return newSliceConcreteValueSerializer(type_, elemSerializer) + return newSliceConcreteValueSerializer(type_, elemSerializer, r.isXlang) } case reflect.Array: elem := type_.Elem() @@ -1393,7 +1393,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return &arrayConcreteValueSerializer{ type_: type_, elemSerializer: elemSerializer, - referencable: nullable(type_.Elem()), + referencable: isRefType(type_.Elem(), r.isXlang), }, nil } case reflect.Map: @@ -1417,19 +1417,9 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } } - // Determine key/value referencability - // In xlang mode, strings are value types and should not be reference-tracked - keyReferencable := nullable(type_.Key()) - valueReferencable := nullable(type_.Elem()) - if r.isXlang { - // In xlang mode, strings are value types (not reference-tracked) - if type_.Key().Kind() == reflect.String { - keyReferencable = false - } - if type_.Elem().Kind() == reflect.String { - valueReferencable = false - } - } + // Determine key/value referencability using isRefType which handles xlang mode + keyReferencable := isRefType(type_.Key(), r.isXlang) + valueReferencable := isRefType(type_.Elem(), r.isXlang) return &mapSerializer{ type_: type_, keySerializer: keySerializer, @@ -1504,7 +1494,7 @@ func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, e if err != nil { return nil, err } - return newSliceConcreteValueSerializer(sliceType, elemSerializer) + return newSliceConcreteValueSerializer(sliceType, elemSerializer, r.isXlang) } // GetSetSerializer returns the setSerializer for a map[T]bool type (used to represent sets in Go). @@ -1556,7 +1546,7 @@ func (r *TypeResolver) GetArraySerializer(arrayType reflect.Type) (Serializer, e if err != nil { return nil, err } - return newSliceConcreteValueSerializer(arrayType, elemSerializer) + return newSliceConcreteValueSerializer(arrayType, elemSerializer, r.isXlang) } func isDynamicType(type_ reflect.Type) bool { diff --git a/go/fory/writer.go b/go/fory/writer.go index b518d177fd..510f4351ad 100644 --- a/go/fory/writer.go +++ b/go/fory/writer.go @@ -33,6 +33,7 @@ type WriteContext struct { buffer *ByteBuffer refWriter *RefWriter trackRef bool // Cached flag to avoid indirection + xlang bool // Cross-language serialization mode compatible bool // Schema evolution compatibility mode depth int maxDepth int @@ -43,6 +44,11 @@ type WriteContext struct { err Error // Accumulated error state for deferred checking } +// IsXlang returns whether cross-language serialization mode is enabled +func (c *WriteContext) IsXlang() bool { + return c.xlang +} + // NewWriteContext creates a new write context func NewWriteContext(trackRef bool, maxDepth int) *WriteContext { return &WriteContext{ From 1f63e92ed9b9378e064a4d75b42b609cb2d975e0 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 19:45:12 +0800 Subject: [PATCH 33/35] refactor go map serialization --- go/fory/map.go | 1091 +++++++++++++++++--------------------- go/fory/ref_resolver.go | 7 - go/fory/struct.go | 11 - go/fory/type_def.go | 2 + go/fory/type_resolver.go | 10 +- 5 files changed, 491 insertions(+), 630 deletions(-) diff --git a/go/fory/map.go b/go/fory/map.go index 3ce612e4fc..6aabbe3729 100644 --- a/go/fory/map.go +++ b/go/fory/map.go @@ -22,6 +22,7 @@ import ( "reflect" ) +// Map chunk header flags const ( TRACKING_KEY_REF = 1 << 0 // 0b00000001 KEY_HAS_NULL = 1 << 1 // 0b00000010 @@ -32,694 +33,597 @@ const ( MAX_CHUNK_SIZE = 255 ) +// Combined header constants for null entry cases const ( - KV_NULL = KEY_HAS_NULL | VALUE_HAS_NULL // 0b00010010 - NULL_KEY_VALUE_DECL_TYPE = KEY_HAS_NULL | VALUE_DECL_TYPE // 0b00100010 - NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF = KEY_HAS_NULL | VALUE_DECL_TYPE | TRACKING_VALUE_REF // 0b00101010 - NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE // 0b00010100 - NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF // 0b00010101 + KV_NULL = KEY_HAS_NULL | VALUE_HAS_NULL + NULL_KEY_VALUE_DECL_TYPE = KEY_HAS_NULL | VALUE_DECL_TYPE + NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF = KEY_HAS_NULL | VALUE_DECL_TYPE | TRACKING_VALUE_REF + NULL_VALUE_KEY_DECL_TYPE = VALUE_HAS_NULL | KEY_DECL_TYPE + NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF = VALUE_HAS_NULL | KEY_DECL_TYPE | TRACKING_KEY_REF ) -// writeMapRefAndType handles reference and type writing for map serializers. -// Returns true if the value was already written (nil or ref), false if data should be written. -func writeMapRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, value reflect.Value) bool { - switch refMode { - case RefModeTracking: - if value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return true - } - refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) - if err != nil { - ctx.SetError(FromError(err)) - return false - } - if refWritten { - return true - } - case RefModeNullOnly: - if value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return true - } - ctx.buffer.WriteInt8(NotNullValueFlag) - } - if writeType { - ctx.buffer.WriteVaruint32Small7(uint32(MAP)) - } - return false -} - -// readMapRefAndType handles reference and type reading for map serializers. -// Returns true if a reference was resolved (value already set), false if data should be read. -func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value reflect.Value) bool { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return false - } - if refID < int32(NotNullValueFlag) { - obj := ctx.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) - } - return true - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return true - } - } - if readType { - buf.ReadVaruint32Small7(ctxErr) - } - return false -} - type mapSerializer struct { type_ reflect.Type keySerializer Serializer valueSerializer Serializer keyReferencable bool valueReferencable bool - mapInStruct bool // Use mapInStruct to distinguish concrete map types during deserialization + hasGenerics bool // True when map is a struct field with declared key/value types +} +// Write handles ref tracking and type writing, then delegates to WriteData +func (s mapSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { + if writeMapRefAndType(ctx, refMode, writeType, value) || ctx.HasError() { + return + } + s.WriteData(ctx, value) } +// WriteData serializes map data using chunk protocol func (s mapSerializer) WriteData(ctx *WriteContext, value reflect.Value) { buf := ctx.Buffer() - if value.Kind() == reflect.Interface { - value = value.Elem() - } + value = unwrapInterface(value) length := value.Len() buf.WriteVaruint32(uint32(length)) if length == 0 { return } - typeResolver := ctx.TypeResolver() - // Use declared serializers if available (mapInStruct case) - // Don't clear them - we need them for KEY_DECL_TYPE/VALUE_DECL_TYPE flags - keySerializer := s.keySerializer - valueSerializer := s.valueSerializer + iter := value.MapRange() if !iter.Next() { return } - entryKey, entryVal := iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } + + typeResolver := ctx.TypeResolver() + trackRef := ctx.TrackRef() + entryKey := unwrapInterface(iter.Key()) + entryVal := unwrapInterface(iter.Value()) hasNext := true - // For xlang struct fields (mapInStruct = true), use declared types (set DECL_TYPE flags) - // For internal Go serialization (mapInStruct = false), always write type info (don't set DECL_TYPE flags) - isXlang := s.mapInStruct + for hasNext { + // Phase 1: Handle null entries (single-item chunks) for { - keyValid := isValid(entryKey) - valValid := isValid(entryVal) - if keyValid { - if valValid { - break - } - // Null value case - use DECL_TYPE only for xlang struct fields - if isXlang && keySerializer != nil { - if s.keyReferencable { - buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF) - keySerializer.Write(ctx, RefModeTracking, false, false, entryKey) - if ctx.HasError() { - return - } - } else { - buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE) - keySerializer.WriteData(ctx, entryKey) - if ctx.HasError() { - return - } - } - } else { - // Polymorphic key with null value - keyTypeInfo, typeErr := getActualTypeInfo(entryKey, typeResolver) - if typeErr != nil { - ctx.SetError(FromError(typeErr)) - return - } - // Only set TRACKING_KEY_REF if ref tracking is enabled AND type needs it - chunkHeader := int8(VALUE_HAS_NULL) - trackKeyRef := ctx.TrackRef() && keyTypeInfo.NeedWriteRef - if trackKeyRef { - chunkHeader |= TRACKING_KEY_REF - } - buf.WriteInt8(chunkHeader) - // Write type info - typeResolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) - // Write value with appropriate ref mode - keyRefMode := RefModeNone - if trackKeyRef { - keyRefMode = RefModeTracking - } - keyTypeInfo.Serializer.Write(ctx, keyRefMode, false, false, entryKey) - if ctx.HasError() { - return - } - } + keyValid := entryKey.IsValid() + valValid := entryVal.IsValid() + + if keyValid && valValid { + break // Proceed to regular chunk + } + + if !keyValid && !valValid { + buf.WriteInt8(KV_NULL) + } else if !valValid { + s.writeNullValueEntry(ctx, entryKey, typeResolver, trackRef) } else { - if valValid { - // Null key case - use DECL_TYPE only for xlang struct fields - if isXlang && valueSerializer != nil { - if s.valueReferencable { - buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF) - valueSerializer.Write(ctx, RefModeTracking, false, false, entryVal) - if ctx.HasError() { - return - } - } else { - buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE) - valueSerializer.WriteData(ctx, entryVal) - if ctx.HasError() { - return - } - } - } else { - // Polymorphic value with null key - valueTypeInfo, typeErr := getActualTypeInfo(entryVal, typeResolver) - if typeErr != nil { - ctx.SetError(FromError(typeErr)) - return - } - // Only set TRACKING_VALUE_REF if ref tracking is enabled AND type needs it - chunkHeader := int8(KEY_HAS_NULL) - trackValueRef := ctx.TrackRef() && valueTypeInfo.NeedWriteRef - if trackValueRef { - chunkHeader |= TRACKING_VALUE_REF - } - buf.WriteInt8(chunkHeader) - // Write type info - typeResolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) - // Write value with appropriate ref mode - valueRefMode := RefModeNone - if trackValueRef { - valueRefMode = RefModeTracking - } - valueTypeInfo.Serializer.Write(ctx, valueRefMode, false, false, entryVal) - if ctx.HasError() { - return - } - } - } else { - buf.WriteInt8(KV_NULL) - } + s.writeNullKeyEntry(ctx, entryVal, typeResolver, trackRef) + } + + if ctx.HasError() { + return } + if iter.Next() { - entryKey, entryVal = iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } + entryKey = unwrapInterface(iter.Key()) + entryVal = unwrapInterface(iter.Value()) } else { - hasNext = false - break + return } } - if !hasNext { - break + + // Phase 2: Write regular chunk with same-type entries + hasNext = s.writeChunk(ctx, iter, &entryKey, &entryVal, typeResolver, trackRef) + if ctx.HasError() { + return } - keyCls := getActualType(entryKey) - valueCls := getActualType(entryVal) - buf.WriteInt16(-1) - chunkSizeOffset := buf.writerIndex - chunkHeader := 0 - keyWriteRef := s.keyReferencable - valueWriteRef := s.valueReferencable - // For xlang struct fields, use declared types (set DECL_TYPE flags) - // For internal Go serialization, always write type info - if isXlang && keySerializer != nil { - chunkHeader |= KEY_DECL_TYPE + } +} + +// writeNullValueEntry writes a single entry where the value is null +func (s mapSerializer) writeNullValueEntry(ctx *WriteContext, key reflect.Value, resolver *TypeResolver, trackRef bool) { + buf := ctx.Buffer() + + if s.hasGenerics && s.keySerializer != nil { + if s.keyReferencable && trackRef { + buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF) + s.keySerializer.Write(ctx, RefModeTracking, false, false, key) } else { - keyTypeInfo, _ := getActualTypeInfo(entryKey, typeResolver) - typeResolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) - keySerializer = keyTypeInfo.Serializer - keyWriteRef = keyTypeInfo.NeedWriteRef + buf.WriteInt8(NULL_VALUE_KEY_DECL_TYPE) + s.keySerializer.WriteData(ctx, key) } - if isXlang && valueSerializer != nil { - chunkHeader |= VALUE_DECL_TYPE + return + } + + // Polymorphic key + keyTypeInfo, err := getTypeInfoForValue(key, resolver) + if err != nil { + ctx.SetError(FromError(err)) + return + } + + header := int8(VALUE_HAS_NULL) + writeKeyRef := trackRef && keyTypeInfo.NeedWriteRef + if writeKeyRef { + header |= TRACKING_KEY_REF + } + buf.WriteInt8(header) + resolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) + + refMode := RefModeNone + if writeKeyRef { + refMode = RefModeTracking + } + keyTypeInfo.Serializer.Write(ctx, refMode, false, false, key) +} + +// writeNullKeyEntry writes a single entry where the key is null +func (s mapSerializer) writeNullKeyEntry(ctx *WriteContext, value reflect.Value, resolver *TypeResolver, trackRef bool) { + buf := ctx.Buffer() + + if s.hasGenerics && s.valueSerializer != nil { + if s.valueReferencable && trackRef { + buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF) + s.valueSerializer.Write(ctx, RefModeTracking, false, false, value) } else { - valueTypeInfo, _ := getActualTypeInfo(entryVal, typeResolver) - typeResolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) - valueSerializer = valueTypeInfo.Serializer - valueWriteRef = valueTypeInfo.NeedWriteRef + buf.WriteInt8(NULL_KEY_VALUE_DECL_TYPE) + s.valueSerializer.WriteData(ctx, value) } + return + } - // Only set TRACKING_KEY_REF/TRACKING_VALUE_REF if ref tracking is enabled - trackRef := ctx.TrackRef() - if keyWriteRef && trackRef { - chunkHeader |= TRACKING_KEY_REF - } - if valueWriteRef && trackRef { - chunkHeader |= TRACKING_VALUE_REF - } - buf.PutUint8(chunkSizeOffset-2, uint8(chunkHeader)) - chunkSize := 0 - keyRefMode := RefModeNone - if keyWriteRef && trackRef { - keyRefMode = RefModeTracking - } - valueRefMode := RefModeNone - if valueWriteRef && trackRef { - valueRefMode = RefModeTracking - } - for chunkSize < MAX_CHUNK_SIZE { - if !isValid(entryKey) || !isValid(entryVal) || getActualType(entryKey) != keyCls || getActualType(entryVal) != valueCls { - break - } + // Polymorphic value + valueTypeInfo, err := getTypeInfoForValue(value, resolver) + if err != nil { + ctx.SetError(FromError(err)) + return + } - // WriteData key with optional ref tracking - keySerializer.Write(ctx, keyRefMode, false, false, entryKey) - if ctx.HasError() { - return - } + header := int8(KEY_HAS_NULL) + writeValueRef := trackRef && valueTypeInfo.NeedWriteRef + if writeValueRef { + header |= TRACKING_VALUE_REF + } + buf.WriteInt8(header) + resolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) - // WriteData value with optional ref tracking - valueSerializer.Write(ctx, valueRefMode, false, false, entryVal) - if ctx.HasError() { - return - } + refMode := RefModeNone + if writeValueRef { + refMode = RefModeTracking + } + valueTypeInfo.Serializer.Write(ctx, refMode, false, false, value) +} - chunkSize++ +// writeChunk writes a chunk of entries with the same key/value types +func (s mapSerializer) writeChunk(ctx *WriteContext, iter *reflect.MapIter, entryKey, entryVal *reflect.Value, resolver *TypeResolver, trackRef bool) bool { + buf := ctx.Buffer() + keyType := (*entryKey).Type() + valueType := (*entryVal).Type() - if iter.Next() { - entryKey, entryVal = iter.Key(), iter.Value() - if entryKey.Kind() == reflect.Interface { - entryKey = entryKey.Elem() - } - if entryVal.Kind() == reflect.Interface { - entryVal = entryVal.Elem() - } - } else { - hasNext = false - break - } + // Reserve space: header (1 byte) + size (1 byte) + headerOffset := buf.writerIndex + buf.WriteInt16(-1) + + header := 0 + var keySer, valSer Serializer + keyWriteRef := s.keyReferencable + valueWriteRef := s.valueReferencable + + // Determine key serializer and write type info if needed + if s.hasGenerics && s.keySerializer != nil { + header |= KEY_DECL_TYPE + keySer = s.keySerializer + } else { + keyTypeInfo, _ := getTypeInfoForValue(*entryKey, resolver) + resolver.WriteTypeInfo(buf, keyTypeInfo, ctx.Err()) + keySer = keyTypeInfo.Serializer + keyWriteRef = keyTypeInfo.NeedWriteRef + } + + // Determine value serializer and write type info if needed + if s.hasGenerics && s.valueSerializer != nil { + header |= VALUE_DECL_TYPE + valSer = s.valueSerializer + } else { + valueTypeInfo, _ := getTypeInfoForValue(*entryVal, resolver) + resolver.WriteTypeInfo(buf, valueTypeInfo, ctx.Err()) + valSer = valueTypeInfo.Serializer + valueWriteRef = valueTypeInfo.NeedWriteRef + } + + // Set ref tracking flags + keyRefMode := RefModeNone + if keyWriteRef && trackRef { + header |= TRACKING_KEY_REF + keyRefMode = RefModeTracking + } + valueRefMode := RefModeNone + if valueWriteRef && trackRef { + header |= TRACKING_VALUE_REF + valueRefMode = RefModeTracking + } + + buf.PutUint8(headerOffset, uint8(header)) + + // Write entries with same type + chunkSize := 0 + for chunkSize < MAX_CHUNK_SIZE { + k := *entryKey + v := *entryVal + + // Break if null or type changed + if !k.IsValid() || !v.IsValid() || k.Type() != keyType || v.Type() != valueType { + break + } + + keySer.Write(ctx, keyRefMode, false, false, k) + if ctx.HasError() { + return false + } + valSer.Write(ctx, valueRefMode, false, false, v) + if ctx.HasError() { + return false + } + chunkSize++ + + if iter.Next() { + *entryKey = unwrapInterface(iter.Key()) + *entryVal = unwrapInterface(iter.Value()) + } else { + buf.PutUint8(headerOffset+1, uint8(chunkSize)) + return false } - keySerializer = s.keySerializer - valueSerializer = s.valueSerializer - buf.PutUint8(chunkSizeOffset-1, uint8(chunkSize)) } + + buf.PutUint8(headerOffset+1, uint8(chunkSize)) + return true } -func (s mapSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - done := writeMapRefAndType(ctx, refMode, writeType, value) - if done || ctx.HasError() { +// Read handles ref tracking and type reading, then delegates to ReadData +func (s mapSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { + if readMapRefAndType(ctx, refMode, readType, value) || ctx.HasError() { return } - s.WriteData(ctx, value) -} - -func (s mapSerializer) writeObj(ctx *WriteContext, serializer Serializer, obj reflect.Value) { - serializer.WriteData(ctx, obj) + s.ReadData(ctx, value.Type(), value) } +// ReadData deserializes map data using chunk protocol func (s mapSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { buf := ctx.Buffer() ctxErr := ctx.Err() refResolver := ctx.RefResolver() - if s.type_ == nil { - s.type_ = type_ - } + typeResolver := ctx.TypeResolver() + // Initialize map if value.IsNil() { - isIfaceMap := func(t reflect.Type) bool { - return t.Kind() == reflect.Map && - t.Key().Kind() == reflect.Interface && - t.Elem().Kind() == reflect.Interface - } - // case 1: A map inside a struct will have a fixed key and value type. - // case 2: The user has specified the type of the map explicitly. - // Otherwise, a generic map type will be used - switch { - case s.mapInStruct: - value.Set(reflect.MakeMap(type_)) - case !isIfaceMap(type_): - value.Set(reflect.MakeMap(type_)) - default: + mapType := type_ + // For interface{} maps without declared types, use map[interface{}]interface{} + if !s.hasGenerics && type_.Key().Kind() == reflect.Interface && type_.Elem().Kind() == reflect.Interface { iface := reflect.TypeOf((*interface{})(nil)).Elem() - newMapType := reflect.MapOf(iface, iface) - value.Set(reflect.MakeMap(newMapType)) + mapType = reflect.MapOf(iface, iface) } + value.Set(reflect.MakeMap(mapType)) } - refResolver.Reference(value) + size := int(buf.ReadVaruint32(ctxErr)) - var chunkHeader uint8 - if size > 0 { - chunkHeader = buf.ReadUint8(ctxErr) + if size == 0 || ctx.HasError() { + return } + + chunkHeader := buf.ReadUint8(ctxErr) if ctx.HasError() { return } keyType := type_.Key() valueType := type_.Elem() - keySer := s.keySerializer - valSer := s.valueSerializer - typeResolver := ctx.TypeResolver() for size > 0 { + // Phase 1: Handle null entries for { keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0 valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0 - var k, v reflect.Value - if !keyHasNull { - if !valueHasNull { - break - } else { - // Null value case: read key only - keyDeclared := (chunkHeader & KEY_DECL_TYPE) != 0 - trackKeyRef := (chunkHeader & TRACKING_KEY_REF) != 0 - - // When trackKeyRef is set and type is not declared, Java writes: - // ref flag + type info + data - // So we need to read ref flag first, then type info, then data - if trackKeyRef && !keyDeclared { - // Read ref flag first - refID, err := refResolver.TryPreserveRefId(buf) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := refResolver.GetReadObject(refID) - if obj.IsValid() { - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(obj, nullVal) - } - size-- - if size == 0 { - return - } - chunkHeader = buf.ReadUint8(ctxErr) - continue - } - - // Read type info - ti := typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = ti.Serializer - keyType = ti.Type - - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() - - // Read data (ref already handled) - keySer.ReadData(ctx, keyType, k) - if ctx.HasError() { - return - } - refResolver.Reference(k) - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(k, nullVal) - } else { - // ReadData type info if not declared - var keyTypeInfo *TypeInfo - if !keyDeclared { - keyTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = keyTypeInfo.Serializer - keyType = keyTypeInfo.Type - } - - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() - - // Use ReadWithTypeInfo if type was read, otherwise Read - keyRefMode := RefModeNone - if trackKeyRef { - keyRefMode = RefModeTracking - } - if keyTypeInfo != nil { - keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) - } else { - keySer.Read(ctx, keyRefMode, false, false, k) - } - if ctx.HasError() { - return - } - // Use zero value for null value (nil for interface{}/pointer types) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(k, nullVal) - } + + if !keyHasNull && !valueHasNull { + break // Proceed to regular chunk + } + + if keyHasNull && valueHasNull { + value.SetMapIndex(reflect.Zero(keyType), reflect.Zero(valueType)) + } else if valueHasNull { + k := s.readNullValueEntry(ctx, chunkHeader, keyType, typeResolver, refResolver) + if ctx.HasError() { + return } + value.SetMapIndex(k, reflect.Zero(valueType)) } else { - if !valueHasNull { - // Null key case: read value only - valueDeclared := (chunkHeader & VALUE_DECL_TYPE) != 0 - trackValueRef := (chunkHeader & TRACKING_VALUE_REF) != 0 - - // When trackValueRef is set and type is not declared, Java writes: - // ref flag + type info + data - // So we need to read ref flag first, then type info, then data - if trackValueRef && !valueDeclared { - // Read ref flag first - refID, err := refResolver.TryPreserveRefId(buf) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference to existing object - obj := refResolver.GetReadObject(refID) - if obj.IsValid() { - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, obj) - } - size-- - if size == 0 { - return - } - chunkHeader = buf.ReadUint8(ctxErr) - continue - } - - // Read type info - ti := typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - valSer = ti.Serializer - valueType = ti.Type - - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() - - // Read data (ref already handled) - valSer.ReadData(ctx, valueType, v) - if ctx.HasError() { - return - } - refResolver.Reference(v) - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, v) - } else { - // ReadData type info if not declared - var valueTypeInfo *TypeInfo - if !valueDeclared { - valueTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - valSer = valueTypeInfo.Serializer - valueType = valueTypeInfo.Type - } - - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() - - // Use ReadWithTypeInfo if type was read, otherwise Read - valueRefMode := RefModeNone - if trackValueRef { - valueRefMode = RefModeTracking - } - if valueTypeInfo != nil { - valSer.ReadWithTypeInfo(ctx, valueRefMode, valueTypeInfo, v) - } else { - valSer.Read(ctx, valueRefMode, false, false, v) - } - if ctx.HasError() { - return - } - // Use zero value for null key (nil for interface{}/pointer types) - nullKey := reflect.Zero(value.Type().Key()) - value.SetMapIndex(nullKey, v) - } - } else { - // Both key and value are null - nullKey := reflect.Zero(value.Type().Key()) - nullVal := reflect.Zero(value.Type().Elem()) - value.SetMapIndex(nullKey, nullVal) + v := s.readNullKeyEntry(ctx, chunkHeader, valueType, typeResolver, refResolver) + if ctx.HasError() { + return } + value.SetMapIndex(reflect.Zero(keyType), v) } size-- if size == 0 { return - } else { - chunkHeader = buf.ReadUint8(ctxErr) + } + chunkHeader = buf.ReadUint8(ctxErr) + if ctx.HasError() { + return } } - trackKeyRef := (chunkHeader & TRACKING_KEY_REF) != 0 - trackValRef := (chunkHeader & TRACKING_VALUE_REF) != 0 - keyDeclType := (chunkHeader & KEY_DECL_TYPE) != 0 - valDeclType := (chunkHeader & VALUE_DECL_TYPE) != 0 - chunkSize := int(buf.ReadUint8(ctxErr)) + // Phase 2: Read regular chunk + size = s.readChunk(ctx, value, chunkHeader, size, keyType, valueType, typeResolver) + if ctx.HasError() { + return + } - // ReadData type info if not declared - var keyTypeInfo *TypeInfo - if !keyDeclType { - keyTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { - return - } - keySer = keyTypeInfo.Serializer - keyType = keyTypeInfo.Type - } else if keySer == nil { - // KEY_DECL_TYPE is set but we don't have a serializer - get one from the map's key type - keySer, _ = typeResolver.getSerializerByType(keyType, false) - } - var valueTypeInfo *TypeInfo - if !valDeclType { - valueTypeInfo = typeResolver.ReadTypeInfo(buf, ctxErr) - if ctxErr.HasError() { + if size > 0 { + chunkHeader = buf.ReadUint8(ctxErr) + if ctx.HasError() { return } - valSer = valueTypeInfo.Serializer - valueType = valueTypeInfo.Type - } else if valSer == nil { - // VALUE_DECL_TYPE is set but we don't have a serializer - get one from the map's value type - valSer, _ = typeResolver.getSerializerByType(valueType, false) } + } +} + +// readNullValueEntry reads an entry where value is null, returns the key +func (s mapSerializer) readNullValueEntry(ctx *ReadContext, header uint8, keyType reflect.Type, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + buf := ctx.Buffer() + ctxErr := ctx.Err() + keyDeclared := (header & KEY_DECL_TYPE) != 0 + trackKeyRef := (header & TRACKING_KEY_REF) != 0 - keyRefMode := RefModeNone - if trackKeyRef { - keyRefMode = RefModeTracking + return s.readSingleValue(ctx, buf, ctxErr, keyDeclared, trackKeyRef, keyType, s.keySerializer, resolver, refResolver) +} + +// readNullKeyEntry reads an entry where key is null, returns the value +func (s mapSerializer) readNullKeyEntry(ctx *ReadContext, header uint8, valueType reflect.Type, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + buf := ctx.Buffer() + ctxErr := ctx.Err() + valueDeclared := (header & VALUE_DECL_TYPE) != 0 + trackValueRef := (header & TRACKING_VALUE_REF) != 0 + + return s.readSingleValue(ctx, buf, ctxErr, valueDeclared, trackValueRef, valueType, s.valueSerializer, resolver, refResolver) +} + +// readSingleValue reads a single key or value with proper ref/type handling +func (s mapSerializer) readSingleValue(ctx *ReadContext, buf *ByteBuffer, ctxErr *Error, isDeclared, trackRef bool, staticType reflect.Type, declaredSer Serializer, resolver *TypeResolver, refResolver *RefResolver) reflect.Value { + // When ref tracking AND not declared, ref flag comes before type info + if trackRef && !isDeclared { + refID, err := refResolver.TryPreserveRefId(buf) + if err != nil { + ctx.SetError(FromError(err)) + return reflect.Value{} } - valRefMode := RefModeNone - if trackValRef { - valRefMode = RefModeTracking + if refID < int32(NotNullValueFlag) { + return refResolver.GetReadObject(refID) } - for i := 0; i < chunkSize; i++ { - var k, v reflect.Value - // ReadData key - kt := keyType - if kt == nil { - kt = value.Type().Key() - } - k = reflect.New(kt).Elem() + // Read type info and data + ti := resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return reflect.Value{} + } - // Use ReadWithTypeInfo if type was read, otherwise Read - if keyTypeInfo != nil { - keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) - } else { - keySer.Read(ctx, keyRefMode, false, false, k) - } - if ctx.HasError() { - return - } + valType := ti.Type + if valType == nil { + valType = staticType + } + v := reflect.New(valType).Elem() + ti.Serializer.ReadData(ctx, valType, v) + if ctx.HasError() { + return reflect.Value{} + } + refResolver.Reference(v) + return v + } - // ReadData value - vt := valueType - if vt == nil { - vt = value.Type().Elem() - } - v = reflect.New(vt).Elem() + // Read type info if not declared + var typeInfo *TypeInfo + var ser Serializer + valType := staticType - // Use ReadWithTypeInfo if type was read, otherwise Read - if valueTypeInfo != nil { - valSer.ReadWithTypeInfo(ctx, valRefMode, valueTypeInfo, v) - } else { - valSer.Read(ctx, valRefMode, false, false, v) - } - if ctx.HasError() { - return - } - // Unwrap interfaces if they're not the map's type - if k.Kind() == reflect.Interface { - k = k.Elem() - } - if v.Kind() == reflect.Interface { - v = v.Elem() - } - setMapValue(value, k, v) - size-- + if !isDeclared { + typeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return reflect.Value{} + } + ser = typeInfo.Serializer + valType = typeInfo.Type + } else { + ser = declaredSer + if ser == nil { + ser, _ = resolver.getSerializerByType(staticType, false) } + } + + if valType == nil { + valType = staticType + } + v := reflect.New(valType).Elem() + + refMode := RefModeNone + if trackRef { + refMode = RefModeTracking + } + + if typeInfo != nil { + ser.ReadWithTypeInfo(ctx, refMode, typeInfo, v) + } else { + ser.Read(ctx, refMode, false, false, v) + } + + return v +} +// readChunk reads a chunk of entries, returns remaining size +func (s mapSerializer) readChunk(ctx *ReadContext, mapVal reflect.Value, header uint8, size int, keyType, valueType reflect.Type, resolver *TypeResolver) int { + buf := ctx.Buffer() + ctxErr := ctx.Err() + + trackKeyRef := (header & TRACKING_KEY_REF) != 0 + trackValRef := (header & TRACKING_VALUE_REF) != 0 + keyDeclType := (header & KEY_DECL_TYPE) != 0 + valDeclType := (header & VALUE_DECL_TYPE) != 0 + + chunkSize := int(buf.ReadUint8(ctxErr)) + if ctx.HasError() { + return 0 + } + + // Read type info if not declared + var keyTypeInfo, valueTypeInfo *TypeInfo + var keySer, valSer Serializer + + if !keyDeclType { + keyTypeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return 0 + } + keySer = keyTypeInfo.Serializer + keyType = keyTypeInfo.Type + } else { keySer = s.keySerializer + if keySer == nil { + keySer, _ = resolver.getSerializerByType(keyType, false) + } + } + + if !valDeclType { + valueTypeInfo = resolver.ReadTypeInfo(buf, ctxErr) + if ctxErr.HasError() { + return 0 + } + valSer = valueTypeInfo.Serializer + valueType = valueTypeInfo.Type + } else { valSer = s.valueSerializer - if size > 0 { - chunkHeader = buf.ReadUint8(ctxErr) + if valSer == nil { + valSer, _ = resolver.getSerializerByType(valueType, false) } } -} -func (s mapSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - done := readMapRefAndType(ctx, refMode, readType, value) - if done || ctx.HasError() { - return + keyRefMode := RefModeNone + if trackKeyRef { + keyRefMode = RefModeTracking } - s.ReadData(ctx, value.Type(), value) + valRefMode := RefModeNone + if trackValRef { + valRefMode = RefModeTracking + } + + for i := 0; i < chunkSize; i++ { + k := reflect.New(keyType).Elem() + if keyTypeInfo != nil { + keySer.ReadWithTypeInfo(ctx, keyRefMode, keyTypeInfo, k) + } else { + keySer.Read(ctx, keyRefMode, false, false, k) + } + if ctx.HasError() { + return 0 + } + + v := reflect.New(valueType).Elem() + if valueTypeInfo != nil { + valSer.ReadWithTypeInfo(ctx, valRefMode, valueTypeInfo, v) + } else { + valSer.Read(ctx, valRefMode, false, false, v) + } + if ctx.HasError() { + return 0 + } + + setMapValue(mapVal, unwrapInterface(k), unwrapInterface(v)) + size-- + } + + return size } func (s mapSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again s.Read(ctx, refMode, false, false, value) } -func (s mapSerializer) readObj( - ctx *ReadContext, - v *reflect.Value, - serializer Serializer, -) { - serializer.ReadData(ctx, v.Type(), *v) +// Helper functions + +// writeMapRefAndType handles reference and type writing for maps. +// Returns true if value was already written (nil or ref). +func writeMapRefAndType(ctx *WriteContext, refMode RefMode, writeType bool, value reflect.Value) bool { + switch refMode { + case RefModeTracking: + if value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return true + } + refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) + if err != nil { + ctx.SetError(FromError(err)) + return false + } + if refWritten { + return true + } + case RefModeNullOnly: + if value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return true + } + ctx.buffer.WriteInt8(NotNullValueFlag) + } + if writeType { + ctx.buffer.WriteVaruint32Small7(uint32(MAP)) + } + return false } -func getActualType(v reflect.Value) reflect.Type { - if v.Kind() == reflect.Interface && !v.IsNil() { - return v.Elem().Type() +// readMapRefAndType handles reference and type reading for maps. +// Returns true if a reference was resolved. +func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value reflect.Value) bool { + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, err := ctx.RefResolver().TryPreserveRefId(buf) + if err != nil { + ctx.SetError(FromError(err)) + return false + } + if refID < int32(NotNullValueFlag) { + obj := ctx.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return true + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return true + } } - return v.Type() + if readType { + buf.ReadVaruint32Small7(ctxErr) + } + return false +} + +func unwrapInterface(v reflect.Value) reflect.Value { + // For map serialization, we need to unwrap interfaces including nil ones + // A nil interface should become an invalid (zero) Value for proper null detection + if v.Kind() == reflect.Interface { + return v.Elem() + } + return v +} + +// UnwrapReflectValue is exported for use by other packages +func UnwrapReflectValue(v reflect.Value) reflect.Value { + return unwrapInterface(v) } -func getActualTypeInfo(v reflect.Value, resolver *TypeResolver) (*TypeInfo, error) { +func getTypeInfoForValue(v reflect.Value, resolver *TypeResolver) (*TypeInfo, error) { if v.Kind() == reflect.Interface && !v.IsNil() { elem := v.Elem() if !elem.IsValid() { @@ -730,44 +634,23 @@ func getActualTypeInfo(v reflect.Value, resolver *TypeResolver) (*TypeInfo, erro return resolver.getTypeInfo(v, true) } -func UnwrapReflectValue(v reflect.Value) reflect.Value { - for v.Kind() == reflect.Interface && !v.IsNil() { - v = v.Elem() - } - return v -} - -func isValid(v reflect.Value) bool { - // Zero values are valid, so apply this change temporarily. - return v.IsValid() -} - -// setMapValue sets a key-value pair into a map, handling interface types where -// the concrete type may need to be wrapped in a pointer to implement the interface. +// setMapValue sets a key-value pair into a map, handling interface types func setMapValue(mapVal, key, value reflect.Value) { mapKeyType := mapVal.Type().Key() mapValueType := mapVal.Type().Elem() - // Handle key finalKey := key - if mapKeyType.Kind() == reflect.Interface { - if !key.Type().AssignableTo(mapKeyType) { - // Try pointer - common case where interface has pointer receivers - ptr := reflect.New(key.Type()) - ptr.Elem().Set(key) - finalKey = ptr - } + if mapKeyType.Kind() == reflect.Interface && !key.Type().AssignableTo(mapKeyType) { + ptr := reflect.New(key.Type()) + ptr.Elem().Set(key) + finalKey = ptr } - // Handle value finalValue := value - if mapValueType.Kind() == reflect.Interface { - if !value.Type().AssignableTo(mapValueType) { - // Try pointer - common case where interface has pointer receivers - ptr := reflect.New(value.Type()) - ptr.Elem().Set(value) - finalValue = ptr - } + if mapValueType.Kind() == reflect.Interface && !value.Type().AssignableTo(mapValueType) { + ptr := reflect.New(value.Type()) + ptr.Elem().Set(value) + finalValue = ptr } mapVal.SetMapIndex(finalKey, finalValue) diff --git a/go/fory/ref_resolver.go b/go/fory/ref_resolver.go index fdc9ced81f..8003a1a6f9 100644 --- a/go/fory/ref_resolver.go +++ b/go/fory/ref_resolver.go @@ -230,23 +230,16 @@ func (r *RefResolver) PreserveRefId() (int32, error) { func (r *RefResolver) TryPreserveRefId(buffer *ByteBuffer) (int32, error) { var ctxErr Error - pos := buffer.ReaderIndex() headFlag := buffer.ReadInt8(&ctxErr) if ctxErr.HasError() { return 0, ctxErr } - if DebugOutputEnabled() { - fmt.Printf("[Go][fory-debug] TryPreserveRefId at position %d: headFlag=%d\n", pos, headFlag) - } if headFlag == RefFlag { // read ref id and get object from ref resolver refId := buffer.ReadVaruint32(&ctxErr) if ctxErr.HasError() { return 0, ctxErr } - if DebugOutputEnabled() { - fmt.Printf("[Go][fory-debug] TryPreserveRefId: REF_FLAG, refId=%d\n", refId) - } r.readObject = r.GetReadObject(int32(refId)) } else { r.readObject = reflect.Value{} diff --git a/go/fory/struct.go b/go/fory/struct.go index 02d0307848..4b00a3b4c2 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -289,16 +289,10 @@ func (s *structSerializer) WriteData(ctx *WriteContext, value reflect.Value) { // Lazy initialization if !s.initialized { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] structSerializer.WriteData: calling initialize for type=%v\n", s.type_) - } if err := s.initialize(ctx.TypeResolver()); err != nil { ctx.SetError(FromError(err)) return } - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] structSerializer.WriteData: after initialize, remainingFields=%d\n", len(s.remainingFields)) - } } buf := ctx.Buffer() @@ -642,12 +636,7 @@ func (s *structSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value // In compatible mode with meta share, struct hash is not written if !ctx.Compatible() { err := ctx.Err() - pos := buf.ReaderIndex() structHash := buf.ReadInt32(err) - if DebugOutputEnabled() { - fmt.Printf("[Go][fory-debug] Reading struct hash for %s at position %d: read=%d, expected=%d\n", - s.type_.String(), pos, structHash, s.structHash) - } if structHash != s.structHash { ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) return diff --git a/go/fory/type_def.go b/go/fory/type_def.go index 8fe7711d5c..78895f9c9a 100644 --- a/go/fory/type_def.go +++ b/go/fory/type_def.go @@ -808,6 +808,7 @@ func (m *MapFieldType) getTypeInfo(f *Fory) (TypeInfo, error) { mapSerializer := &mapSerializer{ keySerializer: keyInfo.Serializer, valueSerializer: valueInfo.Serializer, + hasGenerics: true, } return TypeInfo{Type: mapType, Serializer: mapSerializer}, nil } @@ -828,6 +829,7 @@ func (m *MapFieldType) getTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo mapSerializer := &mapSerializer{ keySerializer: keyInfo.Serializer, valueSerializer: valueInfo.Serializer, + hasGenerics: true, } return TypeInfo{Type: mapType, Serializer: mapSerializer}, nil } diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index 4dca046856..a45b3c2f75 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -753,17 +753,11 @@ func (r *TypeResolver) getTypeInfo(value reflect.Value, create bool) (*TypeInfo, typeString := value.Type() typePtr := typePointer(typeString) if cachedInfo, ok := r.typePointerCache[typePtr]; ok { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] getTypeInfo: found in cache type=%v serializer=%T\n", typeString, cachedInfo.Serializer) - } return cachedInfo, nil } // Slow path: map lookup by reflect.Type if info, ok := r.typesInfo[typeString]; ok { - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] getTypeInfo: found in typesInfo type=%v serializer=%T\n", typeString, info.Serializer) - } if info.Serializer == nil { /* Lazy initialize serializer if not created yet @@ -1426,10 +1420,10 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s valueSerializer: valueSerializer, keyReferencable: keyReferencable, valueReferencable: valueReferencable, - mapInStruct: mapInStruct, + hasGenerics: mapInStruct, }, nil } else { - return mapSerializer{mapInStruct: mapInStruct}, nil + return mapSerializer{hasGenerics: mapInStruct}, nil } case reflect.Struct: serializer := r.typeToSerializers[type_] From 517ee7c9426afde0b03e06c366ea900c03512b03 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 19:47:22 +0800 Subject: [PATCH 34/35] rename slice serializer --- go/fory/slice.go | 30 +- go/fory/slice_dyn.go | 6 +- go/fory/struct.go | 3932 +++++++++++++++++++------------------- go/fory/type_resolver.go | 14 +- 4 files changed, 1991 insertions(+), 1991 deletions(-) diff --git a/go/fory/slice.go b/go/fory/slice.go index 539f329b14..4b40374fd3 100644 --- a/go/fory/slice.go +++ b/go/fory/slice.go @@ -106,45 +106,45 @@ func isNull(v reflect.Value) bool { } } -// sliceConcreteValueSerializer serialize a slice whose elem is not an interface or pointer to interface. -// Use newSliceConcreteValueSerializer to create instances with proper type validation. +// sliceSerializer serialize a slice whose elem is not an interface or pointer to interface. +// Use newSliceSerializer to create instances with proper type validation. // This serializer uses LIST protocol for non-primitive element types. -type sliceConcreteValueSerializer struct { +type sliceSerializer struct { type_ reflect.Type elemSerializer Serializer referencable bool } -// newSliceConcreteValueSerializer creates a sliceConcreteValueSerializer for slices with concrete element types. +// newSliceSerializer creates a sliceSerializer for slices with concrete element types. // It returns an error if the element type is an interface, pointer to interface, or a primitive type. // Primitive numeric types (bool, int8, int16, int32, int64, uint8, float32, float64) must use // dedicated primitive slice serializers that use ARRAY protocol (binary size + binary). -func newSliceConcreteValueSerializer(type_ reflect.Type, elemSerializer Serializer, xlang bool) (*sliceConcreteValueSerializer, error) { +func newSliceSerializer(type_ reflect.Type, elemSerializer Serializer, xlang bool) (*sliceSerializer, error) { elem := type_.Elem() if elem.Kind() == reflect.Interface { - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support interface element type: %v", type_) + return nil, fmt.Errorf("sliceSerializer does not support interface element type: %v", type_) } if elem.Kind() == reflect.Ptr && elem.Elem().Kind() == reflect.Interface { - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support pointer to interface element type: %v", type_) + return nil, fmt.Errorf("sliceSerializer does not support pointer to interface element type: %v", type_) } // Primitive numeric types must use dedicated primitive slice serializers (ARRAY protocol) switch elem.Kind() { case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Float32, reflect.Float64: - return nil, fmt.Errorf("sliceConcreteValueSerializer does not support primitive element type %v: use dedicated primitive slice serializer", type_) + return nil, fmt.Errorf("sliceSerializer does not support primitive element type %v: use dedicated primitive slice serializer", type_) } - return &sliceConcreteValueSerializer{ + return &sliceSerializer{ type_: type_, elemSerializer: elemSerializer, referencable: isRefType(elem, xlang), }, nil } -func (s *sliceConcreteValueSerializer) WriteData(ctx *WriteContext, value reflect.Value) { +func (s *sliceSerializer) WriteData(ctx *WriteContext, value reflect.Value) { s.writeDataWithGenerics(ctx, value, false) } -func (s *sliceConcreteValueSerializer) writeDataWithGenerics(ctx *WriteContext, value reflect.Value, hasGenerics bool) { +func (s *sliceSerializer) writeDataWithGenerics(ctx *WriteContext, value reflect.Value, hasGenerics bool) { length := value.Len() buf := ctx.Buffer() @@ -233,7 +233,7 @@ func (s *sliceConcreteValueSerializer) writeDataWithGenerics(ctx *WriteContext, } } -func (s *sliceConcreteValueSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { +func (s *sliceSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { done := writeSliceRefAndType(ctx, refMode, writeType, value, LIST) if done || ctx.HasError() { return @@ -241,7 +241,7 @@ func (s *sliceConcreteValueSerializer) Write(ctx *WriteContext, refMode RefMode, s.writeDataWithGenerics(ctx, value, hasGenerics) } -func (s *sliceConcreteValueSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { +func (s *sliceSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { done := readSliceRefAndType(ctx, refMode, readType, value) if done || ctx.HasError() { return @@ -249,12 +249,12 @@ func (s *sliceConcreteValueSerializer) Read(ctx *ReadContext, refMode RefMode, r s.ReadData(ctx, value.Type(), value) } -func (s *sliceConcreteValueSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { +func (s *sliceSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { // typeInfo is already read, don't read it again s.Read(ctx, refMode, false, false, value) } -func (s *sliceConcreteValueSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { +func (s *sliceSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { buf := ctx.Buffer() ctxErr := ctx.Err() length := int(buf.ReadVaruint32(ctxErr)) diff --git a/go/fory/slice_dyn.go b/go/fory/slice_dyn.go index 64cd22ce38..69c339d857 100644 --- a/go/fory/slice_dyn.go +++ b/go/fory/slice_dyn.go @@ -26,7 +26,7 @@ import ( // element values at runtime. // This serializer is designed for slices with any interface element type // (e.g., []interface{}, []io.Reader, []fmt.Stringer, or pointers to interfaces). -// For slices with concrete element types, use sliceConcreteValueSerializer instead. +// For slices with concrete element types, use sliceSerializer instead. type sliceDynSerializer struct { elemType reflect.Type isInterfaceElem bool @@ -35,7 +35,7 @@ type sliceDynSerializer struct { // newSliceDynSerializer creates a new sliceDynSerializer. // This serializer is ONLY for slices with interface or pointer to interface element types. -// For other slice types, use sliceConcreteValueSerializer instead. +// For other slice types, use sliceSerializer instead. func newSliceDynSerializer(elemType reflect.Type) (sliceDynSerializer, error) { // Nil element type is allowed for fully dynamic slices (e.g., []interface{}) if elemType == nil { @@ -48,7 +48,7 @@ func newSliceDynSerializer(elemType reflect.Type) (sliceDynSerializer, error) { isPointerToInterface := elemType.Kind() == reflect.Ptr && elemType.Elem().Kind() == reflect.Interface if !isInterface && !isPointerToInterface { return sliceDynSerializer{}, fmt.Errorf( - "sliceDynSerializer only supports interface or pointer to interface element types, got %v; use sliceConcreteValueSerializer for other types", elemType) + "sliceDynSerializer only supports interface or pointer to interface element types, got %v; use sliceSerializer for other types", elemType) } return sliceDynSerializer{ elemType: elemType, diff --git a/go/fory/struct.go b/go/fory/struct.go index 4b00a3b4c2..aa0a1cd510 100644 --- a/go/fory/struct.go +++ b/go/fory/struct.go @@ -18,78 +18,78 @@ package fory import ( - "encoding/binary" - "errors" - "fmt" - "math" - "reflect" - "sort" - "strings" - "unicode" - "unicode/utf8" - "unsafe" - - "github.com/spaolacci/murmur3" + "encoding/binary" + "errors" + "fmt" + "math" + "reflect" + "sort" + "strings" + "unicode" + "unicode/utf8" + "unsafe" + + "github.com/spaolacci/murmur3" ) // FieldInfo stores field metadata computed ENTIRELY at init time. // All flags and decisions are pre-computed to eliminate runtime checks. type FieldInfo struct { - Name string - Offset uintptr - Type reflect.Type - StaticId StaticTypeId - TypeId TypeId // Fory type ID for the serializer - Serializer Serializer - Referencable bool - FieldIndex int // -1 if field doesn't exist in current struct (for compatible mode) - FieldDef FieldDef // original FieldDef from remote TypeDef (for compatible mode skip) - - // Pre-computed sizes and offsets (for fixed primitives) - FixedSize int // 0 if not fixed-size, else 1/2/4/8 - WriteOffset int // Offset within fixed-fields buffer region (sum of preceding field sizes) - - // Pre-computed flags for serialization (computed at init time) - RefMode RefMode // ref mode for serializer.Write/Read - WriteType bool // whether to write type info (true for struct fields in compatible mode) - HasGenerics bool // whether element types are known from TypeDef (for container fields) - - // Tag-based configuration (from fory struct tags) - TagID int // -1 = use field name, >=0 = use tag ID - HasForyTag bool // Whether field has explicit fory tag - TagRefSet bool // Whether ref was explicitly set via fory tag - TagRef bool // The ref value from fory tag (only valid if TagRefSet is true) - TagNullableSet bool // Whether nullable was explicitly set via fory tag - TagNullable bool // The nullable value from fory tag (only valid if TagNullableSet is true) + Name string + Offset uintptr + Type reflect.Type + StaticId StaticTypeId + TypeId TypeId // Fory type ID for the serializer + Serializer Serializer + Referencable bool + FieldIndex int // -1 if field doesn't exist in current struct (for compatible mode) + FieldDef FieldDef // original FieldDef from remote TypeDef (for compatible mode skip) + + // Pre-computed sizes and offsets (for fixed primitives) + FixedSize int // 0 if not fixed-size, else 1/2/4/8 + WriteOffset int // Offset within fixed-fields buffer region (sum of preceding field sizes) + + // Pre-computed flags for serialization (computed at init time) + RefMode RefMode // ref mode for serializer.Write/Read + WriteType bool // whether to write type info (true for struct fields in compatible mode) + HasGenerics bool // whether element types are known from TypeDef (for container fields) + + // Tag-based configuration (from fory struct tags) + TagID int // -1 = use field name, >=0 = use tag ID + HasForyTag bool // Whether field has explicit fory tag + TagRefSet bool // Whether ref was explicitly set via fory tag + TagRef bool // The ref value from fory tag (only valid if TagRefSet is true) + TagNullableSet bool // Whether nullable was explicitly set via fory tag + TagNullable bool // The nullable value from fory tag (only valid if TagNullableSet is true) } // fieldHasNonPrimitiveSerializer returns true if the field has a serializer with a non-primitive type ID. // This is used to skip the fast path for fields like enums where StaticId is int32 but the serializer // writes a different format (e.g., unsigned varint for enum ordinals vs signed zigzag for int32). func fieldHasNonPrimitiveSerializer(field *FieldInfo) bool { - if field.Serializer == nil { - return false - } - // ENUM (numeric ID), NAMED_ENUM (namespace/typename), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT - // all require special serialization and should not use the primitive fast path - // Note: ENUM uses unsigned Varuint32Small7 for ordinals, not signed zigzag varint - // Use internal type ID (low 8 bits) since registered types have composite TypeIds like (userID << 8) | internalID - internalTypeId := TypeId(field.TypeId & 0xFF) - switch internalTypeId { - case ENUM, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: - return true - default: - return false - } + if field.Serializer == nil { + return false + } + // ENUM (numeric ID), NAMED_ENUM (namespace/typename), NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT + // all require special serialization and should not use the primitive fast path + // Note: ENUM uses unsigned Varuint32Small7 for ordinals, not signed zigzag varint + // Use internal type ID (low 8 bits) since registered types have composite TypeIds like (userID << 8) | internalID + internalTypeId := TypeId(field.TypeId & 0xFF) + switch internalTypeId { + case ENUM, NAMED_ENUM, NAMED_STRUCT, NAMED_COMPATIBLE_STRUCT, NAMED_EXT: + return true + default: + return false + } } // isEnumField checks if a field is an enum type based on its TypeId func isEnumField(field *FieldInfo) bool { - if field.Serializer == nil { - return false - } - internalTypeId := field.TypeId & 0xFF - return internalTypeId == ENUM || internalTypeId == NAMED_ENUM + if field.Serializer == nil { + return false + } + internalTypeId := field.TypeId & 0xFF + return internalTypeId == ENUM || internalTypeId == NAMED_ENUM } // writeEnumField writes an enum field respecting the field's RefMode. @@ -97,37 +97,37 @@ func isEnumField(field *FieldInfo) bool { // RefMode determines whether null flag is written, regardless of whether the local type is a pointer. // This is important for compatible mode where remote TypeDef's nullable flag controls the wire format. func writeEnumField(ctx *WriteContext, field *FieldInfo, fieldValue reflect.Value) { - buf := ctx.Buffer() - isPointer := fieldValue.Kind() == reflect.Ptr - - // Write null flag based on RefMode only (not based on whether local type is pointer) - if field.RefMode != RefModeNone { - if isPointer && fieldValue.IsNil() { - buf.WriteInt8(NullFlag) - return - } - buf.WriteInt8(NotNullValueFlag) - } - - // Get the actual value to serialize - targetValue := fieldValue - if isPointer { - if fieldValue.IsNil() { - // RefModeNone but nil pointer - this is a protocol error in schema-consistent mode - // Write zero value as fallback - targetValue = reflect.Zero(field.Type.Elem()) - } else { - targetValue = fieldValue.Elem() - } - } - - // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. - // We need to call the inner enumSerializer directly with the dereferenced value. - if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - ptrSer.valueSerializer.WriteData(ctx, targetValue) - } else { - field.Serializer.WriteData(ctx, targetValue) - } + buf := ctx.Buffer() + isPointer := fieldValue.Kind() == reflect.Ptr + + // Write null flag based on RefMode only (not based on whether local type is pointer) + if field.RefMode != RefModeNone { + if isPointer && fieldValue.IsNil() { + buf.WriteInt8(NullFlag) + return + } + buf.WriteInt8(NotNullValueFlag) + } + + // Get the actual value to serialize + targetValue := fieldValue + if isPointer { + if fieldValue.IsNil() { + // RefModeNone but nil pointer - this is a protocol error in schema-consistent mode + // Write zero value as fallback + targetValue = reflect.Zero(field.Type.Elem()) + } else { + targetValue = fieldValue.Elem() + } + } + + // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. + // We need to call the inner enumSerializer directly with the dereferenced value. + if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + ptrSer.valueSerializer.WriteData(ctx, targetValue) + } else { + field.Serializer.WriteData(ctx, targetValue) + } } // readEnumField reads an enum field respecting the field's RefMode. @@ -135,1465 +135,1465 @@ func writeEnumField(ctx *WriteContext, field *FieldInfo, fieldValue reflect.Valu // This is important for compatible mode where remote TypeDef's nullable flag controls the wire format. // Uses context error state for deferred error checking. func readEnumField(ctx *ReadContext, field *FieldInfo, fieldValue reflect.Value) { - buf := ctx.Buffer() - isPointer := fieldValue.Kind() == reflect.Ptr - - // Read null flag based on RefMode only (not based on whether local type is pointer) - if field.RefMode != RefModeNone { - nullFlag := buf.ReadInt8(ctx.Err()) - if nullFlag == NullFlag { - // For pointer enum fields, leave as nil; for non-pointer, set to zero - if !isPointer { - fieldValue.SetInt(0) - } - return - } - } - - // For pointer enum fields, allocate a new value - targetValue := fieldValue - if isPointer { - newVal := reflect.New(field.Type.Elem()) - fieldValue.Set(newVal) - targetValue = newVal.Elem() - } - - // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. - // We need to call the inner enumSerializer directly with the dereferenced value. - if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - ptrSer.valueSerializer.ReadData(ctx, field.Type.Elem(), targetValue) - } else { - field.Serializer.ReadData(ctx, field.Type, targetValue) - } + buf := ctx.Buffer() + isPointer := fieldValue.Kind() == reflect.Ptr + + // Read null flag based on RefMode only (not based on whether local type is pointer) + if field.RefMode != RefModeNone { + nullFlag := buf.ReadInt8(ctx.Err()) + if nullFlag == NullFlag { + // For pointer enum fields, leave as nil; for non-pointer, set to zero + if !isPointer { + fieldValue.SetInt(0) + } + return + } + } + + // For pointer enum fields, allocate a new value + targetValue := fieldValue + if isPointer { + newVal := reflect.New(field.Type.Elem()) + fieldValue.Set(newVal) + targetValue = newVal.Elem() + } + + // For pointer enum fields, the serializer is ptrToValueSerializer wrapping enumSerializer. + // We need to call the inner enumSerializer directly with the dereferenced value. + if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + ptrSer.valueSerializer.ReadData(ctx, field.Type.Elem(), targetValue) + } else { + field.Serializer.ReadData(ctx, field.Type, targetValue) + } } type structSerializer struct { - // Identity - typeTag string - type_ reflect.Type - structHash int32 - - // Pre-sorted field lists by category (computed at init) - fixedFields []*FieldInfo // fixed-size primitives (bool, int8, int16, float32, float64) - varintFields []*FieldInfo // varint primitives (int32, int64, int) - remainingFields []*FieldInfo // all other fields (string, slice, map, struct, etc.) - - // All fields in protocol order (for compatible mode) - fields []*FieldInfo // all fields in sorted order - fieldMap map[string]*FieldInfo // for compatible reading - fieldDefs []FieldDef // for type_def compatibility - - // Pre-computed buffer sizes - fixedSize int // Total bytes for fixed-size primitives - maxVarintSize int // Max bytes for varints (5 per int32, 10 per int64) - - // Mode flags (set at init) - isCompatibleMode bool // true when compatible=true - typeDefDiffers bool // true when compatible=true AND remote TypeDef != local (requires ordered read) - - // Initialization state - initialized bool + // Identity + typeTag string + type_ reflect.Type + structHash int32 + + // Pre-sorted field lists by category (computed at init) + fixedFields []*FieldInfo // fixed-size primitives (bool, int8, int16, float32, float64) + varintFields []*FieldInfo // varint primitives (int32, int64, int) + remainingFields []*FieldInfo // all other fields (string, slice, map, struct, etc.) + + // All fields in protocol order (for compatible mode) + fields []*FieldInfo // all fields in sorted order + fieldMap map[string]*FieldInfo // for compatible reading + fieldDefs []FieldDef // for type_def compatibility + + // Pre-computed buffer sizes + fixedSize int // Total bytes for fixed-size primitives + maxVarintSize int // Max bytes for varints (5 per int32, 10 per int64) + + // Mode flags (set at init) + isCompatibleMode bool // true when compatible=true + typeDefDiffers bool // true when compatible=true AND remote TypeDef != local (requires ordered read) + + // Initialization state + initialized bool } // newStructSerializer creates a new structSerializer with the given parameters. // typeTag can be empty and will be derived from type_.Name() if not provided. // fieldDefs can be nil for local structs without remote schema. func newStructSerializer(type_ reflect.Type, typeTag string, fieldDefs []FieldDef) *structSerializer { - if typeTag == "" && type_ != nil { - typeTag = type_.Name() - } - return &structSerializer{ - type_: type_, - typeTag: typeTag, - fieldDefs: fieldDefs, - } + if typeTag == "" && type_ != nil { + typeTag = type_.Name() + } + return &structSerializer{ + type_: type_, + typeTag: typeTag, + fieldDefs: fieldDefs, + } } // initialize performs eager initialization of the struct serializer. // This should be called at registration time to pre-compute all field metadata. func (s *structSerializer) initialize(typeResolver *TypeResolver) error { - if s.initialized { - return nil - } - - // Ensure type is set - if s.type_ == nil { - return errors.New("struct type not set") - } - - // Normalize pointer types - for s.type_.Kind() == reflect.Ptr { - s.type_ = s.type_.Elem() - } - - // Build fields from type or fieldDefs - if s.fieldDefs != nil { - if err := s.initFieldsFromDefsWithResolver(typeResolver); err != nil { - return err - } - } else { - if err := s.initFieldsFromTypeResolver(typeResolver); err != nil { - return err - } - } - - // Compute struct hash - s.structHash = s.computeHash() - - // Set compatible mode flag - s.isCompatibleMode = typeResolver.Compatible() - - s.initialized = true - return nil + if s.initialized { + return nil + } + + // Ensure type is set + if s.type_ == nil { + return errors.New("struct type not set") + } + + // Normalize pointer types + for s.type_.Kind() == reflect.Ptr { + s.type_ = s.type_.Elem() + } + + // Build fields from type or fieldDefs + if s.fieldDefs != nil { + if err := s.initFieldsFromDefsWithResolver(typeResolver); err != nil { + return err + } + } else { + if err := s.initFieldsFromTypeResolver(typeResolver); err != nil { + return err + } + } + + // Compute struct hash + s.structHash = s.computeHash() + + // Set compatible mode flag + s.isCompatibleMode = typeResolver.Compatible() + + s.initialized = true + return nil } func (s *structSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - switch refMode { - case RefModeTracking: - if value.Kind() == reflect.Ptr && value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return - } - refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) - if err != nil { - ctx.SetError(FromError(err)) - return - } - if refWritten { - return - } - case RefModeNullOnly: - if value.Kind() == reflect.Ptr && value.IsNil() { - ctx.buffer.WriteInt8(NullFlag) - return - } - ctx.buffer.WriteInt8(NotNullValueFlag) - } - if writeType { - // Structs have dynamic type IDs, need to look up from TypeResolver - typeInfo, err := ctx.TypeResolver().getTypeInfo(value, true) - if err != nil { - ctx.SetError(FromError(err)) - return - } - ctx.TypeResolver().WriteTypeInfo(ctx.buffer, typeInfo, ctx.Err()) - } - s.WriteData(ctx, value) + switch refMode { + case RefModeTracking: + if value.Kind() == reflect.Ptr && value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return + } + refWritten, err := ctx.RefResolver().WriteRefOrNull(ctx.buffer, value) + if err != nil { + ctx.SetError(FromError(err)) + return + } + if refWritten { + return + } + case RefModeNullOnly: + if value.Kind() == reflect.Ptr && value.IsNil() { + ctx.buffer.WriteInt8(NullFlag) + return + } + ctx.buffer.WriteInt8(NotNullValueFlag) + } + if writeType { + // Structs have dynamic type IDs, need to look up from TypeResolver + typeInfo, err := ctx.TypeResolver().getTypeInfo(value, true) + if err != nil { + ctx.SetError(FromError(err)) + return + } + ctx.TypeResolver().WriteTypeInfo(ctx.buffer, typeInfo, ctx.Err()) + } + s.WriteData(ctx, value) } func (s *structSerializer) WriteData(ctx *WriteContext, value reflect.Value) { - // Early error check - skip all intermediate checks for normal path performance - if ctx.HasError() { - return - } - - // Lazy initialization - if !s.initialized { - if err := s.initialize(ctx.TypeResolver()); err != nil { - ctx.SetError(FromError(err)) - return - } - } - - buf := ctx.Buffer() - - // Dereference pointer if needed - if value.Kind() == reflect.Ptr { - if value.IsNil() { - ctx.SetError(SerializationError("cannot write nil pointer")) - return - } - value = value.Elem() - } - - // In compatible mode with meta share, struct hash is not written - if !ctx.Compatible() { - buf.WriteInt32(s.structHash) - } - - // Check if value is addressable for unsafe access - canUseUnsafe := value.CanAddr() - var ptr unsafe.Pointer - if canUseUnsafe { - ptr = unsafe.Pointer(value.UnsafeAddr()) - } - - // ========================================================================== - // Phase 1: Fixed-size primitives (bool, int8, int16, float32, float64) - // - Reserve once, inline unsafe writes with endian handling, update index once - // - field.WriteOffset computed at init time - // ========================================================================== - if canUseUnsafe && s.fixedSize > 0 { - buf.Reserve(s.fixedSize) - baseOffset := buf.WriterIndex() - data := buf.GetData() - - for _, field := range s.fixedFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - bufOffset := baseOffset + field.WriteOffset - switch field.StaticId { - case ConcreteTypeBool: - if *(*bool)(fieldPtr) { - data[bufOffset] = 1 - } else { - data[bufOffset] = 0 - } - case ConcreteTypeInt8: - data[bufOffset] = *(*byte)(fieldPtr) - case ConcreteTypeInt16: - if isLittleEndian { - *(*int16)(unsafe.Pointer(&data[bufOffset])) = *(*int16)(fieldPtr) - } else { - binary.LittleEndian.PutUint16(data[bufOffset:], uint16(*(*int16)(fieldPtr))) - } - case ConcreteTypeFloat32: - if isLittleEndian { - *(*float32)(unsafe.Pointer(&data[bufOffset])) = *(*float32)(fieldPtr) - } else { - binary.LittleEndian.PutUint32(data[bufOffset:], math.Float32bits(*(*float32)(fieldPtr))) - } - case ConcreteTypeFloat64: - if isLittleEndian { - *(*float64)(unsafe.Pointer(&data[bufOffset])) = *(*float64)(fieldPtr) - } else { - binary.LittleEndian.PutUint64(data[bufOffset:], math.Float64bits(*(*float64)(fieldPtr))) - } - } - } - // Update writer index ONCE after all fixed fields - buf.SetWriterIndex(baseOffset + s.fixedSize) - } else if len(s.fixedFields) > 0 { - // Fallback to reflect-based access for unaddressable values - for _, field := range s.fixedFields { - fieldValue := value.Field(field.FieldIndex) - switch field.StaticId { - case ConcreteTypeBool: - buf.WriteBool(fieldValue.Bool()) - case ConcreteTypeInt8: - buf.WriteByte_(byte(fieldValue.Int())) - case ConcreteTypeInt16: - buf.WriteInt16(int16(fieldValue.Int())) - case ConcreteTypeFloat32: - buf.WriteFloat32(float32(fieldValue.Float())) - case ConcreteTypeFloat64: - buf.WriteFloat64(fieldValue.Float()) - } - } - } - - // ========================================================================== - // Phase 2: Varint primitives (int32, int64, int) - // - Reserve max size, track offset locally, update index once at end - // ========================================================================== - if canUseUnsafe && s.maxVarintSize > 0 { - buf.Reserve(s.maxVarintSize) - offset := buf.WriterIndex() - - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - offset += buf.UnsafePutVarInt32(offset, *(*int32)(fieldPtr)) - case ConcreteTypeInt64: - offset += buf.UnsafePutVarInt64(offset, *(*int64)(fieldPtr)) - case ConcreteTypeInt: - offset += buf.UnsafePutVarInt64(offset, int64(*(*int)(fieldPtr))) - } - } - // Update writer index ONCE after all varint fields - buf.SetWriterIndex(offset) - } else if len(s.varintFields) > 0 { - // Fallback to reflect-based access for unaddressable values - for _, field := range s.varintFields { - fieldValue := value.Field(field.FieldIndex) - switch field.StaticId { - case ConcreteTypeInt32: - buf.WriteVarint32(int32(fieldValue.Int())) - case ConcreteTypeInt64, ConcreteTypeInt: - buf.WriteVarint64(fieldValue.Int()) - } - } - } - - // ========================================================================== - // Phase 3: Remaining fields (strings, slices, maps, structs, enums) - // - These require per-field handling (ref flags, type info, serializers) - // - No intermediate error checks - trade error path performance for normal path - // ========================================================================== - for _, field := range s.remainingFields { - s.writeRemainingField(ctx, ptr, field, value) - } + // Early error check - skip all intermediate checks for normal path performance + if ctx.HasError() { + return + } + + // Lazy initialization + if !s.initialized { + if err := s.initialize(ctx.TypeResolver()); err != nil { + ctx.SetError(FromError(err)) + return + } + } + + buf := ctx.Buffer() + + // Dereference pointer if needed + if value.Kind() == reflect.Ptr { + if value.IsNil() { + ctx.SetError(SerializationError("cannot write nil pointer")) + return + } + value = value.Elem() + } + + // In compatible mode with meta share, struct hash is not written + if !ctx.Compatible() { + buf.WriteInt32(s.structHash) + } + + // Check if value is addressable for unsafe access + canUseUnsafe := value.CanAddr() + var ptr unsafe.Pointer + if canUseUnsafe { + ptr = unsafe.Pointer(value.UnsafeAddr()) + } + + // ========================================================================== + // Phase 1: Fixed-size primitives (bool, int8, int16, float32, float64) + // - Reserve once, inline unsafe writes with endian handling, update index once + // - field.WriteOffset computed at init time + // ========================================================================== + if canUseUnsafe && s.fixedSize > 0 { + buf.Reserve(s.fixedSize) + baseOffset := buf.WriterIndex() + data := buf.GetData() + + for _, field := range s.fixedFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + bufOffset := baseOffset + field.WriteOffset + switch field.StaticId { + case ConcreteTypeBool: + if *(*bool)(fieldPtr) { + data[bufOffset] = 1 + } else { + data[bufOffset] = 0 + } + case ConcreteTypeInt8: + data[bufOffset] = *(*byte)(fieldPtr) + case ConcreteTypeInt16: + if isLittleEndian { + *(*int16)(unsafe.Pointer(&data[bufOffset])) = *(*int16)(fieldPtr) + } else { + binary.LittleEndian.PutUint16(data[bufOffset:], uint16(*(*int16)(fieldPtr))) + } + case ConcreteTypeFloat32: + if isLittleEndian { + *(*float32)(unsafe.Pointer(&data[bufOffset])) = *(*float32)(fieldPtr) + } else { + binary.LittleEndian.PutUint32(data[bufOffset:], math.Float32bits(*(*float32)(fieldPtr))) + } + case ConcreteTypeFloat64: + if isLittleEndian { + *(*float64)(unsafe.Pointer(&data[bufOffset])) = *(*float64)(fieldPtr) + } else { + binary.LittleEndian.PutUint64(data[bufOffset:], math.Float64bits(*(*float64)(fieldPtr))) + } + } + } + // Update writer index ONCE after all fixed fields + buf.SetWriterIndex(baseOffset + s.fixedSize) + } else if len(s.fixedFields) > 0 { + // Fallback to reflect-based access for unaddressable values + for _, field := range s.fixedFields { + fieldValue := value.Field(field.FieldIndex) + switch field.StaticId { + case ConcreteTypeBool: + buf.WriteBool(fieldValue.Bool()) + case ConcreteTypeInt8: + buf.WriteByte_(byte(fieldValue.Int())) + case ConcreteTypeInt16: + buf.WriteInt16(int16(fieldValue.Int())) + case ConcreteTypeFloat32: + buf.WriteFloat32(float32(fieldValue.Float())) + case ConcreteTypeFloat64: + buf.WriteFloat64(fieldValue.Float()) + } + } + } + + // ========================================================================== + // Phase 2: Varint primitives (int32, int64, int) + // - Reserve max size, track offset locally, update index once at end + // ========================================================================== + if canUseUnsafe && s.maxVarintSize > 0 { + buf.Reserve(s.maxVarintSize) + offset := buf.WriterIndex() + + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + offset += buf.UnsafePutVarInt32(offset, *(*int32)(fieldPtr)) + case ConcreteTypeInt64: + offset += buf.UnsafePutVarInt64(offset, *(*int64)(fieldPtr)) + case ConcreteTypeInt: + offset += buf.UnsafePutVarInt64(offset, int64(*(*int)(fieldPtr))) + } + } + // Update writer index ONCE after all varint fields + buf.SetWriterIndex(offset) + } else if len(s.varintFields) > 0 { + // Fallback to reflect-based access for unaddressable values + for _, field := range s.varintFields { + fieldValue := value.Field(field.FieldIndex) + switch field.StaticId { + case ConcreteTypeInt32: + buf.WriteVarint32(int32(fieldValue.Int())) + case ConcreteTypeInt64, ConcreteTypeInt: + buf.WriteVarint64(fieldValue.Int()) + } + } + } + + // ========================================================================== + // Phase 3: Remaining fields (strings, slices, maps, structs, enums) + // - These require per-field handling (ref flags, type info, serializers) + // - No intermediate error checks - trade error path performance for normal path + // ========================================================================== + for _, field := range s.remainingFields { + s.writeRemainingField(ctx, ptr, field, value) + } } // writeRemainingField writes a non-primitive field (string, slice, map, struct, enum) func (s *structSerializer) writeRemainingField(ctx *WriteContext, ptr unsafe.Pointer, field *FieldInfo, value reflect.Value) { - buf := ctx.Buffer() - - // Fast path dispatch using pre-computed StaticId - // ptr must be valid (addressable value) - if ptr != nil { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeString: - if field.RefMode == RefModeTracking { - break // Fall through to slow path - } - // Only write null flag if RefMode requires it (nullable field) - if field.RefMode == RefModeNullOnly { - buf.WriteInt8(NotNullValueFlag) - } - ctx.WriteString(*(*string)(fieldPtr)) - return - case ConcreteTypeEnum: - // Enums don't track refs - always use fast path - writeEnumField(ctx, field, value.Field(field.FieldIndex)) - return - case ConcreteTypeStringSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringSlice(*(*[]string)(fieldPtr), field.RefMode, false, true) - return - case ConcreteTypeBoolSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteBoolSlice(*(*[]bool)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt8Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt8Slice(*(*[]int8)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeByteSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteByteSlice(*(*[]byte)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt16Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt16Slice(*(*[]int16)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt32Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt32Slice(*(*[]int32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeInt64Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteInt64Slice(*(*[]int64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeIntSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteIntSlice(*(*[]int)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeUintSlice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteUintSlice(*(*[]uint)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeFloat32Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteFloat32Slice(*(*[]float32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeFloat64Slice: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteFloat64Slice(*(*[]float64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringStringMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringStringMap(*(*map[string]string)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringInt64Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringInt64Map(*(*map[string]int64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringInt32Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringInt32Map(*(*map[string]int32)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringIntMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringIntMap(*(*map[string]int)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringFloat64Map: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteStringFloat64Map(*(*map[string]float64)(fieldPtr), field.RefMode, false) - return - case ConcreteTypeStringBoolMap: - // NOTE: map[string]bool is used to represent SETs in Go xlang mode. - // We CANNOT use the fast path here because it writes MAP format, - // but the data should be written in SET format. Fall through to slow path - // which uses setSerializer to correctly write the SET format. - break - case ConcreteTypeIntIntMap: - if field.RefMode == RefModeTracking { - break - } - ctx.WriteIntIntMap(*(*map[int]int)(fieldPtr), field.RefMode, false) - return - } - } - - // Slow path: use full serializer - fieldValue := value.Field(field.FieldIndex) - if field.Serializer != nil { - field.Serializer.Write(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.WriteValue(fieldValue, RefModeTracking, true) - } + buf := ctx.Buffer() + + // Fast path dispatch using pre-computed StaticId + // ptr must be valid (addressable value) + if ptr != nil { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeString: + if field.RefMode == RefModeTracking { + break // Fall through to slow path + } + // Only write null flag if RefMode requires it (nullable field) + if field.RefMode == RefModeNullOnly { + buf.WriteInt8(NotNullValueFlag) + } + ctx.WriteString(*(*string)(fieldPtr)) + return + case ConcreteTypeEnum: + // Enums don't track refs - always use fast path + writeEnumField(ctx, field, value.Field(field.FieldIndex)) + return + case ConcreteTypeStringSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringSlice(*(*[]string)(fieldPtr), field.RefMode, false, true) + return + case ConcreteTypeBoolSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteBoolSlice(*(*[]bool)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt8Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt8Slice(*(*[]int8)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeByteSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteByteSlice(*(*[]byte)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt16Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt16Slice(*(*[]int16)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt32Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt32Slice(*(*[]int32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeInt64Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteInt64Slice(*(*[]int64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeIntSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteIntSlice(*(*[]int)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeUintSlice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteUintSlice(*(*[]uint)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeFloat32Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteFloat32Slice(*(*[]float32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeFloat64Slice: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteFloat64Slice(*(*[]float64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringStringMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringStringMap(*(*map[string]string)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringInt64Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringInt64Map(*(*map[string]int64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringInt32Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringInt32Map(*(*map[string]int32)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringIntMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringIntMap(*(*map[string]int)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringFloat64Map: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteStringFloat64Map(*(*map[string]float64)(fieldPtr), field.RefMode, false) + return + case ConcreteTypeStringBoolMap: + // NOTE: map[string]bool is used to represent SETs in Go xlang mode. + // We CANNOT use the fast path here because it writes MAP format, + // but the data should be written in SET format. Fall through to slow path + // which uses setSerializer to correctly write the SET format. + break + case ConcreteTypeIntIntMap: + if field.RefMode == RefModeTracking { + break + } + ctx.WriteIntIntMap(*(*map[int]int)(fieldPtr), field.RefMode, false) + return + } + } + + // Slow path: use full serializer + fieldValue := value.Field(field.FieldIndex) + if field.Serializer != nil { + field.Serializer.Write(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.WriteValue(fieldValue, RefModeTracking, true) + } } func (s *structSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found - obj := ctx.RefResolver().GetReadObject(refID) - if obj.IsValid() { - value.Set(obj) - } - return - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return - } - } - if readType { - // Read type info - in compatible mode this returns the serializer with remote fieldDefs - typeID := buf.ReadVaruint32Small7(ctxErr) - internalTypeID := TypeId(typeID & 0xFF) - // Check if this is a struct type that needs type meta reading - if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { - // For struct types in compatible mode, use the serializer from TypeInfo - typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) - // Use the serializer from TypeInfo which has the remote field definitions - if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { - structSer.ReadData(ctx, value.Type(), value) - return - } - } - } - s.ReadData(ctx, value.Type(), value) + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) + if refErr != nil { + ctx.SetError(FromError(refErr)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found + obj := ctx.RefResolver().GetReadObject(refID) + if obj.IsValid() { + value.Set(obj) + } + return + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return + } + } + if readType { + // Read type info - in compatible mode this returns the serializer with remote fieldDefs + typeID := buf.ReadVaruint32Small7(ctxErr) + internalTypeID := TypeId(typeID & 0xFF) + // Check if this is a struct type that needs type meta reading + if IsNamespacedType(TypeId(typeID)) || internalTypeID == COMPATIBLE_STRUCT || internalTypeID == STRUCT { + // For struct types in compatible mode, use the serializer from TypeInfo + typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(buf, typeID, ctxErr) + // Use the serializer from TypeInfo which has the remote field definitions + if structSer, ok := typeInfo.Serializer.(*structSerializer); ok && len(structSer.fieldDefs) > 0 { + structSer.ReadData(ctx, value.Type(), value) + return + } + } + } + s.ReadData(ctx, value.Type(), value) } func (s *structSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { - // Early error check - skip all intermediate checks for normal path performance - if ctx.HasError() { - return - } - - // Lazy initialization - if !s.initialized { - if err := s.initialize(ctx.TypeResolver()); err != nil { - ctx.SetError(FromError(err)) - return - } - } - - buf := ctx.Buffer() - if value.Kind() == reflect.Ptr { - if value.IsNil() { - value.Set(reflect.New(type_.Elem())) - } - value = value.Elem() - type_ = type_.Elem() - } - - // In compatible mode with meta share, struct hash is not written - if !ctx.Compatible() { - err := ctx.Err() - structHash := buf.ReadInt32(err) - if structHash != s.structHash { - ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) - return - } - } - - // Use ordered reading only when TypeDef differs from local type (schema evolution) - // When types match (typeDefDiffers=false), use grouped reading for better performance - if s.typeDefDiffers { - s.readFieldsInOrder(ctx, value) - return - } - - // Check if value is addressable for unsafe access - if !value.CanAddr() { - s.readFieldsInOrder(ctx, value) - return - } - - // ========================================================================== - // Grouped reading for matching types (optimized path) - // - Types match, so all fields exist locally (no FieldIndex < 0 checks) - // - Use UnsafeGet at pre-computed offsets, update reader index once per phase - // ========================================================================== - ptr := unsafe.Pointer(value.UnsafeAddr()) - - // Phase 1: Fixed-size primitives (inline unsafe reads with endian handling) - if s.fixedSize > 0 { - baseOffset := buf.ReaderIndex() - data := buf.GetData() - - for _, field := range s.fixedFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - bufOffset := baseOffset + field.WriteOffset - switch field.StaticId { - case ConcreteTypeBool: - *(*bool)(fieldPtr) = data[bufOffset] != 0 - case ConcreteTypeInt8: - *(*int8)(fieldPtr) = int8(data[bufOffset]) - case ConcreteTypeInt16: - if isLittleEndian { - *(*int16)(fieldPtr) = *(*int16)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*int16)(fieldPtr) = int16(binary.LittleEndian.Uint16(data[bufOffset:])) - } - case ConcreteTypeFloat32: - if isLittleEndian { - *(*float32)(fieldPtr) = *(*float32)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*float32)(fieldPtr) = math.Float32frombits(binary.LittleEndian.Uint32(data[bufOffset:])) - } - case ConcreteTypeFloat64: - if isLittleEndian { - *(*float64)(fieldPtr) = *(*float64)(unsafe.Pointer(&data[bufOffset])) - } else { - *(*float64)(fieldPtr) = math.Float64frombits(binary.LittleEndian.Uint64(data[bufOffset:])) - } - } - } - // Update reader index ONCE after all fixed fields - buf.SetReaderIndex(baseOffset + s.fixedSize) - } - - // Phase 2: Varint primitives (must read sequentially - variable length) - // Use unsafe reads when we have enough buffer remaining - if s.maxVarintSize > 0 && buf.remaining() >= s.maxVarintSize { - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.UnsafeReadVarint32() - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.UnsafeReadVarint64() - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.UnsafeReadVarint64()) - } - } - } else if len(s.varintFields) > 0 { - // Slow path with bounds checking - err := ctx.Err() - for _, field := range s.varintFields { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.ReadVarint32(err) - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.ReadVarint64(err) - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) - } - } - } - - // Phase 3: Remaining fields (strings, slices, maps, structs, enums) - // No intermediate error checks - trade error path performance for normal path - for _, field := range s.remainingFields { - s.readRemainingField(ctx, ptr, field, value) - } + // Early error check - skip all intermediate checks for normal path performance + if ctx.HasError() { + return + } + + // Lazy initialization + if !s.initialized { + if err := s.initialize(ctx.TypeResolver()); err != nil { + ctx.SetError(FromError(err)) + return + } + } + + buf := ctx.Buffer() + if value.Kind() == reflect.Ptr { + if value.IsNil() { + value.Set(reflect.New(type_.Elem())) + } + value = value.Elem() + type_ = type_.Elem() + } + + // In compatible mode with meta share, struct hash is not written + if !ctx.Compatible() { + err := ctx.Err() + structHash := buf.ReadInt32(err) + if structHash != s.structHash { + ctx.SetError(HashMismatchError(structHash, s.structHash, s.type_.String())) + return + } + } + + // Use ordered reading only when TypeDef differs from local type (schema evolution) + // When types match (typeDefDiffers=false), use grouped reading for better performance + if s.typeDefDiffers { + s.readFieldsInOrder(ctx, value) + return + } + + // Check if value is addressable for unsafe access + if !value.CanAddr() { + s.readFieldsInOrder(ctx, value) + return + } + + // ========================================================================== + // Grouped reading for matching types (optimized path) + // - Types match, so all fields exist locally (no FieldIndex < 0 checks) + // - Use UnsafeGet at pre-computed offsets, update reader index once per phase + // ========================================================================== + ptr := unsafe.Pointer(value.UnsafeAddr()) + + // Phase 1: Fixed-size primitives (inline unsafe reads with endian handling) + if s.fixedSize > 0 { + baseOffset := buf.ReaderIndex() + data := buf.GetData() + + for _, field := range s.fixedFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + bufOffset := baseOffset + field.WriteOffset + switch field.StaticId { + case ConcreteTypeBool: + *(*bool)(fieldPtr) = data[bufOffset] != 0 + case ConcreteTypeInt8: + *(*int8)(fieldPtr) = int8(data[bufOffset]) + case ConcreteTypeInt16: + if isLittleEndian { + *(*int16)(fieldPtr) = *(*int16)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*int16)(fieldPtr) = int16(binary.LittleEndian.Uint16(data[bufOffset:])) + } + case ConcreteTypeFloat32: + if isLittleEndian { + *(*float32)(fieldPtr) = *(*float32)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*float32)(fieldPtr) = math.Float32frombits(binary.LittleEndian.Uint32(data[bufOffset:])) + } + case ConcreteTypeFloat64: + if isLittleEndian { + *(*float64)(fieldPtr) = *(*float64)(unsafe.Pointer(&data[bufOffset])) + } else { + *(*float64)(fieldPtr) = math.Float64frombits(binary.LittleEndian.Uint64(data[bufOffset:])) + } + } + } + // Update reader index ONCE after all fixed fields + buf.SetReaderIndex(baseOffset + s.fixedSize) + } + + // Phase 2: Varint primitives (must read sequentially - variable length) + // Use unsafe reads when we have enough buffer remaining + if s.maxVarintSize > 0 && buf.remaining() >= s.maxVarintSize { + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.UnsafeReadVarint32() + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.UnsafeReadVarint64() + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.UnsafeReadVarint64()) + } + } + } else if len(s.varintFields) > 0 { + // Slow path with bounds checking + err := ctx.Err() + for _, field := range s.varintFields { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.ReadVarint32(err) + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.ReadVarint64(err) + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) + } + } + } + + // Phase 3: Remaining fields (strings, slices, maps, structs, enums) + // No intermediate error checks - trade error path performance for normal path + for _, field := range s.remainingFields { + s.readRemainingField(ctx, ptr, field, value) + } } // readRemainingField reads a non-primitive field (string, slice, map, struct, enum) func (s *structSerializer) readRemainingField(ctx *ReadContext, ptr unsafe.Pointer, field *FieldInfo, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - - // Fast path dispatch using pre-computed StaticId - // ptr must be valid (addressable value) - if ptr != nil { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeString: - if field.RefMode == RefModeTracking { - break // Fall through to slow path for ref tracking - } - // Only read null flag if RefMode requires it (nullable field) - if field.RefMode == RefModeNullOnly { - refFlag := buf.ReadInt8(ctxErr) - if refFlag == NullFlag { - *(*string)(fieldPtr) = "" - return - } - } - *(*string)(fieldPtr) = ctx.ReadString() - return - case ConcreteTypeEnum: - // Enums don't track refs - always use fast path - fieldValue := value.Field(field.FieldIndex) - readEnumField(ctx, field, fieldValue) - return - case ConcreteTypeStringSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]string)(fieldPtr) = ctx.ReadStringSlice(field.RefMode, false) - return - case ConcreteTypeBoolSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]bool)(fieldPtr) = ctx.ReadBoolSlice(field.RefMode, false) - return - case ConcreteTypeInt8Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int8)(fieldPtr) = ctx.ReadInt8Slice(field.RefMode, false) - return - case ConcreteTypeByteSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]byte)(fieldPtr) = ctx.ReadByteSlice(field.RefMode, false) - return - case ConcreteTypeInt16Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int16)(fieldPtr) = ctx.ReadInt16Slice(field.RefMode, false) - return - case ConcreteTypeInt32Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int32)(fieldPtr) = ctx.ReadInt32Slice(field.RefMode, false) - return - case ConcreteTypeInt64Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int64)(fieldPtr) = ctx.ReadInt64Slice(field.RefMode, false) - return - case ConcreteTypeIntSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]int)(fieldPtr) = ctx.ReadIntSlice(field.RefMode, false) - return - case ConcreteTypeUintSlice: - if field.RefMode == RefModeTracking { - break - } - *(*[]uint)(fieldPtr) = ctx.ReadUintSlice(field.RefMode, false) - return - case ConcreteTypeFloat32Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]float32)(fieldPtr) = ctx.ReadFloat32Slice(field.RefMode, false) - return - case ConcreteTypeFloat64Slice: - if field.RefMode == RefModeTracking { - break - } - *(*[]float64)(fieldPtr) = ctx.ReadFloat64Slice(field.RefMode, false) - return - case ConcreteTypeStringStringMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]string)(fieldPtr) = ctx.ReadStringStringMap(field.RefMode, false) - return - case ConcreteTypeStringInt64Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int64)(fieldPtr) = ctx.ReadStringInt64Map(field.RefMode, false) - return - case ConcreteTypeStringInt32Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int32)(fieldPtr) = ctx.ReadStringInt32Map(field.RefMode, false) - return - case ConcreteTypeStringIntMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]int)(fieldPtr) = ctx.ReadStringIntMap(field.RefMode, false) - return - case ConcreteTypeStringFloat64Map: - if field.RefMode == RefModeTracking { - break - } - *(*map[string]float64)(fieldPtr) = ctx.ReadStringFloat64Map(field.RefMode, false) - return - case ConcreteTypeStringBoolMap: - // NOTE: map[string]bool is used to represent SETs in Go xlang mode. - // We CANNOT use the fast path here because it reads MAP format, - // but the data is actually in SET format. Fall through to slow path - // which uses setSerializer to correctly read the SET format. - break - case ConcreteTypeIntIntMap: - if field.RefMode == RefModeTracking { - break - } - *(*map[int]int)(fieldPtr) = ctx.ReadIntIntMap(field.RefMode, false) - return - } - } - - // Slow path: use full serializer - fieldValue := value.Field(field.FieldIndex) - - if field.Serializer != nil { - field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.ReadValue(fieldValue, RefModeTracking, true) - } + buf := ctx.Buffer() + ctxErr := ctx.Err() + + // Fast path dispatch using pre-computed StaticId + // ptr must be valid (addressable value) + if ptr != nil { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeString: + if field.RefMode == RefModeTracking { + break // Fall through to slow path for ref tracking + } + // Only read null flag if RefMode requires it (nullable field) + if field.RefMode == RefModeNullOnly { + refFlag := buf.ReadInt8(ctxErr) + if refFlag == NullFlag { + *(*string)(fieldPtr) = "" + return + } + } + *(*string)(fieldPtr) = ctx.ReadString() + return + case ConcreteTypeEnum: + // Enums don't track refs - always use fast path + fieldValue := value.Field(field.FieldIndex) + readEnumField(ctx, field, fieldValue) + return + case ConcreteTypeStringSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]string)(fieldPtr) = ctx.ReadStringSlice(field.RefMode, false) + return + case ConcreteTypeBoolSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]bool)(fieldPtr) = ctx.ReadBoolSlice(field.RefMode, false) + return + case ConcreteTypeInt8Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int8)(fieldPtr) = ctx.ReadInt8Slice(field.RefMode, false) + return + case ConcreteTypeByteSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]byte)(fieldPtr) = ctx.ReadByteSlice(field.RefMode, false) + return + case ConcreteTypeInt16Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int16)(fieldPtr) = ctx.ReadInt16Slice(field.RefMode, false) + return + case ConcreteTypeInt32Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int32)(fieldPtr) = ctx.ReadInt32Slice(field.RefMode, false) + return + case ConcreteTypeInt64Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int64)(fieldPtr) = ctx.ReadInt64Slice(field.RefMode, false) + return + case ConcreteTypeIntSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]int)(fieldPtr) = ctx.ReadIntSlice(field.RefMode, false) + return + case ConcreteTypeUintSlice: + if field.RefMode == RefModeTracking { + break + } + *(*[]uint)(fieldPtr) = ctx.ReadUintSlice(field.RefMode, false) + return + case ConcreteTypeFloat32Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]float32)(fieldPtr) = ctx.ReadFloat32Slice(field.RefMode, false) + return + case ConcreteTypeFloat64Slice: + if field.RefMode == RefModeTracking { + break + } + *(*[]float64)(fieldPtr) = ctx.ReadFloat64Slice(field.RefMode, false) + return + case ConcreteTypeStringStringMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]string)(fieldPtr) = ctx.ReadStringStringMap(field.RefMode, false) + return + case ConcreteTypeStringInt64Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int64)(fieldPtr) = ctx.ReadStringInt64Map(field.RefMode, false) + return + case ConcreteTypeStringInt32Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int32)(fieldPtr) = ctx.ReadStringInt32Map(field.RefMode, false) + return + case ConcreteTypeStringIntMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]int)(fieldPtr) = ctx.ReadStringIntMap(field.RefMode, false) + return + case ConcreteTypeStringFloat64Map: + if field.RefMode == RefModeTracking { + break + } + *(*map[string]float64)(fieldPtr) = ctx.ReadStringFloat64Map(field.RefMode, false) + return + case ConcreteTypeStringBoolMap: + // NOTE: map[string]bool is used to represent SETs in Go xlang mode. + // We CANNOT use the fast path here because it reads MAP format, + // but the data is actually in SET format. Fall through to slow path + // which uses setSerializer to correctly read the SET format. + break + case ConcreteTypeIntIntMap: + if field.RefMode == RefModeTracking { + break + } + *(*map[int]int)(fieldPtr) = ctx.ReadIntIntMap(field.RefMode, false) + return + } + } + + // Slow path: use full serializer + fieldValue := value.Field(field.FieldIndex) + + if field.Serializer != nil { + field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.ReadValue(fieldValue, RefModeTracking, true) + } } // readFieldsInOrder reads fields in the order they appear in s.fields (TypeDef order) // This is used in compatible mode where Java writes fields in TypeDef order func (s *structSerializer) readFieldsInOrder(ctx *ReadContext, value reflect.Value) { - buf := ctx.Buffer() - canUseUnsafe := value.CanAddr() - var ptr unsafe.Pointer - if canUseUnsafe { - ptr = unsafe.Pointer(value.UnsafeAddr()) - } - err := ctx.Err() - - for _, field := range s.fields { - if field.FieldIndex < 0 { - s.skipField(ctx, field) - if ctx.HasError() { - return - } - continue - } - - // Fast path for fixed-size primitive types (no ref flag) - // Use error-aware methods with deferred checking - if canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeBool: - *(*bool)(fieldPtr) = buf.ReadBool(err) - case ConcreteTypeInt8: - *(*int8)(fieldPtr) = buf.ReadInt8(err) - case ConcreteTypeInt16: - *(*int16)(fieldPtr) = buf.ReadInt16(err) - case ConcreteTypeFloat32: - *(*float32)(fieldPtr) = buf.ReadFloat32(err) - case ConcreteTypeFloat64: - *(*float64)(fieldPtr) = buf.ReadFloat64(err) - } - continue - } - - // Fast path for varint primitive types (no ref flag) - // Skip fast path if field has a serializer with a non-primitive type (e.g., NAMED_ENUM) - if canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { - fieldPtr := unsafe.Add(ptr, field.Offset) - switch field.StaticId { - case ConcreteTypeInt32: - *(*int32)(fieldPtr) = buf.ReadVarint32(err) - case ConcreteTypeInt64: - *(*int64)(fieldPtr) = buf.ReadVarint64(err) - case ConcreteTypeInt: - *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) - } - continue - } - - // Get field value for slow paths - fieldValue := value.Field(field.FieldIndex) - - // Slow path for primitives when not addressable - if !canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { - switch field.StaticId { - case ConcreteTypeBool: - fieldValue.SetBool(buf.ReadBool(err)) - case ConcreteTypeInt8: - fieldValue.SetInt(int64(buf.ReadInt8(err))) - case ConcreteTypeInt16: - fieldValue.SetInt(int64(buf.ReadInt16(err))) - case ConcreteTypeFloat32: - fieldValue.SetFloat(float64(buf.ReadFloat32(err))) - case ConcreteTypeFloat64: - fieldValue.SetFloat(buf.ReadFloat64(err)) - } - continue - } - - if !canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { - switch field.StaticId { - case ConcreteTypeInt32: - fieldValue.SetInt(int64(buf.ReadVarint32(err))) - case ConcreteTypeInt64, ConcreteTypeInt: - fieldValue.SetInt(buf.ReadVarint64(err)) - } - continue - } - - if isEnumField(field) { - readEnumField(ctx, field, fieldValue) - continue - } - - // Slow path for non-primitives (all need ref flag per xlang spec) - if field.Serializer != nil { - // Use pre-computed RefMode and WriteType from field initialization - field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) - } else { - ctx.ReadValue(fieldValue, RefModeTracking, true) - } - } + buf := ctx.Buffer() + canUseUnsafe := value.CanAddr() + var ptr unsafe.Pointer + if canUseUnsafe { + ptr = unsafe.Pointer(value.UnsafeAddr()) + } + err := ctx.Err() + + for _, field := range s.fields { + if field.FieldIndex < 0 { + s.skipField(ctx, field) + if ctx.HasError() { + return + } + continue + } + + // Fast path for fixed-size primitive types (no ref flag) + // Use error-aware methods with deferred checking + if canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeBool: + *(*bool)(fieldPtr) = buf.ReadBool(err) + case ConcreteTypeInt8: + *(*int8)(fieldPtr) = buf.ReadInt8(err) + case ConcreteTypeInt16: + *(*int16)(fieldPtr) = buf.ReadInt16(err) + case ConcreteTypeFloat32: + *(*float32)(fieldPtr) = buf.ReadFloat32(err) + case ConcreteTypeFloat64: + *(*float64)(fieldPtr) = buf.ReadFloat64(err) + } + continue + } + + // Fast path for varint primitive types (no ref flag) + // Skip fast path if field has a serializer with a non-primitive type (e.g., NAMED_ENUM) + if canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { + fieldPtr := unsafe.Add(ptr, field.Offset) + switch field.StaticId { + case ConcreteTypeInt32: + *(*int32)(fieldPtr) = buf.ReadVarint32(err) + case ConcreteTypeInt64: + *(*int64)(fieldPtr) = buf.ReadVarint64(err) + case ConcreteTypeInt: + *(*int)(fieldPtr) = int(buf.ReadVarint64(err)) + } + continue + } + + // Get field value for slow paths + fieldValue := value.Field(field.FieldIndex) + + // Slow path for primitives when not addressable + if !canUseUnsafe && isFixedSizePrimitive(field.StaticId, field.Referencable) { + switch field.StaticId { + case ConcreteTypeBool: + fieldValue.SetBool(buf.ReadBool(err)) + case ConcreteTypeInt8: + fieldValue.SetInt(int64(buf.ReadInt8(err))) + case ConcreteTypeInt16: + fieldValue.SetInt(int64(buf.ReadInt16(err))) + case ConcreteTypeFloat32: + fieldValue.SetFloat(float64(buf.ReadFloat32(err))) + case ConcreteTypeFloat64: + fieldValue.SetFloat(buf.ReadFloat64(err)) + } + continue + } + + if !canUseUnsafe && isVarintPrimitive(field.StaticId, field.Referencable) && !fieldHasNonPrimitiveSerializer(field) { + switch field.StaticId { + case ConcreteTypeInt32: + fieldValue.SetInt(int64(buf.ReadVarint32(err))) + case ConcreteTypeInt64, ConcreteTypeInt: + fieldValue.SetInt(buf.ReadVarint64(err)) + } + continue + } + + if isEnumField(field) { + readEnumField(ctx, field, fieldValue) + continue + } + + // Slow path for non-primitives (all need ref flag per xlang spec) + if field.Serializer != nil { + // Use pre-computed RefMode and WriteType from field initialization + field.Serializer.Read(ctx, field.RefMode, field.WriteType, field.HasGenerics, fieldValue) + } else { + ctx.ReadValue(fieldValue, RefModeTracking, true) + } + } } // skipField skips a field that doesn't exist or is incompatible // Uses context error state for deferred error checking. func (s *structSerializer) skipField(ctx *ReadContext, field *FieldInfo) { - if field.FieldDef.name != "" { - fieldDefIsStructType := isStructFieldType(field.FieldDef.fieldType) - // Use FieldDef's trackingRef and nullable to determine if ref flag was written by Java - // Java writes ref flag based on its FieldDef, not Go's field type - readRefFlag := field.FieldDef.trackingRef || field.FieldDef.nullable - SkipFieldValueWithTypeFlag(ctx, field.FieldDef, readRefFlag, ctx.Compatible() && fieldDefIsStructType) - return - } - // No FieldDef available, read into temp value - tempValue := reflect.New(field.Type).Elem() - if field.Serializer != nil { - readType := ctx.Compatible() && isStructField(field.Type) - refMode := RefModeNone - if field.Referencable { - refMode = RefModeTracking - } - field.Serializer.Read(ctx, refMode, readType, false, tempValue) - } else { - ctx.ReadValue(tempValue, RefModeTracking, true) - } + if field.FieldDef.name != "" { + fieldDefIsStructType := isStructFieldType(field.FieldDef.fieldType) + // Use FieldDef's trackingRef and nullable to determine if ref flag was written by Java + // Java writes ref flag based on its FieldDef, not Go's field type + readRefFlag := field.FieldDef.trackingRef || field.FieldDef.nullable + SkipFieldValueWithTypeFlag(ctx, field.FieldDef, readRefFlag, ctx.Compatible() && fieldDefIsStructType) + return + } + // No FieldDef available, read into temp value + tempValue := reflect.New(field.Type).Elem() + if field.Serializer != nil { + readType := ctx.Compatible() && isStructField(field.Type) + refMode := RefModeNone + if field.Referencable { + refMode = RefModeTracking + } + field.Serializer.Read(ctx, refMode, readType, false, tempValue) + } else { + ctx.ReadValue(tempValue, RefModeTracking, true) + } } func (s *structSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again - s.Read(ctx, refMode, false, false, value) + // typeInfo is already read, don't read it again + s.Read(ctx, refMode, false, false, value) } // initFieldsFromContext initializes fields using context's type resolver (for WriteContext) // initFieldsFromTypeResolver initializes fields from local struct type using TypeResolver func (s *structSerializer) initFieldsFromTypeResolver(typeResolver *TypeResolver) error { - // If we have fieldDefs from type_def (remote meta), use them - if len(s.fieldDefs) > 0 { - return s.initFieldsFromDefsWithResolver(typeResolver) - } - - // Otherwise initialize from local struct type - type_ := s.type_ - var fields []*FieldInfo - var fieldNames []string - var serializers []Serializer - var typeIds []TypeId - var nullables []bool - var tagIDs []int - - for i := 0; i < type_.NumField(); i++ { - field := type_.Field(i) - firstRune, _ := utf8.DecodeRuneInString(field.Name) - if unicode.IsLower(firstRune) { - continue // skip unexported fields - } - - // Parse fory struct tag and check for ignore - foryTag := ParseForyTag(field) - if foryTag.Ignore { - continue // skip ignored fields - } - - fieldType := field.Type - - var fieldSerializer Serializer - // For interface{} fields, don't get a serializer - use WriteValue/ReadValue instead - // which will handle polymorphic types dynamically - if fieldType.Kind() != reflect.Interface { - // Get serializer for all non-interface field types - fieldSerializer, _ = typeResolver.getSerializerByType(fieldType, true) - } - - // Use TypeResolver helper methods for arrays and slices - if fieldType.Kind() == reflect.Array && fieldType.Elem().Kind() != reflect.Interface { - fieldSerializer, _ = typeResolver.GetArraySerializer(fieldType) - } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() != reflect.Interface { - fieldSerializer, _ = typeResolver.GetSliceSerializer(fieldType) - } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.Interface { - // For struct fields with interface element types, use sliceDynSerializer - fieldSerializer = mustNewSliceDynSerializer(fieldType.Elem()) - } - - // Get TypeId for the serializer, fallback to deriving from kind - fieldTypeId := typeResolver.getTypeIdByType(fieldType) - if fieldTypeId == 0 { - fieldTypeId = typeIdFromKind(fieldType) - } - // Calculate nullable flag for serialization (wire format): - // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. - // Only pointer types are nullable by default. - // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, - // so they are nullable by default. - // Can be overridden by explicit fory tag `fory:"nullable"`. - internalId := TypeId(fieldTypeId & 0xFF) - isEnum := internalId == ENUM || internalId == NAMED_ENUM - - // Determine nullable based on mode - // In xlang mode: only pointer types are nullable by default (per xlang spec) - // In native mode: Go's natural semantics - all nil-able types are nullable - // This ensures proper interoperability with Java/other languages in xlang mode. - var nullableFlag bool - if typeResolver.fory.config.IsXlang { - // xlang mode: only pointer types are nullable by default per xlang spec - // Slices and maps are NOT nullable - they serialize as empty when nil - nullableFlag = fieldType.Kind() == reflect.Ptr - } else { - // Native mode: Go's natural semantics - all nil-able types are nullable - nullableFlag = fieldType.Kind() == reflect.Ptr || - fieldType.Kind() == reflect.Slice || - fieldType.Kind() == reflect.Map || - fieldType.Kind() == reflect.Interface - } - if foryTag.NullableSet { - // Override nullable flag if explicitly set in fory tag - nullableFlag = foryTag.Nullable - } - // Primitives are never nullable, regardless of tag - if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum { - nullableFlag = false - } - - // Calculate ref tracking - use tag override if explicitly set - trackRef := typeResolver.TrackRef() - if foryTag.RefSet { - trackRef = foryTag.Ref - } - - // Pre-compute RefMode based on (possibly overridden) trackRef and nullable - // For pointer-to-struct fields, enable ref tracking when trackRef is enabled, - // regardless of nullable flag. This is necessary to detect circular references. - refMode := RefModeNone - isStructPointer := fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.Struct - if trackRef && (nullableFlag || isStructPointer) { - refMode = RefModeTracking - } else if nullableFlag { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(fieldType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(fieldType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] initFieldsFromTypeResolver: field=%s type=%v staticId=%d refMode=%v nullableFlag=%v serializer=%T\n", - SnakeCase(field.Name), fieldType, staticId, refMode, nullableFlag, fieldSerializer) - } - - fieldInfo := &FieldInfo{ - Name: SnakeCase(field.Name), - Offset: field.Offset, - Type: fieldType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: nullableFlag, // Use same logic as TypeDef's nullable flag for consistent ref handling - FieldIndex: i, - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - TagID: foryTag.ID, - HasForyTag: foryTag.HasTag, - TagRefSet: foryTag.RefSet, - TagRef: foryTag.Ref, - TagNullableSet: foryTag.NullableSet, - TagNullable: foryTag.Nullable, - } - fields = append(fields, fieldInfo) - fieldNames = append(fieldNames, fieldInfo.Name) - serializers = append(serializers, fieldSerializer) - typeIds = append(typeIds, fieldTypeId) - nullables = append(nullables, nullableFlag) - tagIDs = append(tagIDs, foryTag.ID) - } - - // Sort fields according to specification using nullable info and tag IDs for consistent ordering - serializers, fieldNames = sortFields(typeResolver, fieldNames, serializers, typeIds, nullables, tagIDs) - order := make(map[string]int, len(fieldNames)) - for idx, name := range fieldNames { - order[name] = idx - } - - sort.SliceStable(fields, func(i, j int) bool { - oi, okI := order[fields[i].Name] - oj, okJ := order[fields[j].Name] - switch { - case okI && okJ: - return oi < oj - case okI: - return true - case okJ: - return false - default: - return false - } - }) - - s.fields = fields - s.groupFields() - return nil + // If we have fieldDefs from type_def (remote meta), use them + if len(s.fieldDefs) > 0 { + return s.initFieldsFromDefsWithResolver(typeResolver) + } + + // Otherwise initialize from local struct type + type_ := s.type_ + var fields []*FieldInfo + var fieldNames []string + var serializers []Serializer + var typeIds []TypeId + var nullables []bool + var tagIDs []int + + for i := 0; i < type_.NumField(); i++ { + field := type_.Field(i) + firstRune, _ := utf8.DecodeRuneInString(field.Name) + if unicode.IsLower(firstRune) { + continue // skip unexported fields + } + + // Parse fory struct tag and check for ignore + foryTag := ParseForyTag(field) + if foryTag.Ignore { + continue // skip ignored fields + } + + fieldType := field.Type + + var fieldSerializer Serializer + // For interface{} fields, don't get a serializer - use WriteValue/ReadValue instead + // which will handle polymorphic types dynamically + if fieldType.Kind() != reflect.Interface { + // Get serializer for all non-interface field types + fieldSerializer, _ = typeResolver.getSerializerByType(fieldType, true) + } + + // Use TypeResolver helper methods for arrays and slices + if fieldType.Kind() == reflect.Array && fieldType.Elem().Kind() != reflect.Interface { + fieldSerializer, _ = typeResolver.GetArraySerializer(fieldType) + } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() != reflect.Interface { + fieldSerializer, _ = typeResolver.GetSliceSerializer(fieldType) + } else if fieldType.Kind() == reflect.Slice && fieldType.Elem().Kind() == reflect.Interface { + // For struct fields with interface element types, use sliceDynSerializer + fieldSerializer = mustNewSliceDynSerializer(fieldType.Elem()) + } + + // Get TypeId for the serializer, fallback to deriving from kind + fieldTypeId := typeResolver.getTypeIdByType(fieldType) + if fieldTypeId == 0 { + fieldTypeId = typeIdFromKind(fieldType) + } + // Calculate nullable flag for serialization (wire format): + // - In xlang mode: Per xlang spec, fields are NON-NULLABLE by default. + // Only pointer types are nullable by default. + // - In native mode: Go's natural semantics apply - slice/map/interface can be nil, + // so they are nullable by default. + // Can be overridden by explicit fory tag `fory:"nullable"`. + internalId := TypeId(fieldTypeId & 0xFF) + isEnum := internalId == ENUM || internalId == NAMED_ENUM + + // Determine nullable based on mode + // In xlang mode: only pointer types are nullable by default (per xlang spec) + // In native mode: Go's natural semantics - all nil-able types are nullable + // This ensures proper interoperability with Java/other languages in xlang mode. + var nullableFlag bool + if typeResolver.fory.config.IsXlang { + // xlang mode: only pointer types are nullable by default per xlang spec + // Slices and maps are NOT nullable - they serialize as empty when nil + nullableFlag = fieldType.Kind() == reflect.Ptr + } else { + // Native mode: Go's natural semantics - all nil-able types are nullable + nullableFlag = fieldType.Kind() == reflect.Ptr || + fieldType.Kind() == reflect.Slice || + fieldType.Kind() == reflect.Map || + fieldType.Kind() == reflect.Interface + } + if foryTag.NullableSet { + // Override nullable flag if explicitly set in fory tag + nullableFlag = foryTag.Nullable + } + // Primitives are never nullable, regardless of tag + if isNonNullablePrimitiveKind(fieldType.Kind()) && !isEnum { + nullableFlag = false + } + + // Calculate ref tracking - use tag override if explicitly set + trackRef := typeResolver.TrackRef() + if foryTag.RefSet { + trackRef = foryTag.Ref + } + + // Pre-compute RefMode based on (possibly overridden) trackRef and nullable + // For pointer-to-struct fields, enable ref tracking when trackRef is enabled, + // regardless of nullable flag. This is necessary to detect circular references. + refMode := RefModeNone + isStructPointer := fieldType.Kind() == reflect.Ptr && fieldType.Elem().Kind() == reflect.Struct + if trackRef && (nullableFlag || isStructPointer) { + refMode = RefModeTracking + } else if nullableFlag { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(fieldType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(fieldType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] initFieldsFromTypeResolver: field=%s type=%v staticId=%d refMode=%v nullableFlag=%v serializer=%T\n", + SnakeCase(field.Name), fieldType, staticId, refMode, nullableFlag, fieldSerializer) + } + + fieldInfo := &FieldInfo{ + Name: SnakeCase(field.Name), + Offset: field.Offset, + Type: fieldType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: nullableFlag, // Use same logic as TypeDef's nullable flag for consistent ref handling + FieldIndex: i, + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + TagID: foryTag.ID, + HasForyTag: foryTag.HasTag, + TagRefSet: foryTag.RefSet, + TagRef: foryTag.Ref, + TagNullableSet: foryTag.NullableSet, + TagNullable: foryTag.Nullable, + } + fields = append(fields, fieldInfo) + fieldNames = append(fieldNames, fieldInfo.Name) + serializers = append(serializers, fieldSerializer) + typeIds = append(typeIds, fieldTypeId) + nullables = append(nullables, nullableFlag) + tagIDs = append(tagIDs, foryTag.ID) + } + + // Sort fields according to specification using nullable info and tag IDs for consistent ordering + serializers, fieldNames = sortFields(typeResolver, fieldNames, serializers, typeIds, nullables, tagIDs) + order := make(map[string]int, len(fieldNames)) + for idx, name := range fieldNames { + order[name] = idx + } + + sort.SliceStable(fields, func(i, j int) bool { + oi, okI := order[fields[i].Name] + oj, okJ := order[fields[j].Name] + switch { + case okI && okJ: + return oi < oj + case okI: + return true + case okJ: + return false + default: + return false + } + }) + + s.fields = fields + s.groupFields() + return nil } // groupFields categorizes fields into fixedFields, varintFields, and remainingFields. // Also computes pre-computed sizes and WriteOffset for batch buffer reservation. func (s *structSerializer) groupFields() { - s.fixedFields = nil - s.varintFields = nil - s.remainingFields = nil - s.fixedSize = 0 - s.maxVarintSize = 0 - - for _, field := range s.fields { - // Fields with non-primitive serializers (NAMED_ENUM, NAMED_STRUCT, etc.) - // must go to remainingFields to use their serializer's type info writing - hasNonPrimitive := fieldHasNonPrimitiveSerializer(field) - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] groupFields: field=%s TypeId=%d internalId=%d hasNonPrimitive=%v\n", - field.Name, field.TypeId, field.TypeId&0xFF, hasNonPrimitive) - } - if hasNonPrimitive { - s.remainingFields = append(s.remainingFields, field) - } else if isFixedSizePrimitive(field.StaticId, field.Referencable) { - // Compute FixedSize and WriteOffset for this field - field.FixedSize = getFixedSizeByStaticId(field.StaticId) - field.WriteOffset = s.fixedSize - s.fixedSize += field.FixedSize - s.fixedFields = append(s.fixedFields, field) - } else if isVarintPrimitive(field.StaticId, field.Referencable) { - s.maxVarintSize += getVarintMaxSizeByStaticId(field.StaticId) - s.varintFields = append(s.varintFields, field) - } else { - s.remainingFields = append(s.remainingFields, field) - } - } + s.fixedFields = nil + s.varintFields = nil + s.remainingFields = nil + s.fixedSize = 0 + s.maxVarintSize = 0 + + for _, field := range s.fields { + // Fields with non-primitive serializers (NAMED_ENUM, NAMED_STRUCT, etc.) + // must go to remainingFields to use their serializer's type info writing + hasNonPrimitive := fieldHasNonPrimitiveSerializer(field) + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] groupFields: field=%s TypeId=%d internalId=%d hasNonPrimitive=%v\n", + field.Name, field.TypeId, field.TypeId&0xFF, hasNonPrimitive) + } + if hasNonPrimitive { + s.remainingFields = append(s.remainingFields, field) + } else if isFixedSizePrimitive(field.StaticId, field.Referencable) { + // Compute FixedSize and WriteOffset for this field + field.FixedSize = getFixedSizeByStaticId(field.StaticId) + field.WriteOffset = s.fixedSize + s.fixedSize += field.FixedSize + s.fixedFields = append(s.fixedFields, field) + } else if isVarintPrimitive(field.StaticId, field.Referencable) { + s.maxVarintSize += getVarintMaxSizeByStaticId(field.StaticId) + s.varintFields = append(s.varintFields, field) + } else { + s.remainingFields = append(s.remainingFields, field) + } + } } // initFieldsFromDefsWithResolver initializes fields from remote fieldDefs using typeResolver func (s *structSerializer) initFieldsFromDefsWithResolver(typeResolver *TypeResolver) error { - type_ := s.type_ - if type_ == nil { - // Type is not known - we'll create an interface{} placeholder - // This happens when deserializing unknown types in compatible mode - // For now, we'll create fields that discard all data - var fields []*FieldInfo - for _, def := range s.fieldDefs { - fieldSerializer, _ := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - remoteType := remoteTypeInfo.Type - if remoteType == nil { - remoteType = reflect.TypeOf((*interface{})(nil)).Elem() - } - // Get TypeId from FieldType's TypeId method - fieldTypeId := def.fieldType.TypeId() - // Pre-compute RefMode based on trackRef and FieldDef flags - refMode := RefModeNone - if def.trackingRef { - refMode = RefModeTracking - } else if def.nullable { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(remoteType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(remoteType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - - fieldInfo := &FieldInfo{ - Name: def.name, - Offset: 0, - Type: remoteType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: def.nullable, // Use remote nullable flag - FieldIndex: -1, // Mark as non-existent field to discard data - FieldDef: def, // Save original FieldDef for skipping - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - } - fields = append(fields, fieldInfo) - } - s.fields = fields - s.groupFields() - s.typeDefDiffers = true // Unknown type, must use ordered reading - return nil - } - - // Build maps from field names and tag IDs to struct field indices - fieldNameToIndex := make(map[string]int) - fieldNameToOffset := make(map[string]uintptr) - fieldNameToType := make(map[string]reflect.Type) - fieldTagIDToIndex := make(map[int]int) // tag ID -> struct field index - fieldTagIDToOffset := make(map[int]uintptr) // tag ID -> field offset - fieldTagIDToType := make(map[int]reflect.Type) // tag ID -> field type - fieldTagIDToName := make(map[int]string) // tag ID -> snake_case field name - for i := 0; i < type_.NumField(); i++ { - field := type_.Field(i) - - // Parse fory tag and skip ignored fields - foryTag := ParseForyTag(field) - if foryTag.Ignore { - continue - } - - name := SnakeCase(field.Name) - fieldNameToIndex[name] = i - fieldNameToOffset[name] = field.Offset - fieldNameToType[name] = field.Type - - // Also index by tag ID if present - if foryTag.ID >= 0 { - fieldTagIDToIndex[foryTag.ID] = i - fieldTagIDToOffset[foryTag.ID] = field.Offset - fieldTagIDToType[foryTag.ID] = field.Type - fieldTagIDToName[foryTag.ID] = name - } - } - - var fields []*FieldInfo - - for _, def := range s.fieldDefs { - fieldSerializer, err := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) - if err != nil || fieldSerializer == nil { - // If we can't get serializer from typeID, try to get it from the Go type - // This can happen when the type isn't registered in typeIDToTypeInfo - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - if remoteTypeInfo.Type != nil { - fieldSerializer, _ = typeResolver.getSerializerByType(remoteTypeInfo.Type, true) - } - } - - // Get the remote type from fieldDef - remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) - remoteType := remoteTypeInfo.Type - // Track if type lookup failed - we'll need to skip such fields - // Note: DynamicFieldType.getTypeInfoWithResolver returns interface{} (not nil) when lookup fails - emptyInterfaceType := reflect.TypeOf((*interface{})(nil)).Elem() - typeLookupFailed := remoteType == nil || remoteType == emptyInterfaceType - if remoteType == nil { - remoteType = emptyInterfaceType - } - - // For struct-like fields, even if TypeDef lookup fails, we can try to read - // the field because type resolution happens at read time from the buffer. - // The type name might map to a different local type. - isStructLikeField := isStructFieldType(def.fieldType) - - // Try to find corresponding local field - // First try to match by tag ID (if remote def uses tag ID) - // Then fall back to matching by field name - fieldIndex := -1 - var offset uintptr - var fieldType reflect.Type - var localFieldName string - var localType reflect.Type - var exists bool - - if def.tagID >= 0 { - // Try to match by tag ID - if idx, ok := fieldTagIDToIndex[def.tagID]; ok { - exists = true - fieldIndex = idx // Will be overwritten if types are compatible - localType = fieldTagIDToType[def.tagID] - offset = fieldTagIDToOffset[def.tagID] - localFieldName = fieldTagIDToName[def.tagID] - _ = fieldIndex // Use to avoid compiler warning, will be set properly below - } - } - - // Fall back to name-based matching if tag ID match failed - if !exists && def.name != "" { - if idx, ok := fieldNameToIndex[def.name]; ok { - exists = true - localType = fieldNameToType[def.name] - offset = fieldNameToOffset[def.name] - localFieldName = def.name - _ = idx // Will be set properly below - } - } - - if exists { - idx := fieldNameToIndex[localFieldName] - if def.tagID >= 0 { - idx = fieldTagIDToIndex[def.tagID] - } - // Check if types are compatible - // For primitive types: skip if types don't match - // For struct-like types: allow read even if TypeDef lookup failed, - // because runtime type resolution by name might work - shouldRead := false - isPolymorphicField := def.fieldType.TypeId() == UNKNOWN - defTypeId := def.fieldType.TypeId() - // Check if field is an enum - either by type ID or by serializer type - // The type ID may be a composite value with namespace bits, so check the low 8 bits - internalDefTypeId := defTypeId & 0xFF - isEnumField := internalDefTypeId == NAMED_ENUM || internalDefTypeId == ENUM - if !isEnumField && fieldSerializer != nil { - _, isEnumField = fieldSerializer.(*enumSerializer) - } - if isPolymorphicField && localType.Kind() == reflect.Interface { - // For polymorphic (UNKNOWN) fields with interface{} local type, - // allow reading - the actual type will be determined at runtime - shouldRead = true - fieldType = localType - } else if typeLookupFailed && isEnumField { - // For enum fields with failed TypeDef lookup (NAMED_ENUM stores by namespace/typename, not typeId), - // check if local field is a numeric type (Go enums are int-based) - // Also handle pointer enum fields (*EnumType) - localKind := localType.Kind() - elemKind := localKind - if localKind == reflect.Ptr { - elemKind = localType.Elem().Kind() - } - if isNumericKind(elemKind) { - shouldRead = true - fieldType = localType - // Get the serializer for the base type (the enum type, not the pointer) - baseType := localType - if localKind == reflect.Ptr { - baseType = localType.Elem() - } - fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) - } - } else if typeLookupFailed && isStructLikeField { - // For struct fields with failed TypeDef lookup, check if local field can hold a struct - localKind := localType.Kind() - if localKind == reflect.Ptr { - localKind = localType.Elem().Kind() - } - if localKind == reflect.Struct || localKind == reflect.Interface { - shouldRead = true - fieldType = localType // Use local type for struct fields - } - } else if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) { - // For collection fields with failed type lookup (e.g., List with interface element type), - // check if local type is a slice with interface element type (e.g., []Animal) - // The type lookup fails because sliceConcreteValueSerializer doesn't support interface elements - if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { - shouldRead = true - fieldType = localType - } - } else if !typeLookupFailed && typesCompatible(localType, remoteType) { - shouldRead = true - fieldType = localType - } - - if shouldRead { - fieldIndex = idx - // offset was already set above when matching by tag ID or field name - // For struct-like fields with failed type lookup, get the serializer for the local type - if typeLookupFailed && isStructLikeField && fieldSerializer == nil { - fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) - } - // For collection fields with interface element types, use sliceDynSerializer - if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) && fieldSerializer == nil { - if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { - fieldSerializer = mustNewSliceDynSerializer(localType.Elem()) - } - } - // If local type is *T and remote type is T, we need the serializer for *T - // This handles Java's Integer/Long (nullable boxed types) mapping to Go's *int32/*int64 - if localType.Kind() == reflect.Ptr && localType.Elem() == remoteType { - fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) - } - // For pointer enum fields (*EnumType), get the serializer for the base enum type - // The struct read/write code will handle pointer dereferencing - if isEnumField && localType.Kind() == reflect.Ptr { - baseType := localType.Elem() - fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) - if DebugOutputEnabled() { - fmt.Printf("[fory-debug] pointer enum field %s: localType=%v baseType=%v serializer=%T\n", - def.name, localType, baseType, fieldSerializer) - } - } - // For array fields, use array serializers (not slice serializers) even if typeID maps to slice serializer - // The typeID (INT16_ARRAY, etc.) is shared between arrays and slices, but we need the correct - // serializer based on the actual Go type - if localType.Kind() == reflect.Array { - elemType := localType.Elem() - switch elemType.Kind() { - case reflect.Bool: - fieldSerializer = boolArraySerializer{arrayType: localType} - case reflect.Int8: - fieldSerializer = int8ArraySerializer{arrayType: localType} - case reflect.Int16: - fieldSerializer = int16ArraySerializer{arrayType: localType} - case reflect.Int32: - fieldSerializer = int32ArraySerializer{arrayType: localType} - case reflect.Int64: - fieldSerializer = int64ArraySerializer{arrayType: localType} - case reflect.Uint8: - fieldSerializer = uint8ArraySerializer{arrayType: localType} - case reflect.Float32: - fieldSerializer = float32ArraySerializer{arrayType: localType} - case reflect.Float64: - fieldSerializer = float64ArraySerializer{arrayType: localType} - case reflect.Int: - if reflect.TypeOf(int(0)).Size() == 8 { - fieldSerializer = int64ArraySerializer{arrayType: localType} - } else { - fieldSerializer = int32ArraySerializer{arrayType: localType} - } - } - } - } else { - // Types are incompatible or unknown - use remote type but mark field as not settable - fieldType = remoteType - fieldIndex = -1 - offset = 0 // Don't set offset for incompatible fields - } - } else { - // Field doesn't exist locally, use type from fieldDef - fieldType = remoteType - } - - // Get TypeId from FieldType's TypeId method - fieldTypeId := def.fieldType.TypeId() - // Pre-compute RefMode based on FieldDef flags (trackingRef and nullable) - refMode := RefModeNone - if def.trackingRef { - refMode = RefModeTracking - } else if def.nullable { - refMode = RefModeNullOnly - } - // Pre-compute WriteType: true for struct fields in compatible mode - writeType := typeResolver.Compatible() && isStructField(fieldType) - - // Pre-compute StaticId, with special handling for enum fields - staticId := GetStaticTypeId(fieldType) - if fieldSerializer != nil { - if _, ok := fieldSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - staticId = ConcreteTypeEnum - } - } - } - - // Determine field name: use local field name if matched, otherwise use def.name - fieldName := def.name - if localFieldName != "" { - fieldName = localFieldName - } - - fieldInfo := &FieldInfo{ - Name: fieldName, - Offset: offset, - Type: fieldType, - StaticId: staticId, - TypeId: fieldTypeId, - Serializer: fieldSerializer, - Referencable: def.nullable, // Use remote nullable flag - FieldIndex: fieldIndex, - FieldDef: def, // Save original FieldDef for skipping - RefMode: refMode, - WriteType: writeType, - HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types - TagID: def.tagID, - HasForyTag: def.tagID >= 0, - } - fields = append(fields, fieldInfo) - } - - s.fields = fields - s.groupFields() - - // Compute typeDefDiffers: true if any field doesn't exist locally or has type mismatch - // When typeDefDiffers is false, we can use grouped reading for better performance - s.typeDefDiffers = false - for _, field := range fields { - if field.FieldIndex < 0 { - // Field exists in remote TypeDef but not locally - s.typeDefDiffers = true - break - } - } - - return nil + type_ := s.type_ + if type_ == nil { + // Type is not known - we'll create an interface{} placeholder + // This happens when deserializing unknown types in compatible mode + // For now, we'll create fields that discard all data + var fields []*FieldInfo + for _, def := range s.fieldDefs { + fieldSerializer, _ := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + remoteType := remoteTypeInfo.Type + if remoteType == nil { + remoteType = reflect.TypeOf((*interface{})(nil)).Elem() + } + // Get TypeId from FieldType's TypeId method + fieldTypeId := def.fieldType.TypeId() + // Pre-compute RefMode based on trackRef and FieldDef flags + refMode := RefModeNone + if def.trackingRef { + refMode = RefModeTracking + } else if def.nullable { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(remoteType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(remoteType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + + fieldInfo := &FieldInfo{ + Name: def.name, + Offset: 0, + Type: remoteType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: def.nullable, // Use remote nullable flag + FieldIndex: -1, // Mark as non-existent field to discard data + FieldDef: def, // Save original FieldDef for skipping + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + } + fields = append(fields, fieldInfo) + } + s.fields = fields + s.groupFields() + s.typeDefDiffers = true // Unknown type, must use ordered reading + return nil + } + + // Build maps from field names and tag IDs to struct field indices + fieldNameToIndex := make(map[string]int) + fieldNameToOffset := make(map[string]uintptr) + fieldNameToType := make(map[string]reflect.Type) + fieldTagIDToIndex := make(map[int]int) // tag ID -> struct field index + fieldTagIDToOffset := make(map[int]uintptr) // tag ID -> field offset + fieldTagIDToType := make(map[int]reflect.Type) // tag ID -> field type + fieldTagIDToName := make(map[int]string) // tag ID -> snake_case field name + for i := 0; i < type_.NumField(); i++ { + field := type_.Field(i) + + // Parse fory tag and skip ignored fields + foryTag := ParseForyTag(field) + if foryTag.Ignore { + continue + } + + name := SnakeCase(field.Name) + fieldNameToIndex[name] = i + fieldNameToOffset[name] = field.Offset + fieldNameToType[name] = field.Type + + // Also index by tag ID if present + if foryTag.ID >= 0 { + fieldTagIDToIndex[foryTag.ID] = i + fieldTagIDToOffset[foryTag.ID] = field.Offset + fieldTagIDToType[foryTag.ID] = field.Type + fieldTagIDToName[foryTag.ID] = name + } + } + + var fields []*FieldInfo + + for _, def := range s.fieldDefs { + fieldSerializer, err := getFieldTypeSerializerWithResolver(typeResolver, def.fieldType) + if err != nil || fieldSerializer == nil { + // If we can't get serializer from typeID, try to get it from the Go type + // This can happen when the type isn't registered in typeIDToTypeInfo + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + if remoteTypeInfo.Type != nil { + fieldSerializer, _ = typeResolver.getSerializerByType(remoteTypeInfo.Type, true) + } + } + + // Get the remote type from fieldDef + remoteTypeInfo, _ := def.fieldType.getTypeInfoWithResolver(typeResolver) + remoteType := remoteTypeInfo.Type + // Track if type lookup failed - we'll need to skip such fields + // Note: DynamicFieldType.getTypeInfoWithResolver returns interface{} (not nil) when lookup fails + emptyInterfaceType := reflect.TypeOf((*interface{})(nil)).Elem() + typeLookupFailed := remoteType == nil || remoteType == emptyInterfaceType + if remoteType == nil { + remoteType = emptyInterfaceType + } + + // For struct-like fields, even if TypeDef lookup fails, we can try to read + // the field because type resolution happens at read time from the buffer. + // The type name might map to a different local type. + isStructLikeField := isStructFieldType(def.fieldType) + + // Try to find corresponding local field + // First try to match by tag ID (if remote def uses tag ID) + // Then fall back to matching by field name + fieldIndex := -1 + var offset uintptr + var fieldType reflect.Type + var localFieldName string + var localType reflect.Type + var exists bool + + if def.tagID >= 0 { + // Try to match by tag ID + if idx, ok := fieldTagIDToIndex[def.tagID]; ok { + exists = true + fieldIndex = idx // Will be overwritten if types are compatible + localType = fieldTagIDToType[def.tagID] + offset = fieldTagIDToOffset[def.tagID] + localFieldName = fieldTagIDToName[def.tagID] + _ = fieldIndex // Use to avoid compiler warning, will be set properly below + } + } + + // Fall back to name-based matching if tag ID match failed + if !exists && def.name != "" { + if idx, ok := fieldNameToIndex[def.name]; ok { + exists = true + localType = fieldNameToType[def.name] + offset = fieldNameToOffset[def.name] + localFieldName = def.name + _ = idx // Will be set properly below + } + } + + if exists { + idx := fieldNameToIndex[localFieldName] + if def.tagID >= 0 { + idx = fieldTagIDToIndex[def.tagID] + } + // Check if types are compatible + // For primitive types: skip if types don't match + // For struct-like types: allow read even if TypeDef lookup failed, + // because runtime type resolution by name might work + shouldRead := false + isPolymorphicField := def.fieldType.TypeId() == UNKNOWN + defTypeId := def.fieldType.TypeId() + // Check if field is an enum - either by type ID or by serializer type + // The type ID may be a composite value with namespace bits, so check the low 8 bits + internalDefTypeId := defTypeId & 0xFF + isEnumField := internalDefTypeId == NAMED_ENUM || internalDefTypeId == ENUM + if !isEnumField && fieldSerializer != nil { + _, isEnumField = fieldSerializer.(*enumSerializer) + } + if isPolymorphicField && localType.Kind() == reflect.Interface { + // For polymorphic (UNKNOWN) fields with interface{} local type, + // allow reading - the actual type will be determined at runtime + shouldRead = true + fieldType = localType + } else if typeLookupFailed && isEnumField { + // For enum fields with failed TypeDef lookup (NAMED_ENUM stores by namespace/typename, not typeId), + // check if local field is a numeric type (Go enums are int-based) + // Also handle pointer enum fields (*EnumType) + localKind := localType.Kind() + elemKind := localKind + if localKind == reflect.Ptr { + elemKind = localType.Elem().Kind() + } + if isNumericKind(elemKind) { + shouldRead = true + fieldType = localType + // Get the serializer for the base type (the enum type, not the pointer) + baseType := localType + if localKind == reflect.Ptr { + baseType = localType.Elem() + } + fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) + } + } else if typeLookupFailed && isStructLikeField { + // For struct fields with failed TypeDef lookup, check if local field can hold a struct + localKind := localType.Kind() + if localKind == reflect.Ptr { + localKind = localType.Elem().Kind() + } + if localKind == reflect.Struct || localKind == reflect.Interface { + shouldRead = true + fieldType = localType // Use local type for struct fields + } + } else if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) { + // For collection fields with failed type lookup (e.g., List with interface element type), + // check if local type is a slice with interface element type (e.g., []Animal) + // The type lookup fails because sliceSerializer doesn't support interface elements + if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { + shouldRead = true + fieldType = localType + } + } else if !typeLookupFailed && typesCompatible(localType, remoteType) { + shouldRead = true + fieldType = localType + } + + if shouldRead { + fieldIndex = idx + // offset was already set above when matching by tag ID or field name + // For struct-like fields with failed type lookup, get the serializer for the local type + if typeLookupFailed && isStructLikeField && fieldSerializer == nil { + fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) + } + // For collection fields with interface element types, use sliceDynSerializer + if typeLookupFailed && (defTypeId == LIST || defTypeId == SET) && fieldSerializer == nil { + if localType.Kind() == reflect.Slice && localType.Elem().Kind() == reflect.Interface { + fieldSerializer = mustNewSliceDynSerializer(localType.Elem()) + } + } + // If local type is *T and remote type is T, we need the serializer for *T + // This handles Java's Integer/Long (nullable boxed types) mapping to Go's *int32/*int64 + if localType.Kind() == reflect.Ptr && localType.Elem() == remoteType { + fieldSerializer, _ = typeResolver.getSerializerByType(localType, true) + } + // For pointer enum fields (*EnumType), get the serializer for the base enum type + // The struct read/write code will handle pointer dereferencing + if isEnumField && localType.Kind() == reflect.Ptr { + baseType := localType.Elem() + fieldSerializer, _ = typeResolver.getSerializerByType(baseType, true) + if DebugOutputEnabled() { + fmt.Printf("[fory-debug] pointer enum field %s: localType=%v baseType=%v serializer=%T\n", + def.name, localType, baseType, fieldSerializer) + } + } + // For array fields, use array serializers (not slice serializers) even if typeID maps to slice serializer + // The typeID (INT16_ARRAY, etc.) is shared between arrays and slices, but we need the correct + // serializer based on the actual Go type + if localType.Kind() == reflect.Array { + elemType := localType.Elem() + switch elemType.Kind() { + case reflect.Bool: + fieldSerializer = boolArraySerializer{arrayType: localType} + case reflect.Int8: + fieldSerializer = int8ArraySerializer{arrayType: localType} + case reflect.Int16: + fieldSerializer = int16ArraySerializer{arrayType: localType} + case reflect.Int32: + fieldSerializer = int32ArraySerializer{arrayType: localType} + case reflect.Int64: + fieldSerializer = int64ArraySerializer{arrayType: localType} + case reflect.Uint8: + fieldSerializer = uint8ArraySerializer{arrayType: localType} + case reflect.Float32: + fieldSerializer = float32ArraySerializer{arrayType: localType} + case reflect.Float64: + fieldSerializer = float64ArraySerializer{arrayType: localType} + case reflect.Int: + if reflect.TypeOf(int(0)).Size() == 8 { + fieldSerializer = int64ArraySerializer{arrayType: localType} + } else { + fieldSerializer = int32ArraySerializer{arrayType: localType} + } + } + } + } else { + // Types are incompatible or unknown - use remote type but mark field as not settable + fieldType = remoteType + fieldIndex = -1 + offset = 0 // Don't set offset for incompatible fields + } + } else { + // Field doesn't exist locally, use type from fieldDef + fieldType = remoteType + } + + // Get TypeId from FieldType's TypeId method + fieldTypeId := def.fieldType.TypeId() + // Pre-compute RefMode based on FieldDef flags (trackingRef and nullable) + refMode := RefModeNone + if def.trackingRef { + refMode = RefModeTracking + } else if def.nullable { + refMode = RefModeNullOnly + } + // Pre-compute WriteType: true for struct fields in compatible mode + writeType := typeResolver.Compatible() && isStructField(fieldType) + + // Pre-compute StaticId, with special handling for enum fields + staticId := GetStaticTypeId(fieldType) + if fieldSerializer != nil { + if _, ok := fieldSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } else if ptrSer, ok := fieldSerializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + staticId = ConcreteTypeEnum + } + } + } + + // Determine field name: use local field name if matched, otherwise use def.name + fieldName := def.name + if localFieldName != "" { + fieldName = localFieldName + } + + fieldInfo := &FieldInfo{ + Name: fieldName, + Offset: offset, + Type: fieldType, + StaticId: staticId, + TypeId: fieldTypeId, + Serializer: fieldSerializer, + Referencable: def.nullable, // Use remote nullable flag + FieldIndex: fieldIndex, + FieldDef: def, // Save original FieldDef for skipping + RefMode: refMode, + WriteType: writeType, + HasGenerics: isCollectionType(fieldTypeId), // Container fields have declared element types + TagID: def.tagID, + HasForyTag: def.tagID >= 0, + } + fields = append(fields, fieldInfo) + } + + s.fields = fields + s.groupFields() + + // Compute typeDefDiffers: true if any field doesn't exist locally or has type mismatch + // When typeDefDiffers is false, we can use grouped reading for better performance + s.typeDefDiffers = false + for _, field := range fields { + if field.FieldIndex < 0 { + // Field exists in remote TypeDef but not locally + s.typeDefDiffers = true + break + } + } + + return nil } // isNonNullablePrimitiveKind returns true for Go kinds that map to Java primitive types // These are the types that cannot be null in Java and should have nullable=0 in hash computation func isNonNullablePrimitiveKind(kind reflect.Kind) bool { - switch kind { - case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64, reflect.Int, reflect.Uint: - return true - default: - return false - } + switch kind { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.Int, reflect.Uint: + return true + default: + return false + } } // isInternalTypeWithoutTypeMeta checks if a type is serialized without type meta per xlang spec. @@ -1606,40 +1606,40 @@ func isNonNullablePrimitiveKind(kind reflect.Kind) bool { // - Map: | ref meta | value data | // Only struct/enum/ext types need type meta: | ref flag | type meta | value data | func isInternalTypeWithoutTypeMeta(t reflect.Type) bool { - kind := t.Kind() - // String type - no type meta needed - if kind == reflect.String { - return true - } - // Slice (list or byte slice) - no type meta needed - if kind == reflect.Slice { - return true - } - // Map type - no type meta needed - if kind == reflect.Map { - return true - } - // Pointer to primitive - no type meta needed - if kind == reflect.Ptr { - elemKind := t.Elem().Kind() - switch elemKind { - case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Int, reflect.Float32, reflect.Float64, reflect.String: - return true - } - } - return false + kind := t.Kind() + // String type - no type meta needed + if kind == reflect.String { + return true + } + // Slice (list or byte slice) - no type meta needed + if kind == reflect.Slice { + return true + } + // Map type - no type meta needed + if kind == reflect.Map { + return true + } + // Pointer to primitive - no type meta needed + if kind == reflect.Ptr { + elemKind := t.Elem().Kind() + switch elemKind { + case reflect.Bool, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Int, reflect.Float32, reflect.Float64, reflect.String: + return true + } + } + return false } // isStructField checks if a type is a struct type (directly or via pointer) func isStructField(t reflect.Type) bool { - if t.Kind() == reflect.Struct { - return true - } - if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { - return true - } - return false + if t.Kind() == reflect.Struct { + return true + } + if t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct { + return true + } + return false } // isStructFieldType checks if a FieldType represents a type that needs type info written @@ -1647,34 +1647,34 @@ func isStructField(t reflect.Type) bool { // In compatible mode, Java writes type info for struct and ext types, but NOT for enum types // Enum fields only have null flag + ordinal, no type ID func isStructFieldType(ft FieldType) bool { - if ft == nil { - return false - } - typeId := ft.TypeId() - // Check base type IDs that need type info (struct and ext, NOT enum) - // Always check the internal type ID (low byte) to handle composite type IDs - // which may be negative when stored as int32 (e.g., -2288 = (short)128784) - internalTypeId := TypeId(typeId & 0xFF) - switch internalTypeId { - case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, - EXT, NAMED_EXT: - return true - } - return false + if ft == nil { + return false + } + typeId := ft.TypeId() + // Check base type IDs that need type info (struct and ext, NOT enum) + // Always check the internal type ID (low byte) to handle composite type IDs + // which may be negative when stored as int32 (e.g., -2288 = (short)128784) + internalTypeId := TypeId(typeId & 0xFF) + switch internalTypeId { + case STRUCT, NAMED_STRUCT, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, + EXT, NAMED_EXT: + return true + } + return false } // FieldFingerprintInfo contains the information needed to compute a field's fingerprint. type FieldFingerprintInfo struct { - // FieldID is the tag ID if configured (>= 0), or -1 to use field name - FieldID int - // FieldName is the snake_case field name (used when FieldID < 0) - FieldName string - // TypeID is the Fory type ID for the field - TypeID TypeId - // Ref is true if reference tracking is enabled for this field - Ref bool - // Nullable is true if null flag is written for this field - Nullable bool + // FieldID is the tag ID if configured (>= 0), or -1 to use field name + FieldID int + // FieldName is the snake_case field name (used when FieldID < 0) + FieldName string + // TypeID is the Fory type ID for the field + TypeID TypeId + // Ref is true if reference tracking is enabled for this field + Ref bool + // Nullable is true if null flag is written for this field + Nullable bool } // ComputeStructFingerprint computes the fingerprint string for a struct type. @@ -1698,177 +1698,177 @@ type FieldFingerprintInfo struct { // Different nullable/ref settings will produce different fingerprints, // ensuring schema compatibility is properly validated. func ComputeStructFingerprint(fields []FieldFingerprintInfo) string { - // Sort fields by their identifier (field ID or name) - type fieldWithKey struct { - field FieldFingerprintInfo - sortKey string - } - fieldsWithKeys := make([]fieldWithKey, 0, len(fields)) - for _, field := range fields { - var sortKey string - if field.FieldID >= 0 { - sortKey = fmt.Sprintf("%d", field.FieldID) - } else { - sortKey = field.FieldName - } - fieldsWithKeys = append(fieldsWithKeys, fieldWithKey{field: field, sortKey: sortKey}) - } - - sort.Slice(fieldsWithKeys, func(i, j int) bool { - return fieldsWithKeys[i].sortKey < fieldsWithKeys[j].sortKey - }) - - var sb strings.Builder - for _, fw := range fieldsWithKeys { - // Field identifier - sb.WriteString(fw.sortKey) - sb.WriteString(",") - // Type ID - sb.WriteString(fmt.Sprintf("%d", fw.field.TypeID)) - sb.WriteString(",") - // Ref flag - if fw.field.Ref { - sb.WriteString("1") - } else { - sb.WriteString("0") - } - sb.WriteString(",") - // Nullable flag - if fw.field.Nullable { - sb.WriteString("1") - } else { - sb.WriteString("0") - } - sb.WriteString(";") - } - return sb.String() + // Sort fields by their identifier (field ID or name) + type fieldWithKey struct { + field FieldFingerprintInfo + sortKey string + } + fieldsWithKeys := make([]fieldWithKey, 0, len(fields)) + for _, field := range fields { + var sortKey string + if field.FieldID >= 0 { + sortKey = fmt.Sprintf("%d", field.FieldID) + } else { + sortKey = field.FieldName + } + fieldsWithKeys = append(fieldsWithKeys, fieldWithKey{field: field, sortKey: sortKey}) + } + + sort.Slice(fieldsWithKeys, func(i, j int) bool { + return fieldsWithKeys[i].sortKey < fieldsWithKeys[j].sortKey + }) + + var sb strings.Builder + for _, fw := range fieldsWithKeys { + // Field identifier + sb.WriteString(fw.sortKey) + sb.WriteString(",") + // Type ID + sb.WriteString(fmt.Sprintf("%d", fw.field.TypeID)) + sb.WriteString(",") + // Ref flag + if fw.field.Ref { + sb.WriteString("1") + } else { + sb.WriteString("0") + } + sb.WriteString(",") + // Nullable flag + if fw.field.Nullable { + sb.WriteString("1") + } else { + sb.WriteString("0") + } + sb.WriteString(";") + } + return sb.String() } func (s *structSerializer) computeHash() int32 { - // Build FieldFingerprintInfo for each field - fields := make([]FieldFingerprintInfo, 0, len(s.fields)) - for _, field := range s.fields { - var typeId TypeId - isEnumField := false - if field.Serializer == nil { - typeId = UNKNOWN - } else { - typeId = field.TypeId - // Check if this is an enum serializer (directly or wrapped in ptrToValueSerializer) - if _, ok := field.Serializer.(*enumSerializer); ok { - isEnumField = true - typeId = UNKNOWN - } else if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { - if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { - isEnumField = true - typeId = UNKNOWN - } - } - // For user-defined types (struct, ext types), use UNKNOWN in fingerprint - // This matches Java's behavior where user-defined types return UNKNOWN - // to ensure consistent fingerprint computation across languages - if isUserDefinedType(int16(typeId)) { - typeId = UNKNOWN - } - // For fixed-size arrays with primitive elements, use primitive array type IDs - if field.Type.Kind() == reflect.Array { - elemKind := field.Type.Elem().Kind() - switch elemKind { - case reflect.Int8: - typeId = INT8_ARRAY - case reflect.Int16: - typeId = INT16_ARRAY - case reflect.Int32: - typeId = INT32_ARRAY - case reflect.Int64: - typeId = INT64_ARRAY - case reflect.Float32: - typeId = FLOAT32_ARRAY - case reflect.Float64: - typeId = FLOAT64_ARRAY - default: - typeId = LIST - } - } else if field.Type.Kind() == reflect.Slice { - typeId = LIST - } else if field.Type.Kind() == reflect.Map { - // map[T]bool is used to represent a Set in Go - if field.Type.Elem().Kind() == reflect.Bool { - typeId = SET - } else { - typeId = MAP - } - } - } - - // Determine nullable flag for xlang compatibility: - // - Default: false for ALL fields (xlang default - aligned with all languages) - // - Primitives are always non-nullable - // - Can be overridden by explicit fory tag - nullable := false // Default to nullable=false for xlang mode - if field.TagNullableSet { - // Use explicit tag value if set - nullable = field.TagNullable - } - // Primitives are never nullable, regardless of tag - if isNonNullablePrimitiveKind(field.Type.Kind()) && !isEnumField { - nullable = false - } - - fields = append(fields, FieldFingerprintInfo{ - FieldID: field.TagID, - FieldName: SnakeCase(field.Name), - TypeID: typeId, - // Ref is based on explicit tag annotation only, NOT runtime ref_tracking config - // This allows fingerprint to be computed at compile time for C++/Rust - Ref: field.TagRefSet && field.TagRef, - Nullable: nullable, - }) - } - - hashString := ComputeStructFingerprint(fields) - data := []byte(hashString) - h1, _ := murmur3.Sum128WithSeed(data, 47) - hash := int32(h1 & 0xFFFFFFFF) - - if DebugOutputEnabled() { - fmt.Printf("[Go][fory-debug] struct %v version fingerprint=\"%s\" version hash=%d\n", s.type_, hashString, hash) - } - - if hash == 0 { - panic(fmt.Errorf("hash for type %v is 0", s.type_)) - } - return hash + // Build FieldFingerprintInfo for each field + fields := make([]FieldFingerprintInfo, 0, len(s.fields)) + for _, field := range s.fields { + var typeId TypeId + isEnumField := false + if field.Serializer == nil { + typeId = UNKNOWN + } else { + typeId = field.TypeId + // Check if this is an enum serializer (directly or wrapped in ptrToValueSerializer) + if _, ok := field.Serializer.(*enumSerializer); ok { + isEnumField = true + typeId = UNKNOWN + } else if ptrSer, ok := field.Serializer.(*ptrToValueSerializer); ok { + if _, ok := ptrSer.valueSerializer.(*enumSerializer); ok { + isEnumField = true + typeId = UNKNOWN + } + } + // For user-defined types (struct, ext types), use UNKNOWN in fingerprint + // This matches Java's behavior where user-defined types return UNKNOWN + // to ensure consistent fingerprint computation across languages + if isUserDefinedType(int16(typeId)) { + typeId = UNKNOWN + } + // For fixed-size arrays with primitive elements, use primitive array type IDs + if field.Type.Kind() == reflect.Array { + elemKind := field.Type.Elem().Kind() + switch elemKind { + case reflect.Int8: + typeId = INT8_ARRAY + case reflect.Int16: + typeId = INT16_ARRAY + case reflect.Int32: + typeId = INT32_ARRAY + case reflect.Int64: + typeId = INT64_ARRAY + case reflect.Float32: + typeId = FLOAT32_ARRAY + case reflect.Float64: + typeId = FLOAT64_ARRAY + default: + typeId = LIST + } + } else if field.Type.Kind() == reflect.Slice { + typeId = LIST + } else if field.Type.Kind() == reflect.Map { + // map[T]bool is used to represent a Set in Go + if field.Type.Elem().Kind() == reflect.Bool { + typeId = SET + } else { + typeId = MAP + } + } + } + + // Determine nullable flag for xlang compatibility: + // - Default: false for ALL fields (xlang default - aligned with all languages) + // - Primitives are always non-nullable + // - Can be overridden by explicit fory tag + nullable := false // Default to nullable=false for xlang mode + if field.TagNullableSet { + // Use explicit tag value if set + nullable = field.TagNullable + } + // Primitives are never nullable, regardless of tag + if isNonNullablePrimitiveKind(field.Type.Kind()) && !isEnumField { + nullable = false + } + + fields = append(fields, FieldFingerprintInfo{ + FieldID: field.TagID, + FieldName: SnakeCase(field.Name), + TypeID: typeId, + // Ref is based on explicit tag annotation only, NOT runtime ref_tracking config + // This allows fingerprint to be computed at compile time for C++/Rust + Ref: field.TagRefSet && field.TagRef, + Nullable: nullable, + }) + } + + hashString := ComputeStructFingerprint(fields) + data := []byte(hashString) + h1, _ := murmur3.Sum128WithSeed(data, 47) + hash := int32(h1 & 0xFFFFFFFF) + + if DebugOutputEnabled() { + fmt.Printf("[Go][fory-debug] struct %v version fingerprint=\"%s\" version hash=%d\n", s.type_, hashString, hash) + } + + if hash == 0 { + panic(fmt.Errorf("hash for type %v is 0", s.type_)) + } + return hash } // GetStructHash returns the struct hash for a given type using the provided TypeResolver. // This is used by codegen serializers to get the hash at runtime. func GetStructHash(type_ reflect.Type, resolver *TypeResolver) int32 { - ser := newStructSerializer(type_, "", nil) - if err := ser.initialize(resolver); err != nil { - panic(fmt.Errorf("failed to initialize struct serializer for hash computation: %v", err)) - } - return ser.structHash + ser := newStructSerializer(type_, "", nil) + if err := ser.initialize(resolver); err != nil { + panic(fmt.Errorf("failed to initialize struct serializer for hash computation: %v", err)) + } + return ser.structHash } // Field sorting helpers type triple struct { - typeID int16 - serializer Serializer - name string - nullable bool - tagID int // -1 = use field name, >=0 = use tag ID for sorting + typeID int16 + serializer Serializer + name string + nullable bool + tagID int // -1 = use field name, >=0 = use tag ID for sorting } // getFieldSortKey returns the sort key for a field. // If tagID >= 0, returns the tag ID as string (for tag-based sorting). // Otherwise returns the snake_case field name. func (t triple) getSortKey() string { - if t.tagID >= 0 { - return fmt.Sprintf("%d", t.tagID) - } - return SnakeCase(t.name) + if t.tagID >= 0 { + return fmt.Sprintf("%d", t.tagID) + } + return SnakeCase(t.name) } // sortFields sorts fields with nullable information to match Java's field ordering. @@ -1876,315 +1876,315 @@ func (t triple) getSortKey() string { // In Go, this corresponds to non-pointer primitives vs pointer-to-primitive. // When tagIDs are provided (>= 0), fields are sorted by tag ID instead of field name. func sortFields( - typeResolver *TypeResolver, - fieldNames []string, - serializers []Serializer, - typeIds []TypeId, - nullables []bool, - tagIDs []int, + typeResolver *TypeResolver, + fieldNames []string, + serializers []Serializer, + typeIds []TypeId, + nullables []bool, + tagIDs []int, ) ([]Serializer, []string) { - var ( - typeTriples []triple - others []triple - userDefined []triple - ) - - for i, name := range fieldNames { - ser := serializers[i] - tagID := TagIDUseFieldName // default: use field name - if tagIDs != nil && i < len(tagIDs) { - tagID = tagIDs[i] - } - if ser == nil { - others = append(others, triple{UNKNOWN, nil, name, nullables[i], tagID}) - continue - } - typeTriples = append(typeTriples, triple{typeIds[i], ser, name, nullables[i], tagID}) - } - // Java orders: primitives, boxed, finals, others, collections, maps - // primitives = non-nullable primitive types (int, long, etc.) - // boxed = nullable boxed types (Integer, Long, etc. which are pointers in Go) - var primitives, boxed, collection, setFields, maps, otherInternalTypeFields []triple - - for _, t := range typeTriples { - switch { - case isPrimitiveType(t.typeID): - // Separate non-nullable primitives from nullable (boxed) primitives - if t.nullable { - boxed = append(boxed, t) - } else { - primitives = append(primitives, t) - } - case isListType(t.typeID), isPrimitiveArrayType(t.typeID): - collection = append(collection, t) - case isSetType(t.typeID): - setFields = append(setFields, t) - case isMapType(t.typeID): - maps = append(maps, t) - case isUserDefinedType(t.typeID): - userDefined = append(userDefined, t) - case t.typeID == UNKNOWN: - others = append(others, t) - default: - otherInternalTypeFields = append(otherInternalTypeFields, t) - } - } - // Sort primitives (non-nullable) - same logic as boxed - // Java sorts by: compressed types last, then by size (largest first), then by type ID (descending) - sortPrimitiveSlice := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - ai, aj := s[i], s[j] - compressI := ai.typeID == INT32 || ai.typeID == INT64 || - ai.typeID == VAR_INT32 || ai.typeID == VAR_INT64 - compressJ := aj.typeID == INT32 || aj.typeID == INT64 || - aj.typeID == VAR_INT32 || aj.typeID == VAR_INT64 - if compressI != compressJ { - return !compressI && compressJ - } - szI, szJ := getPrimitiveTypeSize(ai.typeID), getPrimitiveTypeSize(aj.typeID) - if szI != szJ { - return szI > szJ - } - // Tie-breaker: type ID descending (higher type ID first), then field name - if ai.typeID != aj.typeID { - return ai.typeID > aj.typeID - } - return ai.getSortKey() < aj.getSortKey() - }) - } - sortPrimitiveSlice(primitives) - sortPrimitiveSlice(boxed) - sortByTypeIDThenName := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - if s[i].typeID != s[j].typeID { - return s[i].typeID < s[j].typeID - } - return s[i].getSortKey() < s[j].getSortKey() - }) - } - sortTuple := func(s []triple) { - sort.Slice(s, func(i, j int) bool { - return s[i].getSortKey() < s[j].getSortKey() - }) - } - sortByTypeIDThenName(otherInternalTypeFields) - sortTuple(others) - sortTuple(collection) - sortTuple(setFields) - sortTuple(maps) - sortTuple(userDefined) - - // Java order: primitives, boxed, finals, collections, maps, others - // finals = String and other monomorphic types (otherInternalTypeFields) - // others = userDefined types (structs, enums) and unknown types - all := make([]triple, 0, len(fieldNames)) - all = append(all, primitives...) - all = append(all, boxed...) - all = append(all, otherInternalTypeFields...) // finals (String, etc.) - all = append(all, collection...) - all = append(all, setFields...) - all = append(all, maps...) - all = append(all, userDefined...) // others (structs, enums) - all = append(all, others...) // unknown types - - outSer := make([]Serializer, len(all)) - outNam := make([]string, len(all)) - for i, t := range all { - outSer[i] = t.serializer - outNam[i] = t.name - } - return outSer, outNam + var ( + typeTriples []triple + others []triple + userDefined []triple + ) + + for i, name := range fieldNames { + ser := serializers[i] + tagID := TagIDUseFieldName // default: use field name + if tagIDs != nil && i < len(tagIDs) { + tagID = tagIDs[i] + } + if ser == nil { + others = append(others, triple{UNKNOWN, nil, name, nullables[i], tagID}) + continue + } + typeTriples = append(typeTriples, triple{typeIds[i], ser, name, nullables[i], tagID}) + } + // Java orders: primitives, boxed, finals, others, collections, maps + // primitives = non-nullable primitive types (int, long, etc.) + // boxed = nullable boxed types (Integer, Long, etc. which are pointers in Go) + var primitives, boxed, collection, setFields, maps, otherInternalTypeFields []triple + + for _, t := range typeTriples { + switch { + case isPrimitiveType(t.typeID): + // Separate non-nullable primitives from nullable (boxed) primitives + if t.nullable { + boxed = append(boxed, t) + } else { + primitives = append(primitives, t) + } + case isListType(t.typeID), isPrimitiveArrayType(t.typeID): + collection = append(collection, t) + case isSetType(t.typeID): + setFields = append(setFields, t) + case isMapType(t.typeID): + maps = append(maps, t) + case isUserDefinedType(t.typeID): + userDefined = append(userDefined, t) + case t.typeID == UNKNOWN: + others = append(others, t) + default: + otherInternalTypeFields = append(otherInternalTypeFields, t) + } + } + // Sort primitives (non-nullable) - same logic as boxed + // Java sorts by: compressed types last, then by size (largest first), then by type ID (descending) + sortPrimitiveSlice := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + ai, aj := s[i], s[j] + compressI := ai.typeID == INT32 || ai.typeID == INT64 || + ai.typeID == VAR_INT32 || ai.typeID == VAR_INT64 + compressJ := aj.typeID == INT32 || aj.typeID == INT64 || + aj.typeID == VAR_INT32 || aj.typeID == VAR_INT64 + if compressI != compressJ { + return !compressI && compressJ + } + szI, szJ := getPrimitiveTypeSize(ai.typeID), getPrimitiveTypeSize(aj.typeID) + if szI != szJ { + return szI > szJ + } + // Tie-breaker: type ID descending (higher type ID first), then field name + if ai.typeID != aj.typeID { + return ai.typeID > aj.typeID + } + return ai.getSortKey() < aj.getSortKey() + }) + } + sortPrimitiveSlice(primitives) + sortPrimitiveSlice(boxed) + sortByTypeIDThenName := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + if s[i].typeID != s[j].typeID { + return s[i].typeID < s[j].typeID + } + return s[i].getSortKey() < s[j].getSortKey() + }) + } + sortTuple := func(s []triple) { + sort.Slice(s, func(i, j int) bool { + return s[i].getSortKey() < s[j].getSortKey() + }) + } + sortByTypeIDThenName(otherInternalTypeFields) + sortTuple(others) + sortTuple(collection) + sortTuple(setFields) + sortTuple(maps) + sortTuple(userDefined) + + // Java order: primitives, boxed, finals, collections, maps, others + // finals = String and other monomorphic types (otherInternalTypeFields) + // others = userDefined types (structs, enums) and unknown types + all := make([]triple, 0, len(fieldNames)) + all = append(all, primitives...) + all = append(all, boxed...) + all = append(all, otherInternalTypeFields...) // finals (String, etc.) + all = append(all, collection...) + all = append(all, setFields...) + all = append(all, maps...) + all = append(all, userDefined...) // others (structs, enums) + all = append(all, others...) // unknown types + + outSer := make([]Serializer, len(all)) + outNam := make([]string, len(all)) + for i, t := range all { + outSer[i] = t.serializer + outNam[i] = t.name + } + return outSer, outNam } func typesCompatible(actual, expected reflect.Type) bool { - if actual == nil || expected == nil { - return false - } - if actual == expected { - return true - } - // interface{} can accept any value - if actual.Kind() == reflect.Interface && actual.NumMethod() == 0 { - return true - } - if actual.AssignableTo(expected) || expected.AssignableTo(actual) { - return true - } - if actual.Kind() == reflect.Ptr && actual.Elem() == expected { - return true - } - if expected.Kind() == reflect.Ptr && expected.Elem() == actual { - return true - } - if actual.Kind() == expected.Kind() { - switch actual.Kind() { - case reflect.Slice, reflect.Array: - return elementTypesCompatible(actual.Elem(), expected.Elem()) - case reflect.Map: - return elementTypesCompatible(actual.Key(), expected.Key()) && elementTypesCompatible(actual.Elem(), expected.Elem()) - } - } - if (actual.Kind() == reflect.Array && expected.Kind() == reflect.Slice) || - (actual.Kind() == reflect.Slice && expected.Kind() == reflect.Array) { - return true - } - return false + if actual == nil || expected == nil { + return false + } + if actual == expected { + return true + } + // interface{} can accept any value + if actual.Kind() == reflect.Interface && actual.NumMethod() == 0 { + return true + } + if actual.AssignableTo(expected) || expected.AssignableTo(actual) { + return true + } + if actual.Kind() == reflect.Ptr && actual.Elem() == expected { + return true + } + if expected.Kind() == reflect.Ptr && expected.Elem() == actual { + return true + } + if actual.Kind() == expected.Kind() { + switch actual.Kind() { + case reflect.Slice, reflect.Array: + return elementTypesCompatible(actual.Elem(), expected.Elem()) + case reflect.Map: + return elementTypesCompatible(actual.Key(), expected.Key()) && elementTypesCompatible(actual.Elem(), expected.Elem()) + } + } + if (actual.Kind() == reflect.Array && expected.Kind() == reflect.Slice) || + (actual.Kind() == reflect.Slice && expected.Kind() == reflect.Array) { + return true + } + return false } func elementTypesCompatible(actual, expected reflect.Type) bool { - if actual == nil || expected == nil { - return false - } - if actual == expected || actual.AssignableTo(expected) || expected.AssignableTo(actual) { - return true - } - if actual.Kind() == reflect.Ptr { - return elementTypesCompatible(actual, expected.Elem()) - } - return false + if actual == nil || expected == nil { + return false + } + if actual == expected || actual.AssignableTo(expected) || expected.AssignableTo(actual) { + return true + } + if actual.Kind() == reflect.Ptr { + return elementTypesCompatible(actual, expected.Elem()) + } + return false } // typeIdFromKind derives a TypeId from a reflect.Type's kind // This is used when the type is not registered in typesInfo func typeIdFromKind(type_ reflect.Type) TypeId { - switch type_.Kind() { - case reflect.Bool: - return BOOL - case reflect.Int8: - return INT8 - case reflect.Int16: - return INT16 - case reflect.Int32: - return INT32 - case reflect.Int64, reflect.Int: - return INT64 - case reflect.Uint8: - return UINT8 - case reflect.Uint16: - return UINT16 - case reflect.Uint32: - return UINT32 - case reflect.Uint64, reflect.Uint: - return UINT64 - case reflect.Float32: - return FLOAT - case reflect.Float64: - return DOUBLE - case reflect.String: - return STRING - case reflect.Slice: - // For slices, return the appropriate primitive array type ID based on element type - elemKind := type_.Elem().Kind() - switch elemKind { - case reflect.Bool: - return BOOL_ARRAY - case reflect.Int8: - return INT8_ARRAY - case reflect.Int16: - return INT16_ARRAY - case reflect.Int32: - return INT32_ARRAY - case reflect.Int64, reflect.Int: - return INT64_ARRAY - case reflect.Float32: - return FLOAT32_ARRAY - case reflect.Float64: - return FLOAT64_ARRAY - default: - // Non-primitive slices use LIST - return LIST - } - case reflect.Array: - // For arrays, return the appropriate primitive array type ID based on element type - elemKind := type_.Elem().Kind() - switch elemKind { - case reflect.Bool: - return BOOL_ARRAY - case reflect.Int8: - return INT8_ARRAY - case reflect.Int16: - return INT16_ARRAY - case reflect.Int32: - return INT32_ARRAY - case reflect.Int64, reflect.Int: - return INT64_ARRAY - case reflect.Float32: - return FLOAT32_ARRAY - case reflect.Float64: - return FLOAT64_ARRAY - default: - // Non-primitive arrays use LIST - return LIST - } - case reflect.Map: - // map[T]bool is used to represent a Set in Go - if type_.Elem().Kind() == reflect.Bool { - return SET - } - return MAP - case reflect.Struct: - return NAMED_STRUCT - case reflect.Ptr: - // For pointer types, get the type ID of the element type - return typeIdFromKind(type_.Elem()) - default: - return UNKNOWN - } + switch type_.Kind() { + case reflect.Bool: + return BOOL + case reflect.Int8: + return INT8 + case reflect.Int16: + return INT16 + case reflect.Int32: + return INT32 + case reflect.Int64, reflect.Int: + return INT64 + case reflect.Uint8: + return UINT8 + case reflect.Uint16: + return UINT16 + case reflect.Uint32: + return UINT32 + case reflect.Uint64, reflect.Uint: + return UINT64 + case reflect.Float32: + return FLOAT + case reflect.Float64: + return DOUBLE + case reflect.String: + return STRING + case reflect.Slice: + // For slices, return the appropriate primitive array type ID based on element type + elemKind := type_.Elem().Kind() + switch elemKind { + case reflect.Bool: + return BOOL_ARRAY + case reflect.Int8: + return INT8_ARRAY + case reflect.Int16: + return INT16_ARRAY + case reflect.Int32: + return INT32_ARRAY + case reflect.Int64, reflect.Int: + return INT64_ARRAY + case reflect.Float32: + return FLOAT32_ARRAY + case reflect.Float64: + return FLOAT64_ARRAY + default: + // Non-primitive slices use LIST + return LIST + } + case reflect.Array: + // For arrays, return the appropriate primitive array type ID based on element type + elemKind := type_.Elem().Kind() + switch elemKind { + case reflect.Bool: + return BOOL_ARRAY + case reflect.Int8: + return INT8_ARRAY + case reflect.Int16: + return INT16_ARRAY + case reflect.Int32: + return INT32_ARRAY + case reflect.Int64, reflect.Int: + return INT64_ARRAY + case reflect.Float32: + return FLOAT32_ARRAY + case reflect.Float64: + return FLOAT64_ARRAY + default: + // Non-primitive arrays use LIST + return LIST + } + case reflect.Map: + // map[T]bool is used to represent a Set in Go + if type_.Elem().Kind() == reflect.Bool { + return SET + } + return MAP + case reflect.Struct: + return NAMED_STRUCT + case reflect.Ptr: + // For pointer types, get the type ID of the element type + return typeIdFromKind(type_.Elem()) + default: + return UNKNOWN + } } // skipStructSerializer is a serializer that skips unknown struct data // It reads and discards field data based on fieldDefs from remote TypeDef type skipStructSerializer struct { - fieldDefs []FieldDef + fieldDefs []FieldDef } func (s *skipStructSerializer) WriteData(ctx *WriteContext, value reflect.Value) { - ctx.SetError(SerializationError("skipStructSerializer does not support WriteData - unknown struct type")) + ctx.SetError(SerializationError("skipStructSerializer does not support WriteData - unknown struct type")) } func (s *skipStructSerializer) Write(ctx *WriteContext, refMode RefMode, writeType bool, hasGenerics bool, value reflect.Value) { - ctx.SetError(SerializationError("skipStructSerializer does not support Write - unknown struct type")) + ctx.SetError(SerializationError("skipStructSerializer does not support Write - unknown struct type")) } func (s *skipStructSerializer) ReadData(ctx *ReadContext, type_ reflect.Type, value reflect.Value) { - // Skip all fields based on fieldDefs from remote TypeDef - for _, fieldDef := range s.fieldDefs { - isStructType := isStructFieldType(fieldDef.fieldType) - // Use trackingRef from FieldDef for ref flag decision - SkipFieldValueWithTypeFlag(ctx, fieldDef, fieldDef.trackingRef, ctx.Compatible() && isStructType) - if ctx.HasError() { - return - } - } + // Skip all fields based on fieldDefs from remote TypeDef + for _, fieldDef := range s.fieldDefs { + isStructType := isStructFieldType(fieldDef.fieldType) + // Use trackingRef from FieldDef for ref flag decision + SkipFieldValueWithTypeFlag(ctx, fieldDef, fieldDef.trackingRef, ctx.Compatible() && isStructType) + if ctx.HasError() { + return + } + } } func (s *skipStructSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, hasGenerics bool, value reflect.Value) { - buf := ctx.Buffer() - ctxErr := ctx.Err() - switch refMode { - case RefModeTracking: - refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) - if refErr != nil { - ctx.SetError(FromError(refErr)) - return - } - if refID < int32(NotNullValueFlag) { - // Reference found, nothing to skip - return - } - case RefModeNullOnly: - flag := buf.ReadInt8(ctxErr) - if flag == NullFlag { - return - } - } - if ctx.HasError() { - return - } - s.ReadData(ctx, nil, value) + buf := ctx.Buffer() + ctxErr := ctx.Err() + switch refMode { + case RefModeTracking: + refID, refErr := ctx.RefResolver().TryPreserveRefId(buf) + if refErr != nil { + ctx.SetError(FromError(refErr)) + return + } + if refID < int32(NotNullValueFlag) { + // Reference found, nothing to skip + return + } + case RefModeNullOnly: + flag := buf.ReadInt8(ctxErr) + if flag == NullFlag { + return + } + } + if ctx.HasError() { + return + } + s.ReadData(ctx, nil, value) } func (s *skipStructSerializer) ReadWithTypeInfo(ctx *ReadContext, refMode RefMode, typeInfo *TypeInfo, value reflect.Value) { - // typeInfo is already read, don't read it again - just skip data - s.Read(ctx, refMode, false, false, value) + // typeInfo is already read, don't read it again - just skip data + s.Read(ctx, refMode, false, false, value) } diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go index a45b3c2f75..18ed629fe1 100644 --- a/go/fory/type_resolver.go +++ b/go/fory/type_resolver.go @@ -1347,7 +1347,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s return nil, err } // Always use xlang mode (LIST typeId) for non-primitive slices - return newSliceConcreteValueSerializer(type_, elemSerializer, r.isXlang) + return newSliceSerializer(type_, elemSerializer, r.isXlang) } case reflect.Array: elem := type_.Elem() @@ -1454,7 +1454,7 @@ func (r *TypeResolver) createSerializer(type_ reflect.Type, mapInStruct bool) (s // GetSliceSerializer returns the appropriate serializer for a slice type. // For primitive element types (bool, int8, int16, int32, int64, uint8, float32, float64), // it returns the dedicated primitive slice serializer that uses ARRAY protocol. -// For non-primitive element types, it returns sliceConcreteValueSerializer (LIST protocol). +// For non-primitive element types, it returns sliceSerializer (LIST protocol). func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, error) { if sliceType.Kind() != reflect.Slice { return nil, fmt.Errorf("expected slice type but got %s", sliceType.Kind()) @@ -1483,12 +1483,12 @@ func (r *TypeResolver) GetSliceSerializer(sliceType reflect.Type) (Serializer, e case reflect.Uint: return uintSliceSerializer{}, nil } - // For non-primitive element types, use sliceConcreteValueSerializer + // For non-primitive element types, use sliceSerializer elemSerializer, err := r.getSerializerByType(elemType, false) if err != nil { return nil, err } - return newSliceConcreteValueSerializer(sliceType, elemSerializer, r.isXlang) + return newSliceSerializer(sliceType, elemSerializer, r.isXlang) } // GetSetSerializer returns the setSerializer for a map[T]bool type (used to represent sets in Go). @@ -1504,7 +1504,7 @@ func (r *TypeResolver) GetSetSerializer(setType reflect.Type) (Serializer, error // GetArraySerializer returns the appropriate serializer for an array type. // For primitive element types, it returns the dedicated primitive array serializer (ARRAY protocol). -// For non-primitive element types, it returns sliceConcreteValueSerializer (LIST protocol). +// For non-primitive element types, it returns sliceSerializer (LIST protocol). func (r *TypeResolver) GetArraySerializer(arrayType reflect.Type) (Serializer, error) { if arrayType.Kind() != reflect.Array { return nil, fmt.Errorf("expected array type but got %s", arrayType.Kind()) @@ -1535,12 +1535,12 @@ func (r *TypeResolver) GetArraySerializer(arrayType reflect.Type) (Serializer, e } return int32ArraySerializer{arrayType: arrayType}, nil } - // For non-primitive element types, use sliceConcreteValueSerializer + // For non-primitive element types, use sliceSerializer elemSerializer, err := r.getSerializerByType(elemType, false) if err != nil { return nil, err } - return newSliceConcreteValueSerializer(arrayType, elemSerializer, r.isXlang) + return newSliceSerializer(arrayType, elemSerializer, r.isXlang) } func isDynamicType(type_ reflect.Type) bool { From 5c6f262326b6e96ee5394eaf6e5e6258aa3baa98 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Sat, 3 Jan 2026 19:50:24 +0800 Subject: [PATCH 35/35] lint code --- cpp/fory/serialization/type_resolver.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc index c5045cbc70..3380c553d5 100644 --- a/cpp/fory/serialization/type_resolver.cc +++ b/cpp/fory/serialization/type_resolver.cc @@ -1013,7 +1013,7 @@ int32_t TypeMeta::compute_struct_version(const TypeMeta &meta) { << ", fingerprint=\"" << fingerprint << "\" version=" << static_cast(version) << std::endl; #endif - return static_cast(version); + return static_cast(version); } // ============================================================================