From d13c93a293703c8ea75038a84e07f01792384cbf Mon Sep 17 00:00:00 2001 From: Lucas Fontes Date: Mon, 9 Dec 2024 17:14:05 -0500 Subject: [PATCH 1/3] cm: Implement json Marshal/Unmarshal for List type Signed-off-by: Lucas Fontes --- cm/list.go | 34 +++++++++++++++++++++++++++++++- cm/list_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/cm/list.go b/cm/list.go index 5c896d04..0393eb03 100644 --- a/cm/list.go +++ b/cm/list.go @@ -1,6 +1,15 @@ package cm -import "unsafe" +import ( + "bytes" + "encoding/json" + "unsafe" +) + +// nullLiteral is the JSON representation of a null literal. +// https://pkg.go.dev/encoding/json#Unmarshaler +// By convention, to approximate the behavior of Unmarshal itself, Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. +var nullLiteral = []byte("null") // List represents a Component Model list. // The binary representation of list is similar to a Go slice minus the cap field. @@ -58,3 +67,26 @@ func (l list[T]) Data() *T { func (l list[T]) Len() uintptr { return l.len } + +// MarshalJSON implements json.Marshaler. +func (l list[T]) MarshalJSON() ([]byte, error) { + return json.Marshal(l.Slice()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (l *list[T]) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, nullLiteral) { + return nil + } + + var s []T + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + l.data = unsafe.SliceData([]T(s)) + l.len = uintptr(len(s)) + + return nil +} diff --git a/cm/list_test.go b/cm/list_test.go index 34d674ac..90aa836d 100644 --- a/cm/list_test.go +++ b/cm/list_test.go @@ -2,6 +2,8 @@ package cm import ( "bytes" + "encoding/json" + "reflect" "testing" ) @@ -14,3 +16,53 @@ func TestListMethods(t *testing.T) { t.Errorf("got (%s) != want (%s)", string(got), string(want)) } } + +func TestListJSON(t *testing.T) { + simpleList := []string{"one", "two", "three"} + simpleJSON, err := json.Marshal(simpleList) + if err != nil { + t.Fatal(err) + } + + t.Run("encoding", func(t *testing.T) { + l := ToList(simpleList) + data, err := json.Marshal(l) + if err != nil { + t.Fatal(err) + } + + if got, want := data, simpleJSON; !bytes.Equal(got, want) { + t.Errorf("got (%s) != want (%s)", string(got), string(want)) + } + + var emptyList List[string] + data, err = json.Marshal(emptyList) + if err != nil { + t.Fatal(err) + } + + if got, want := data, nullLiteral; !bytes.Equal(got, want) { + t.Errorf(" got (%s) when should have got nil", string(data)) + } + }) + + t.Run("decoding", func(t *testing.T) { + var decodedList List[string] + if err := json.Unmarshal(simpleJSON, &decodedList); err != nil { + t.Fatal(err) + } + + if got, want := decodedList.Slice(), simpleList; !reflect.DeepEqual(got, want) { + t.Errorf("got (%s) != want (%s)", got, want) + } + + var emptyList List[string] + if err := json.Unmarshal(nullLiteral, &emptyList); err != nil { + t.Fatal(err) + } + + if got, want := emptyList.Slice(), []string(nil); !reflect.DeepEqual(got, want) { + t.Errorf("got (%+v) != want (%+v)", got, want) + } + }) +} From 6d126d7d3dde473832f87c018e64d64b67ebc05b Mon Sep 17 00:00:00 2001 From: Lucas Fontes Date: Tue, 10 Dec 2024 17:46:53 -0500 Subject: [PATCH 2/3] cm: Add json tests to List type Signed-off-by: Lucas Fontes --- cm/list.go | 19 ++- cm/list_test.go | 345 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 317 insertions(+), 47 deletions(-) diff --git a/cm/list.go b/cm/list.go index 0393eb03..b88ce397 100644 --- a/cm/list.go +++ b/cm/list.go @@ -3,6 +3,8 @@ package cm import ( "bytes" "encoding/json" + "fmt" + "strings" "unsafe" ) @@ -70,7 +72,22 @@ func (l list[T]) Len() uintptr { // MarshalJSON implements json.Marshaler. func (l list[T]) MarshalJSON() ([]byte, error) { - return json.Marshal(l.Slice()) + s := l.Slice() + + if s == nil { + return nullLiteral, nil + } + + // NOTE(lxf): Go JSON Encoder will serialize []byte as base64. + // We override that behavior so all int types have the same serialization format. + // []uint8{1,2,3} -> [1,2,3] + // []uint32{1,2,3} -> [1,2,3] + if byteArray, ok := any(s).([]byte); ok { + encoded := strings.Join(strings.Fields(fmt.Sprintf("%d", byteArray)), ",") + return []byte(encoded), nil + } + + return json.Marshal(s) } // UnmarshalJSON implements json.Unmarshaler. diff --git a/cm/list_test.go b/cm/list_test.go index 90aa836d..64560b3c 100644 --- a/cm/list_test.go +++ b/cm/list_test.go @@ -3,6 +3,8 @@ package cm import ( "bytes" "encoding/json" + "fmt" + "math" "reflect" "testing" ) @@ -17,52 +19,303 @@ func TestListMethods(t *testing.T) { } } -func TestListJSON(t *testing.T) { - simpleList := []string{"one", "two", "three"} - simpleJSON, err := json.Marshal(simpleList) - if err != nil { - t.Fatal(err) +type listTestItem struct { + Name string `json:"name"` + Age int `json:"age"` +} + +type listTestInvalid struct { + Name string `json:"name"` + Age int `json:"age"` +} + +type listTestWrapper[T comparable] struct { + raw string + outerList List[T] + innerList []T + err bool +} + +func (w *listTestWrapper[T]) wantErr() bool { + return w.err +} + +func (w *listTestWrapper[T]) outer() any { + return &w.outerList +} + +func (w *listTestWrapper[T]) outerSlice() any { + return w.outerList.Slice() +} + +func (w *listTestWrapper[T]) inner() any { + return w.innerList +} + +func (w *listTestWrapper[T]) rawData() string { + return w.raw +} + +func newListEncoder[T comparable](raw string, want []T, wantErr bool) *listTestWrapper[T] { + return &listTestWrapper[T]{raw: raw, outerList: ToList(want), err: wantErr} +} + +func newListDecoder[T comparable](raw string, want []T, wantErr bool) *listTestWrapper[T] { + return &listTestWrapper[T]{raw: raw, innerList: want, err: wantErr} +} + +type listTester interface { + outer() any + inner() any + outerSlice() any + wantErr() bool + rawData() string +} + +func (_ listTestInvalid) MarshalJSON() ([]byte, error) { + return nil, fmt.Errorf("can't encode") +} + +func (_ *listTestInvalid) UnmarshalJSON(_ []byte) error { + return fmt.Errorf("can't decode") +} + +func TestListMarshalJSON(t *testing.T) { + tests := []struct { + name string + w listTester + }{ + { + name: "encode error", + w: newListEncoder(``, []listTestInvalid{{}}, true), + }, + { + name: "f32 nan", + w: newListEncoder(``, []float32{float32(math.NaN())}, true), + }, + { + name: "f64 nan", + w: newListEncoder(``, []float64{float64(math.NaN())}, true), + }, + { + name: "null", + w: newListEncoder[string](`null`, nil, false), + }, + { + name: "empty", + w: newListEncoder(`[]`, []string{}, false), + }, + { + name: "bool", + w: newListEncoder(`[true,false]`, []bool{true, false}, false), + }, + { + name: "string", + w: newListEncoder(`["one","two","three"]`, []string{"one", "two", "three"}, false), + }, + { + name: "char", + w: newListEncoder(`[104,105,127942]`, []rune{'h', 'i', '🏆'}, false), + }, + { + name: "s8", + w: newListEncoder(`[123,-123,127]`, []int8{123, -123, math.MaxInt8}, false), + }, + { + name: "u8", + w: newListEncoder(`[123,0,255]`, []uint8{123, 0, math.MaxUint8}, false), + }, + { + name: "s16", + w: newListEncoder(`[123,-123,32767]`, []int16{123, -123, math.MaxInt16}, false), + }, + { + name: "u16", + w: newListEncoder(`[123,0,65535]`, []uint16{123, 0, math.MaxUint16}, false), + }, + { + name: "s32", + w: newListEncoder(`[123,-123,2147483647]`, []int32{123, -123, math.MaxInt32}, false), + }, + { + name: "u32", + w: newListEncoder(`[123,0,4294967295]`, []uint32{123, 0, math.MaxUint32}, false), + }, + { + name: "s64", + w: newListEncoder(`[123,-123,9223372036854775807]`, []int64{123, -123, math.MaxInt64}, false), + }, + { + name: "u64", + w: newListEncoder(`[123,0,18446744073709551615]`, []uint64{123, 0, math.MaxUint64}, false), + }, + { + name: "f32", + w: newListEncoder(`[1.01,2,3.4028235e+38]`, []float32{1.01, 2, math.MaxFloat32}, false), + }, + { + name: "f64", + w: newListEncoder(`[1.01,2,1.7976931348623157e+308]`, []float64{1.01, 2, math.MaxFloat64}, false), + }, + { + name: "struct", + w: newListEncoder(`[{"name":"joe","age":10},{"name":"jane","age":20}]`, []listTestItem{{Name: "joe", Age: 10}, {Name: "jane", Age: 20}}, false), + }, + { + name: "list", + w: newListEncoder(`[["one","two","three"],["four","five","six"]]`, []List[string]{ToList([]string{"one", "two", "three"}), ToList([]string{"four", "five", "six"})}, false), + }, } - t.Run("encoding", func(t *testing.T) { - l := ToList(simpleList) - data, err := json.Marshal(l) - if err != nil { - t.Fatal(err) - } - - if got, want := data, simpleJSON; !bytes.Equal(got, want) { - t.Errorf("got (%s) != want (%s)", string(got), string(want)) - } - - var emptyList List[string] - data, err = json.Marshal(emptyList) - if err != nil { - t.Fatal(err) - } - - if got, want := data, nullLiteral; !bytes.Equal(got, want) { - t.Errorf(" got (%s) when should have got nil", string(data)) - } - }) - - t.Run("decoding", func(t *testing.T) { - var decodedList List[string] - if err := json.Unmarshal(simpleJSON, &decodedList); err != nil { - t.Fatal(err) - } - - if got, want := decodedList.Slice(), simpleList; !reflect.DeepEqual(got, want) { - t.Errorf("got (%s) != want (%s)", got, want) - } - - var emptyList List[string] - if err := json.Unmarshal(nullLiteral, &emptyList); err != nil { - t.Fatal(err) - } - - if got, want := emptyList.Slice(), []string(nil); !reflect.DeepEqual(got, want) { - t.Errorf("got (%+v) != want (%+v)", got, want) - } - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.w.outer()) + if err != nil { + if tt.w.wantErr() { + return + } + + t.Fatal(err) + } + + if tt.w.wantErr() { + t.Fatalf("expect error, but got none. got (%s)", string(data)) + } + + if got, want := data, []byte(tt.w.rawData()); !bytes.Equal(got, want) { + t.Errorf("got (%v) != want (%v)", string(got), string(want)) + } + }) + } +} + +func TestListUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + w listTester + }{ + { + name: "decode error", + w: newListDecoder(`["joe"]`, []listTestInvalid{}, true), + }, + { + name: "invalid json", + w: newListDecoder(`[joe]`, []string{}, true), + }, + { + name: "incompatible type", + w: newListDecoder(`[123,456]`, []string{}, true), + }, + { + name: "incompatible bool", + w: newListDecoder(`["true","false"]`, []bool{true, false}, true), + }, + { + name: "incompatible s32", + w: newListDecoder(`["123","-123","2147483647"]`, []int32{}, true), + }, + { + name: "incompatible u32", + w: newListDecoder(`["123","0","4294967295"]`, []uint32{}, true), + }, + + { + name: "null", + w: newListDecoder[string](`null`, nil, false), + }, + { + name: "empty", + w: newListDecoder(`[]`, []string{}, false), + }, + { + name: "bool", + w: newListDecoder(`[true,false]`, []bool{true, false}, false), + }, + { + name: "string", + w: newListDecoder(`["one","two","three"]`, []string{"one", "two", "three"}, false), + }, + { + name: "char", + w: newListDecoder(`[104,105,127942]`, []rune{'h', 'i', '🏆'}, false), + }, + { + name: "s8", + w: newListDecoder(`[123,-123,127]`, []int8{123, -123, math.MaxInt8}, false), + }, + { + name: "u8", + w: newListDecoder(`[123,0,255]`, []uint8{123, 0, math.MaxUint8}, false), + }, + { + name: "s16", + w: newListDecoder(`[123,-123,32767]`, []int16{123, -123, math.MaxInt16}, false), + }, + { + name: "u16", + w: newListDecoder(`[123,0,65535]`, []uint16{123, 0, math.MaxUint16}, false), + }, + { + name: "s32", + w: newListDecoder(`[123,-123,2147483647]`, []int32{123, -123, math.MaxInt32}, false), + }, + { + name: "u32", + w: newListDecoder(`[123,0,4294967295]`, []uint32{123, 0, math.MaxUint32}, false), + }, + { + name: "s64", + w: newListDecoder(`[123,-123,9223372036854775807]`, []int64{123, -123, math.MaxInt64}, false), + }, + { + name: "u64", + w: newListDecoder(`[123,0,18446744073709551615]`, []uint64{123, 0, math.MaxUint64}, false), + }, + { + name: "f32", + w: newListDecoder(`[1.01,2,3.4028235e+38]`, []float32{1.01, 2, math.MaxFloat32}, false), + }, + { + name: "f32 nan", + w: newListDecoder(`[null]`, []float32{0}, false), + }, + { + name: "f64", + w: newListDecoder(`[1.01,2,1.7976931348623157e+308]`, []float64{1.01, 2, math.MaxFloat64}, false), + }, + { + name: "f64 nan", + w: newListDecoder(`[null]`, []float64{0}, false), + }, + { + name: "struct", + w: newListDecoder(`[{"name":"joe","age":10},{"name":"jane","age":20}]`, []listTestItem{{Name: "joe", Age: 10}, {Name: "jane", Age: 20}}, false), + }, + { + name: "list", + w: newListDecoder(`[["one","two","three"],["four","five","six"]]`, []List[string]{ToList([]string{"one", "two", "three"}), ToList([]string{"four", "five", "six"})}, false), + }, + // tuple, result, option, and variant needs json implementation + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := json.Unmarshal([]byte(tt.w.rawData()), tt.w.outer()) + if err != nil { + if tt.w.wantErr() { + return + } + + t.Fatal(err) + } + + if tt.w.wantErr() { + t.Fatalf("expect error, but got none. got (%v)", tt.w.outerSlice()) + } + + if got, want := tt.w.outerSlice(), tt.w.inner(); !reflect.DeepEqual(got, want) { + t.Errorf("got (%v) != want (%v)", got, want) + } + }) + } } From c4cd8d8c26509266647cf502a11543e3847595ae Mon Sep 17 00:00:00 2001 From: Lucas Fontes Date: Tue, 10 Dec 2024 20:30:12 -0500 Subject: [PATCH 3/3] cm: better uint8 handling & skipping tinygo 'defer' Signed-off-by: Lucas Fontes --- cm/list.go | 21 +++++++++++++-------- cm/list_test.go | 8 ++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/cm/list.go b/cm/list.go index b88ce397..57719681 100644 --- a/cm/list.go +++ b/cm/list.go @@ -3,8 +3,6 @@ package cm import ( "bytes" "encoding/json" - "fmt" - "strings" "unsafe" ) @@ -82,12 +80,7 @@ func (l list[T]) MarshalJSON() ([]byte, error) { // We override that behavior so all int types have the same serialization format. // []uint8{1,2,3} -> [1,2,3] // []uint32{1,2,3} -> [1,2,3] - if byteArray, ok := any(s).([]byte); ok { - encoded := strings.Join(strings.Fields(fmt.Sprintf("%d", byteArray)), ",") - return []byte(encoded), nil - } - - return json.Marshal(s) + return json.Marshal(sliceOf(s)) } // UnmarshalJSON implements json.Unmarshaler. @@ -107,3 +100,15 @@ func (l *list[T]) UnmarshalJSON(data []byte) error { return nil } + +type entry[T any] [1]T + +func (v entry[T]) MarshalJSON() ([]byte, error) { + return json.Marshal(v[0]) +} + +type slice[T any] []entry[T] + +func sliceOf[S ~[]E, E any](s S) slice[E] { + return *(*slice[E])(unsafe.Pointer(&s)) +} diff --git a/cm/list_test.go b/cm/list_test.go index 64560b3c..bf0d041b 100644 --- a/cm/list_test.go +++ b/cm/list_test.go @@ -6,6 +6,8 @@ import ( "fmt" "math" "reflect" + "runtime" + "strings" "testing" ) @@ -169,6 +171,12 @@ func TestListMarshalJSON(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // NOTE(lxf): skip marshal errors in tinygo as it uses 'defer' + // needs tinygo 0.35-dev + if tt.w.wantErr() && runtime.Compiler == "tinygo" && strings.Contains(runtime.GOARCH, "wasm") { + return + } + data, err := json.Marshal(tt.w.outer()) if err != nil { if tt.w.wantErr() {