From e322f2a80869e0664360c30a2ac7bcd5d2275525 Mon Sep 17 00:00:00 2001 From: Karthik Date: Thu, 18 Jul 2024 14:58:21 +0530 Subject: [PATCH 1/6] wip --- pkg/repos/db-repo-mongo.go | 39 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index ef2a0b0fc..1eda70d1c 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "regexp" "strings" "time" @@ -171,6 +172,14 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat return nil, errors.Newf("paramter `before` requires paramter `last` to be specified") } + var cursorKey string + + if pagination.OrderBy == ""{ + cursorKey = "_id" + } else { + cursorKey = pagination.OrderBy + } + queryFilter := Filter{} for k, v := range filter { @@ -182,11 +191,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if err != nil { return nil, errors.NewE(err) } - objectID, err := primitive.ObjectIDFromHex(string(aft)) - if err != nil { - return nil, errors.NewE(err) - } - queryFilter["_id"] = bson.M{"$gt": objectID} + queryFilter[cursorKey] = bson.M{"$gte": string(aft)} } if pagination.Before != nil { @@ -194,11 +199,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if err != nil { return nil, errors.NewE(err) } - objectID, err := primitive.ObjectIDFromHex(string(bef)) - if err != nil { - return nil, errors.NewE(err) - } - queryFilter["_id"] = bson.M{"$lt": objectID} + queryFilter[cursorKey] = bson.M{"$lte": string(bef)} } var limit int64 @@ -210,7 +211,6 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat limit = *pagination.First + 1 } - // var results []T curr, err := repo.db.Collection(repo.collectionName).Find( ctx, queryFilter, &options.FindOptions{ Limit: &limit, @@ -238,9 +238,19 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat pageInfo := PageInfo{} + getCursorOfResult := func(r T) string { + if pagination.OrderBy == "" { + v := reflect.ValueOf(r) + field := v.FieldByName(pagination.OrderBy) + return CursorToBase64(Cursor(field.String())) + } else { + return CursorToBase64(Cursor(r.GetPrimitiveID())) + } + } + if len(results) > 0 { - pageInfo.StartCursor = CursorToBase64(Cursor(string(results[0].GetPrimitiveID()))) - pageInfo.EndCursor = CursorToBase64(Cursor(string(results[len(results)-1].GetPrimitiveID()))) + pageInfo.StartCursor = getCursorOfResult(results[0]) + pageInfo.EndCursor = getCursorOfResult(results[len(results)-1]) if pagination.First != nil { pageInfo.HasNextPage = fn.New(len(results) > int(*pagination.First)) @@ -253,7 +263,6 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if pagination.Last != nil { pageInfo.HasNextPage = fn.New(pagination.Before != nil) pageInfo.HasPrevPage = fn.New(len(results) > int(*pagination.Last)) - if pageInfo.HasPrevPage != nil && *pageInfo.HasPrevPage { results = results[:*pagination.Last] } @@ -264,7 +273,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat for i := range results { edges[i] = RecordEdge[T]{ Node: results[i], - Cursor: CursorToBase64(Cursor(results[i].GetPrimitiveID())), + Cursor: getCursorOfResult(results[i]), } } From 434b70fe3cd99d46bff786f1458c6c3c09e12a01 Mon Sep 17 00:00:00 2001 From: Karthik Date: Thu, 18 Jul 2024 15:47:46 +0530 Subject: [PATCH 2/6] wip --- pkg/repos/db-repo-mongo.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index 1eda70d1c..eff955e6a 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -155,6 +155,23 @@ func (repo *dbRepo[T]) FindOne(ctx context.Context, filter Filter) (T, error) { return item, nil } +func getFieldValueByName(s interface{}, fieldName string) (string, error) { + v := reflect.ValueOf(s) + + // Check if the provided interface is a pointer and get the value it points to + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + // Get the field by name + field := v.FieldByName(fieldName) + if !field.IsValid() { + return "", fmt.Errorf("no such field: %s in struct", fieldName) + } + + return field.String(), nil +} + func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, pagination CursorPagination) (*PaginatedRecord[T], error) { if pagination.First != nil && pagination.Last != nil { return nil, errors.Newf("first/last only one of these parameters could be passed on, you have specified both") @@ -238,14 +255,15 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat pageInfo := PageInfo{} - getCursorOfResult := func(r T) string { - if pagination.OrderBy == "" { - v := reflect.ValueOf(r) - field := v.FieldByName(pagination.OrderBy) - return CursorToBase64(Cursor(field.String())) - } else { - return CursorToBase64(Cursor(r.GetPrimitiveID())) + getCursorOfResult := func(r T) (string) { + if cursorKey == "_id" { + return CursorToBase64(Cursor(r.GetId())) + } + val, _ := getFieldValueByName(r, pagination.OrderBy) + if err != nil { + return "" } + return CursorToBase64(Cursor(string(val))) } if len(results) > 0 { From f8920e1301170f8799aa0d9a0256013a01ed332f Mon Sep 17 00:00:00 2001 From: Karthik Date: Thu, 18 Jul 2024 16:10:00 +0530 Subject: [PATCH 3/6] wip --- go.mod | 2 ++ go.sum | 7 ++++-- pkg/repos/db-repo-mongo.go | 47 ++++++++++++++++---------------------- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 9a1985795..70abb8efb 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( ) require ( + github.com/PaesslerAG/jsonpath v0.1.1 github.com/go-chi/chi/v5 v5.0.10 github.com/kloudlite/container-registry-authorizer v0.0.0-20231021122509-161dc30fde55 github.com/miekg/dns v1.1.55 @@ -58,6 +59,7 @@ require ( ) require ( + github.com/PaesslerAG/gval v1.0.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/lipgloss v0.10.0 // indirect github.com/charmbracelet/log v0.4.0 // indirect diff --git a/go.sum b/go.sum index e277383b8..4637c250f 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,11 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= +github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8= +github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= +github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= @@ -175,8 +180,6 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kloudlite/container-registry-authorizer v0.0.0-20231021122509-161dc30fde55 h1:YnZh3TL6AG4EfoInx1/L5zcPHd2QxgLKseJB1KtHjdQ= github.com/kloudlite/container-registry-authorizer v0.0.0-20231021122509-161dc30fde55/go.mod h1:GZj3wZmIw/qCciclRhgQTgmGiqe8wxoVzMXQjbOfnbc= -github.com/kloudlite/operator v0.0.0-20240710071747-9a61e7de9e93 h1:vbF6PPTjgmtE5pNHKdZmTMmjfC4njjGW1EO8m7Njx1w= -github.com/kloudlite/operator v0.0.0-20240710071747-9a61e7de9e93/go.mod h1:c6FiZvYztvr92/UcIUvQurp3oWMrrEK7deAriHckTPw= github.com/kloudlite/operator v0.0.0-20240718072819-c625b77c43b2 h1:K6tlpBcl4PWv6ZiojFTXADYuAvGjcOkrYQbsPvbXaOU= github.com/kloudlite/operator v0.0.0-20240718072819-c625b77c43b2/go.mod h1:c6FiZvYztvr92/UcIUvQurp3oWMrrEK7deAriHckTPw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index eff955e6a..4baeef923 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -4,13 +4,13 @@ import ( "context" "encoding/json" "fmt" - "reflect" + "github.com/PaesslerAG/jsonpath" + "go.mongodb.org/mongo-driver/bson/primitive" "regexp" "strings" "time" "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/fx" "github.com/kloudlite/api/pkg/errors" @@ -155,23 +155,6 @@ func (repo *dbRepo[T]) FindOne(ctx context.Context, filter Filter) (T, error) { return item, nil } -func getFieldValueByName(s interface{}, fieldName string) (string, error) { - v := reflect.ValueOf(s) - - // Check if the provided interface is a pointer and get the value it points to - if v.Kind() == reflect.Ptr { - v = v.Elem() - } - - // Get the field by name - field := v.FieldByName(fieldName) - if !field.IsValid() { - return "", fmt.Errorf("no such field: %s in struct", fieldName) - } - - return field.String(), nil -} - func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, pagination CursorPagination) (*PaginatedRecord[T], error) { if pagination.First != nil && pagination.Last != nil { return nil, errors.Newf("first/last only one of these parameters could be passed on, you have specified both") @@ -255,20 +238,26 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat pageInfo := PageInfo{} - getCursorOfResult := func(r T) (string) { + getCursorOfResult := func(r T) (string, error) { if cursorKey == "_id" { - return CursorToBase64(Cursor(r.GetId())) + return CursorToBase64(Cursor(r.GetId())), nil } - val, _ := getFieldValueByName(r, pagination.OrderBy) + val, err := jsonpath.Get(fmt.Sprintf("$.%s", cursorKey), r) if err != nil { - return "" + return "", errors.NewE(err) } - return CursorToBase64(Cursor(string(val))) + return CursorToBase64(Cursor(fmt.Sprintf("%v", val ))), nil } if len(results) > 0 { - pageInfo.StartCursor = getCursorOfResult(results[0]) - pageInfo.EndCursor = getCursorOfResult(results[len(results)-1]) + pageInfo.StartCursor, err = getCursorOfResult(results[0]) + if err != nil { + return nil, errors.NewE(err) + } + pageInfo.EndCursor, err = getCursorOfResult(results[len(results)-1]) + if err != nil { + return nil, errors.NewE(err) + } if pagination.First != nil { pageInfo.HasNextPage = fn.New(len(results) > int(*pagination.First)) @@ -289,9 +278,13 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat edges := make([]RecordEdge[T], len(results)) for i := range results { + c, err:= getCursorOfResult(results[i]) + if err != nil { + return nil, errors.NewE(err) + } edges[i] = RecordEdge[T]{ Node: results[i], - Cursor: getCursorOfResult(results[i]), + Cursor:c, } } From 4574e0662a5715d1f49c04da2791b96fee14f505 Mon Sep 17 00:00:00 2001 From: Karthik Date: Thu, 18 Jul 2024 16:21:30 +0530 Subject: [PATCH 4/6] wip --- pkg/repos/db-repo-mongo.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index 4baeef923..53b5e48cd 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -82,24 +82,24 @@ func bsonToStruct[T any](r *mongo.SingleResult) (T, error) { return result, nil } -func cursorToStruct[T any](ctx context.Context, curr *mongo.Cursor) ([]T, error) { +func cursorToStruct[T any](ctx context.Context, curr *mongo.Cursor) ([]T, []map[string]any, error) { var m []map[string]any var results []T if err := curr.All(ctx, &m); err != nil { - return results, errors.NewE(err) + return results, m,errors.NewE(err) } b, err := json.Marshal(m) if err != nil { - return results, errors.NewE(err) + return results, m,errors.NewE(err) } if err := json.Unmarshal(b, &results); err != nil { - return results, errors.NewE(err) + return results, m,errors.NewE(err) } - return results, nil + return results, m,nil } func (repo *dbRepo[T]) NewId() ID { @@ -123,8 +123,8 @@ func (repo *dbRepo[T]) Find(ctx context.Context, query Query) ([]T, error) { } return nil, errors.NewE(err) } - - return cursorToStruct[T](ctx, curr) + toStruct, _, err := cursorToStruct[T](ctx, curr) + return toStruct, err } func (repo *dbRepo[T]) Count(ctx context.Context, filter Filter) (int64, error) { @@ -226,7 +226,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat return nil, errors.NewE(err) } - results, err := cursorToStruct[T](ctx, curr) + results, rawResults, err := cursorToStruct[T](ctx, curr) if err != nil { return nil, errors.NewE(err) } @@ -238,11 +238,11 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat pageInfo := PageInfo{} - getCursorOfResult := func(r T) (string, error) { + getCursorOfResult := func(r T,m map[string]any) (string, error) { if cursorKey == "_id" { return CursorToBase64(Cursor(r.GetId())), nil } - val, err := jsonpath.Get(fmt.Sprintf("$.%s", cursorKey), r) + val, err := jsonpath.Get(fmt.Sprintf("$.%s", cursorKey), m) if err != nil { return "", errors.NewE(err) } @@ -250,11 +250,11 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat } if len(results) > 0 { - pageInfo.StartCursor, err = getCursorOfResult(results[0]) + pageInfo.StartCursor, err = getCursorOfResult(results[0], rawResults[0]) if err != nil { return nil, errors.NewE(err) } - pageInfo.EndCursor, err = getCursorOfResult(results[len(results)-1]) + pageInfo.EndCursor, err = getCursorOfResult(results[len(results)-1], rawResults[len(results)-1]) if err != nil { return nil, errors.NewE(err) } @@ -278,7 +278,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat edges := make([]RecordEdge[T], len(results)) for i := range results { - c, err:= getCursorOfResult(results[i]) + c, err:= getCursorOfResult(results[i], rawResults[i]) if err != nil { return nil, errors.NewE(err) } From 59aa2d71c8154aa3ed310a562c56bd267c02a3eb Mon Sep 17 00:00:00 2001 From: Karthik Date: Thu, 18 Jul 2024 16:33:14 +0530 Subject: [PATCH 5/6] wip --- pkg/repos/db-repo-mongo.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index 53b5e48cd..f033d1795 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -250,14 +250,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat } if len(results) > 0 { - pageInfo.StartCursor, err = getCursorOfResult(results[0], rawResults[0]) - if err != nil { - return nil, errors.NewE(err) - } - pageInfo.EndCursor, err = getCursorOfResult(results[len(results)-1], rawResults[len(results)-1]) - if err != nil { - return nil, errors.NewE(err) - } + if pagination.First != nil { pageInfo.HasNextPage = fn.New(len(results) > int(*pagination.First)) @@ -274,6 +267,15 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat results = results[:*pagination.Last] } } + + pageInfo.StartCursor, err = getCursorOfResult(results[0], rawResults[0]) + if err != nil { + return nil, errors.NewE(err) + } + pageInfo.EndCursor, err = getCursorOfResult(results[len(results)-1], rawResults[len(results)-1]) + if err != nil { + return nil, errors.NewE(err) + } } edges := make([]RecordEdge[T], len(results)) From 2695d836caf12da4382a78f8114dcf7f4a12f7f5 Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Thu, 18 Jul 2024 17:40:09 +0530 Subject: [PATCH 6/6] :bug: Fixed cursor-pagination with sorting --- pkg/repos/db-repo-mongo.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/pkg/repos/db-repo-mongo.go b/pkg/repos/db-repo-mongo.go index f033d1795..5f7e2b2c7 100644 --- a/pkg/repos/db-repo-mongo.go +++ b/pkg/repos/db-repo-mongo.go @@ -87,19 +87,19 @@ func cursorToStruct[T any](ctx context.Context, curr *mongo.Cursor) ([]T, []map[ var results []T if err := curr.All(ctx, &m); err != nil { - return results, m,errors.NewE(err) + return results, m, errors.NewE(err) } b, err := json.Marshal(m) if err != nil { - return results, m,errors.NewE(err) + return results, m, errors.NewE(err) } if err := json.Unmarshal(b, &results); err != nil { - return results, m,errors.NewE(err) + return results, m, errors.NewE(err) } - return results, m,nil + return results, m, nil } func (repo *dbRepo[T]) NewId() ID { @@ -174,7 +174,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat var cursorKey string - if pagination.OrderBy == ""{ + if pagination.OrderBy == "" { cursorKey = "_id" } else { cursorKey = pagination.OrderBy @@ -191,7 +191,13 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if err != nil { return nil, errors.NewE(err) } - queryFilter[cursorKey] = bson.M{"$gte": string(aft)} + + if pagination.SortDirection == SortDirectionAsc { + queryFilter[cursorKey] = bson.M{"$gte": string(aft)} + } else { + queryFilter[cursorKey] = bson.M{"$lte": string(aft)} + } + } if pagination.Before != nil { @@ -199,7 +205,12 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if err != nil { return nil, errors.NewE(err) } - queryFilter[cursorKey] = bson.M{"$lte": string(bef)} + + if pagination.SortDirection == SortDirectionAsc { + queryFilter[cursorKey] = bson.M{"$lte": string(bef)} + } else { + queryFilter[cursorKey] = bson.M{"$gte": string(bef)} + } } var limit int64 @@ -238,7 +249,7 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat pageInfo := PageInfo{} - getCursorOfResult := func(r T,m map[string]any) (string, error) { + getCursorOfResult := func(r T, m map[string]any) (string, error) { if cursorKey == "_id" { return CursorToBase64(Cursor(r.GetId())), nil } @@ -246,12 +257,11 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat if err != nil { return "", errors.NewE(err) } - return CursorToBase64(Cursor(fmt.Sprintf("%v", val ))), nil + return CursorToBase64(Cursor(fmt.Sprintf("%v", val))), nil } if len(results) > 0 { - if pagination.First != nil { pageInfo.HasNextPage = fn.New(len(results) > int(*pagination.First)) if pageInfo.HasNextPage != nil && *pageInfo.HasNextPage { @@ -280,13 +290,13 @@ func (repo *dbRepo[T]) FindPaginated(ctx context.Context, filter Filter, paginat edges := make([]RecordEdge[T], len(results)) for i := range results { - c, err:= getCursorOfResult(results[i], rawResults[i]) + c, err := getCursorOfResult(results[i], rawResults[i]) if err != nil { return nil, errors.NewE(err) } edges[i] = RecordEdge[T]{ Node: results[i], - Cursor:c, + Cursor: c, } }