From 449c3a828ce8ca7d046f777bf08c0606fec07186 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Sat, 19 Sep 2020 12:22:57 -0300 Subject: [PATCH 01/26] Add three new triples to the ?test graph of "planner_test.go" (to better test FILTER) Two new triples were added so we could have, for both predicate and object positions, examples on which two triples would have the same latest anchor (expecting 2 rows for "FILTER latest" in this case, in the place of only 1 as it was in the usual case tested so far). For that, the triples added were the one with predicate ""bought"@[2016-04-01T00:00:00-08:00]" and the other with object ""turned"@[2016-04-01T00:00:00-08:00]". The third triple was added so that both "/u" and "/u" would have in common two temporal predicates - this way we can test if only one "FILTER latest(?p)" is working as expected for multiple graph clauses inside of WHERE (if they share that same binding "?p"). For this, the triple added was the one with predicate ""bought"@[2016-01-01T00:00:00-08:00]". --- bql/planner/planner_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bql/planner/planner_test.go b/bql/planner/planner_test.go index 0ed651f8..bd833157 100644 --- a/bql/planner/planner_test.go +++ b/bql/planner/planner_test.go @@ -39,6 +39,8 @@ const ( /u "bought"@[2016-02-01T00:00:00-08:00] /c /u "bought"@[2016-03-01T00:00:00-08:00] /c /u "bought"@[2016-04-01T00:00:00-08:00] /c + /u "bought"@[2016-01-01T00:00:00-08:00] /c + /u "bought"@[2016-04-01T00:00:00-08:00] /c /c "is_a"@[] /t /c "is_a"@[] /t /c "is_a"@[] /t @@ -48,6 +50,7 @@ const ( /l "predicate"@[] "turned"@[2016-03-01T00:00:00-08:00] /l "predicate"@[] "turned"@[2016-04-01T00:00:00-08:00] /l "predicate"@[] "immutable_predicate"@[] + /l "predicate"@[] "turned"@[2016-04-01T00:00:00-08:00] /u "height_cm"@[] "174"^^type:int64 /u "tag"@[] "abc"^^type:text /u "height_cm"@[] "151"^^type:int64 @@ -840,7 +843,7 @@ func TestPlannerQuery(t *testing.T) { } HAVING (?p_id < "in"^^type:text) AND (?time > 2016-02-01T00:00:00-08:00);`, nBindings: 4, - nRows: 2, + nRows: 3, }, { q: `SELECT ?p, ?time @@ -958,7 +961,7 @@ func TestPlannerQuery(t *testing.T) { ?s ?p ?o AT ?o_time };`, nBindings: 3, - nRows: 4, + nRows: 5, }, { q: `SELECT ?s, ?o, ?o_time @@ -978,7 +981,7 @@ func TestPlannerQuery(t *testing.T) { ?s ?p "turned"@[?o_time] };`, nBindings: 2, - nRows: 4, + nRows: 5, }, } From e30c7cde373e2b3f889b9db6bb50bb047ec936c8 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Tue, 25 Aug 2020 15:48:16 -0300 Subject: [PATCH 02/26] Add tests for FILTER with latest anchor (in "planner_test.go") --- bql/planner/planner_test.go | 158 ++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/bql/planner/planner_test.go b/bql/planner/planner_test.go index bd833157..2135ccfd 100644 --- a/bql/planner/planner_test.go +++ b/bql/planner/planner_test.go @@ -983,6 +983,130 @@ func TestPlannerQuery(t *testing.T) { nBindings: 2, nRows: 5, }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /u ?p ?o . + FILTER latest(?p) + };`, + nBindings: 2, + nRows: 1, + }, + { + q: `SELECT ?p_alias, ?o + FROM ?test + WHERE { + /u ?p AS ?p_alias ?o . + FILTER latest(?p_alias) + };`, + nBindings: 2, + nRows: 1, + }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /l ?p ?o . + FILTER latest(?o) + };`, + nBindings: 2, + nRows: 1, + }, + { + q: `SELECT ?p, ?o_alias + FROM ?test + WHERE { + /l ?p ?o AS ?o_alias . + FILTER latest(?o_alias) + };`, + nBindings: 2, + nRows: 1, + }, + { + q: `SELECT ?p1, ?p2 + FROM ?test + WHERE { + /u ?p1 ?o1 . + /item/book<000> ?p2 ?o2 . + FILTER latest(?p1) . + FILTER latest(?p2) + };`, + nBindings: 2, + nRows: 1, + }, + { + q: `SELECT ?s, ?p, ?o + FROM ?test + WHERE { + ?s ?p ?o . + FILTER latest(?p) + };`, + nBindings: 3, + nRows: 3, + }, + { + q: `SELECT ?s, ?p, ?o + FROM ?test + WHERE { + ?s ?p ?o . + FILTER latest(?o) + };`, + nBindings: 3, + nRows: 2, + }, + { + q: `SELECT ?s, ?p_alias, ?o + FROM ?test + WHERE { + ?s "bought"@[2016-03-01T00:00:00-08:00] AS ?p_alias ?o . + FILTER latest(?p_alias) + };`, + nBindings: 3, + nRows: 1, + }, + { + q: `SELECT ?s, ?p, ?o_alias + FROM ?test + WHERE { + ?s ?p "turned"@[2016-03-01T00:00:00-08:00] AS ?o_alias . + FILTER latest(?o_alias) + };`, + nBindings: 3, + nRows: 1, + }, + { + q: `SELECT ?s, ?p, ?o + FROM ?test + WHERE { + ?s "bought"@[?time] ?o . + OPTIONAL { ?s ?p ?o } . + FILTER latest(?p) + };`, + nBindings: 3, + nRows: 6, + }, + { + q: `SELECT ?p + FROM ?test + WHERE { + /u ?p ?o1 . + /u ?p ?o2 . + FILTER latest(?p) + };`, + nBindings: 1, + nRows: 1, + }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /u ?p ?o . + FILTER lAtEsT(?p) . + };`, + nBindings: 2, + nRows: 1, + }, } s, ctx := memory.NewStore(), context.Background() @@ -1068,6 +1192,40 @@ func TestPlannerQueryError(t *testing.T) { } HAVING ?s > /u;`, }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /u ?p ?o . + FILTER latest(?p) . + FILTER latest(?p) + };`, + }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /l ?p ?o . + FILTER latest(?p) . + FILTER latest(?o) + };`, + }, + { + q: `SELECT ?p, ?o + FROM ?test + WHERE { + /u ?p ?o . + FILTER latest(?b_not_exist) + };`, + }, + { + q: `SELECT ?s, ?p, ?o + FROM ?test + WHERE { + ?s ID ?sID ?p ?o . + FILTER latest(?sID) + };`, + }, } s, ctx := memory.NewStore(), context.Background() From eb5a1ec412fd5e6634a882515357addd919dd813 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Tue, 8 Sep 2020 18:32:32 -0300 Subject: [PATCH 03/26] Add "filters" to the "queryPlan" type --- bql/planner/planner.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 33ae06fd..b919fcdf 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -267,6 +267,7 @@ type queryPlan struct { grfsNames []string grfs []storage.Graph cls []*semantic.GraphClause + filters []*semantic.FilterClause tbl *table.Table chanSize int tracer io.Writer @@ -293,6 +294,7 @@ func newQueryPlan(ctx context.Context, store storage.Store, stm *semantic.Statem bndgs: bs, grfsNames: stm.InputGraphNames(), cls: stm.GraphPatternClauses(), + filters: stm.FilterClauses(), tbl: t, chanSize: chanSize, tracer: w, From 1720236f64f46d0bfcebed9daa51c638cc415540 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Wed, 9 Sep 2020 18:57:32 -0300 Subject: [PATCH 04/26] Update String method of queryPlan to show filters as well --- bql/planner/planner.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index b919fcdf..6b16d685 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -883,6 +883,12 @@ func (p *queryPlan) String(ctx context.Context) string { b.WriteString(c.String()) b.WriteString("\n") } + b.WriteString("with filters\n") + for _, f := range p.filters { + b.WriteString("\t") + b.WriteString(f.String()) + b.WriteString("\n") + } b.WriteString("project results using\n") for _, p := range p.stm.Projection() { b.WriteString("\t") From 929af59a2069389158141ff2d79cdf632ef8f2d3 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Wed, 9 Sep 2020 20:35:43 -0300 Subject: [PATCH 05/26] Add FilterOptions to LookupOptions in "storage.go" --- storage/storage.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/storage/storage.go b/storage/storage.go index e787fbd0..40ec8a13 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -43,8 +43,17 @@ type LookupOptions struct { // LatestAnchor only. If set, it will ignore the time boundaries provided and // just use the last available anchor. LatestAnchor bool + + // FilterOptions, if provided, represent the specifications for the filtering to be executed. + FilterOptions *FilteringOptions } +// FilteringOptions represent the storage level specifications for the filtering to be executed. +// Operation below refers to the filter function being applied (eg: "latest"), Field refers to the position of the graph clause it +// will be applied to ("subject", "predicate" or "object") and Value, when specified, contains the second argument of the filter +// function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). +type FilteringOptions struct{ Operation, Field, Value string } + // String returns a readable version of the LookupOptions instance. func (l *LookupOptions) String() string { b := bytes.NewBufferString(" Date: Thu, 10 Sep 2020 12:50:38 -0300 Subject: [PATCH 06/26] Add support for FILTER clauses in the planner --- bql/planner/data_access.go | 7 ++-- bql/planner/planner.go | 69 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/bql/planner/data_access.go b/bql/planner/data_access.go index 59e5b5ef..9aacd627 100644 --- a/bql/planner/data_access.go +++ b/bql/planner/data_access.go @@ -45,9 +45,10 @@ var _ error = (*skippableError)(nil) // provided graph clause. func updateTimeBounds(lo *storage.LookupOptions, cls *semantic.GraphClause) *storage.LookupOptions { nlo := &storage.LookupOptions{ - MaxElements: lo.MaxElements, - LowerAnchor: lo.LowerAnchor, - UpperAnchor: lo.UpperAnchor, + MaxElements: lo.MaxElements, + LowerAnchor: lo.LowerAnchor, + UpperAnchor: lo.UpperAnchor, + FilterOptions: lo.FilterOptions, } if cls.PLowerBound != nil { if lo.LowerAnchor == nil || (lo.LowerAnchor != nil && cls.PLowerBound.After(*lo.LowerAnchor)) { diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 6b16d685..cf61e98d 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -264,6 +264,7 @@ type queryPlan struct { store storage.Store // Prepared plan information. bndgs []string + bndgsMap map[string]int grfsNames []string grfs []storage.Graph cls []*semantic.GraphClause @@ -292,6 +293,7 @@ func newQueryPlan(ctx context.Context, store storage.Store, stm *semantic.Statem stm: stm, store: store, bndgs: bs, + bndgsMap: stm.BindingsMap(), grfsNames: stm.InputGraphNames(), cls: stm.GraphPatternClauses(), filters: stm.FilterClauses(), @@ -641,6 +643,59 @@ func (p *queryPlan) filterOnExistence(ctx context.Context, cls *semantic.GraphCl return grp.Wait() } +// organizeFiltersByBinding takes the filters received as input and organize them in a map +// on which the keys are the bindings of each filter. +func organizeFiltersByBinding(filters []*semantic.FilterClause, bndgsMap map[string]int) (map[string]*semantic.FilterClause, error) { + filtersByBinding := map[string]*semantic.FilterClause{} + for _, f := range filters { + if _, ok := bndgsMap[f.Binding]; !ok { + return nil, fmt.Errorf("binding %q referenced by a filter clause does not exist", f.Binding) + } + if _, ok := filtersByBinding[f.Binding]; ok { + return nil, fmt.Errorf("multiple filters for the same binding are not supported at the moment") + } + filtersByBinding[f.Binding] = f + } + + return filtersByBinding, nil +} + +// addFilterOptions adds FilterOptions to lookup options if the given clause has bindings for which +// filters were defined. +func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filtersByBinding map[string]*semantic.FilterClause) error { + bindingsByField := map[string][]string{ + "predicate": {cls.PBinding, cls.PAlias}, + "object": {cls.OBinding, cls.OAlias}, + } + + for field, bs := range bindingsByField { + for _, b := range bs { + if b == "" { + continue + } + if _, ok := filtersByBinding[b]; !ok { + continue + } + if lo.FilterOptions != nil { + return fmt.Errorf("multiple filters for the same graph clause are not supported at the moment") + } + filter := filtersByBinding[b] + lo.FilterOptions = &storage.FilteringOptions{ + Operation: filter.Operation, + Field: field, + Value: filter.Value, + } + } + } + + return nil +} + +// resetFilterOptions resets FilterOptions in lookup options to nil. +func resetFilterOptions(lo *storage.LookupOptions) { + lo.FilterOptions = (*storage.FilteringOptions)(nil) +} + // processGraphPattern process the query graph pattern to retrieve the // data from the specified graphs. func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupOptions) error { @@ -653,6 +708,11 @@ func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupO Msgs: res, } }) + + filtersByBinding, err := organizeFiltersByBinding(p.filters, p.bndgsMap) + if err != nil { + return err + } for i, c := range p.cls { i, cls := i, *c tracer.Trace(p.tracer, func() *tracer.Arguments { @@ -660,9 +720,13 @@ func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupO Msgs: []string{fmt.Sprintf("Processing clause %d: %v", i, &cls)}, } }) - // The current planner is based on naively executing clauses by - // specificity. + + err = addFilterOptions(lo, &cls, filtersByBinding) + if err != nil { + return err + } unresolvable, err := p.processClause(ctx, &cls, lo) + resetFilterOptions(lo) if err != nil { return err } @@ -671,6 +735,7 @@ func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupO return nil } } + return nil } From cbfffaecc9ccfac401947c757923ee31aad20446 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 10 Sep 2020 13:02:53 -0300 Subject: [PATCH 07/26] Make the volatile driver consume lo.FilterOptions (focusing on latest anchor only for now) --- storage/memory/memory.go | 407 +++++++++++++++++++++++---------------- 1 file changed, 242 insertions(+), 165 deletions(-) diff --git a/storage/memory/memory.go b/storage/memory/memory.go index bcf3cb40..0558c28f 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -274,6 +274,51 @@ func (c *checker) CheckAndUpdate(p *predicate.Predicate) bool { return true } +// latestFilter executes the latest filter operation over memoryTriples following filterOptions. +func latestFilter(memoryTriples map[string]*triple.Triple, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { + if filterOptions.Field != "predicate" && filterOptions.Field != "object" { + return nil, fmt.Errorf(`invalid field %q for "latest" filter operation, can accept only "predicate" or "object"`, filterOptions.Field) + } + + lastTA := make(map[string]*time.Time) + trps := make(map[string]*triple.Triple) + for _, t := range memoryTriples { + var p *predicate.Predicate + if filterOptions.Field == "predicate" { + p = t.Predicate() + } else if pObj, err := t.Object().Predicate(); filterOptions.Field == "object" && err == nil { + p = pObj + } else { + continue + } + if p.Type() != predicate.Temporal { + continue + } + + ppUUID := p.PartialUUID().String() + ta, err := p.TimeAnchor() + if err != nil { + return nil, err + } + if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { + trps[ppUUID] = t + lastTA[ppUUID] = ta + } + } + + return trps, nil +} + +// executeFilter executes the proper filter operation over memoryTriples following the specifications given in filterOptions. +func executeFilter(memoryTriples map[string]*triple.Triple, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { + switch filterOptions.Operation { + case "latest": + return latestFilter(memoryTriples, filterOptions) + default: + return nil, fmt.Errorf("filter operation %q not supported in the driver", filterOptions.Operation) + } +} + // Objects published the objects for the give object and predicate to the // provided channel. func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predicate, lo *storage.LookupOptions, objs chan<- *triple.Object) error { @@ -289,21 +334,22 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica defer close(objs) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxSP[spIdx] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxSP[spIdx], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -318,6 +364,7 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica objs <- t.Object() } } + return nil } @@ -327,6 +374,7 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple if subjs == nil { return fmt.Errorf("cannot provide an empty channel") } + pUUID := UUIDToByteString(p.PartialUUID()) oUUID := UUIDToByteString(o.UUID()) poIdx := pUUID + oUUID @@ -335,21 +383,22 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple defer close(subjs) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxPO[poIdx] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxPO[poIdx], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -364,6 +413,7 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple subjs <- t.Subject() } } + return nil } @@ -373,6 +423,7 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node if prds == nil { return fmt.Errorf("cannot provide an empty channel") } + sUUID := UUIDToByteString(s.UUID()) oUUID := UUIDToByteString(o.UUID()) soIdx := sUUID + oUUID @@ -381,21 +432,22 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node defer close(prds) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxSO[soIdx] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxSO[soIdx], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -410,6 +462,7 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node prds <- t.Predicate() } } + return nil } @@ -419,27 +472,29 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto if prds == nil { return fmt.Errorf("cannot provide an empty channel") } + sUUID := UUIDToByteString(s.UUID()) m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(prds) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxS[sUUID] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxS[sUUID], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -454,6 +509,7 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto prds <- t.Predicate() } } + return nil } @@ -463,27 +519,29 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * if prds == nil { return fmt.Errorf("cannot provide an empty channel") } + oUUID := UUIDToByteString(o.UUID()) m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(prds) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxO[oUUID] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxO[oUUID], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -498,6 +556,7 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * prds <- t.Predicate() } } + return nil } @@ -507,27 +566,29 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + sUUID := UUIDToByteString(s.UUID()) m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxS[sUUID] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxS[sUUID], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -542,6 +603,7 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag trpls <- t } } + return nil } @@ -551,27 +613,29 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + pUUID := UUIDToByteString(p.PartialUUID()) m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxP[pUUID] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxP[pUUID], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -586,6 +650,7 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate trpls <- t } } + return nil } @@ -595,27 +660,29 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + oUUID := UUIDToByteString(o.UUID()) m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxO[oUUID] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxO[oUUID], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -630,6 +697,7 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto trpls <- t } } + return nil } @@ -639,6 +707,7 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + sUUID := UUIDToByteString(s.UUID()) pUUID := UUIDToByteString(p.PartialUUID()) spIdx := sUUID + pUUID @@ -647,21 +716,22 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxSP[spIdx] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxSP[spIdx], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -676,6 +746,7 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node trpls <- t } } + return nil } @@ -685,6 +756,7 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + pUUID := UUIDToByteString(p.PartialUUID()) oUUID := UUIDToByteString(o.UUID()) poIdx := pUUID + oUUID @@ -693,21 +765,22 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idxPO[poIdx] { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idxPO[poIdx], lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -722,6 +795,7 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. trpls <- t } } + return nil } @@ -740,26 +814,28 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c if trpls == nil { return fmt.Errorf("cannot provide an empty channel") } + m.rwmu.RLock() defer m.rwmu.RUnlock() defer close(trpls) if lo.LatestAnchor { - lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) - for _, t := range m.idx { - p := t.Predicate() - ppUUID := p.PartialUUID().String() - if p.Type() == predicate.Temporal { - ta, err := p.TimeAnchor() - if err != nil { - return err - } - if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t - lastTA[ppUUID] = ta - } - } + if lo.FilterOptions != nil { + return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") + } + lo.FilterOptions = &storage.FilteringOptions{ + Operation: "latest", + Field: "predicate", + } + // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". + defer func() { + lo.FilterOptions = (*storage.FilteringOptions)(nil) + }() + } + if lo.FilterOptions != nil { + trps, err := executeFilter(m.idx, lo.FilterOptions) + if err != nil { + return err } for _, trp := range trps { if trp != nil { @@ -774,5 +850,6 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c trpls <- t } } + return nil } From 1da2253d16c1c874e443ad5b5ad8a0e057237d7a Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Fri, 11 Sep 2020 17:31:24 -0300 Subject: [PATCH 08/26] Update String method of LookupOptions to show FilterOptions as well --- storage/memoization/memoization_test.go | 2 +- storage/storage.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/memoization/memoization_test.go b/storage/memoization/memoization_test.go index 05d9ffa3..6c2718ef 100644 --- a/storage/memoization/memoization_test.go +++ b/storage/memoization/memoization_test.go @@ -31,7 +31,7 @@ import ( ) func TestCombinedUUID(t *testing.T) { - want := "op:9dae52f4-9b35-5d5f-bd8e-195d4b16fc30:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" + want := "op:420c34d3-9f56-5191-89de-57468c3e6a93:00000000-0000-0000-0000-000000000000:00000000-0000-0000-0000-000000000000" if got := combinedUUID("op", storage.DefaultLookup, uuid.NIL, uuid.NIL); got != want { t.Errorf("combinedUUID returned the wrong value; got %v, want %v", got, want) } diff --git a/storage/storage.go b/storage/storage.go index 40ec8a13..db5e7656 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -71,6 +71,8 @@ func (l *LookupOptions) String() string { b.WriteString("nil") } b.WriteString(fmt.Sprintf(", LatestAnchor=%v", l.LatestAnchor)) + b.WriteString(", FilterOptions=") + b.WriteString(fmt.Sprintf("%+v", l.FilterOptions)) b.WriteString(">") return b.String() } From e2a2a612263cb6cf5c9e88ab4674799f71914cf9 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Tue, 15 Sep 2020 19:18:48 -0300 Subject: [PATCH 09/26] Minor renaming inside "memory_test.go" to avoid confusion --- storage/memory/memory_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index 8decd9ff..b0c431e0 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -275,7 +275,7 @@ func TestObjects(t *testing.T) { } } -func TestObjectsLastestTemporal(t *testing.T) { +func TestObjectsLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -329,7 +329,7 @@ func TestSubjects(t *testing.T) { } } -func TestSubjectsLatestTemporal(t *testing.T) { +func TestSubjectsLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -380,7 +380,7 @@ func TestPredicatesForSubjectAndObject(t *testing.T) { t.Errorf("g.PredicatesForSubjectAndObject(%s, %s) failed to retrieve 1 predicate, got %d instead", ts[0].Subject(), ts[0].Object(), cnt) } } -func TestPredicatesForSubjectAndObjectLatestTemporal(t *testing.T) { +func TestPredicatesForSubjectAndObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -431,7 +431,7 @@ func TestPredicatesForSubject(t *testing.T) { } } -func TestPredicatesForSubjectLatestTemporal(t *testing.T) { +func TestPredicatesForSubjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -481,7 +481,7 @@ func TestPredicatesForObject(t *testing.T) { t.Errorf("g.PredicatesForObject(%s) failed to retrieve 1 predicate, got %d instead", ts[0].Object(), cnt) } } -func TestPredicatesForObjectLatestTemporal(t *testing.T) { +func TestPredicatesForObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -529,7 +529,7 @@ func TestTriplesForSubject(t *testing.T) { } } -func TestTriplesForSubjectLatestTemporal(t *testing.T) { +func TestTriplesForSubjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -576,7 +576,7 @@ func TestTriplesForPredicate(t *testing.T) { t.Errorf("g.triplesForPredicate(%s) failed to retrieve 3 predicates, got %d instead", ts[0].Predicate(), cnt) } } -func TestTriplesForPredicateLatestTemporal(t *testing.T) { +func TestTriplesForPredicateLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -624,7 +624,7 @@ func TestTriplesForObject(t *testing.T) { } } -func TestTriplesForObjectLatestTemporal(t *testing.T) { +func TestTriplesForObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -714,7 +714,7 @@ func TestTriplesForSubjectAndPredicate(t *testing.T) { t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s) failed to retrieve 3 predicates, got %d instead", ts[0].Subject(), ts[0].Predicate(), cnt) } } -func TestTriplesForSubjectAndPredicateLatestTemporal(t *testing.T) { +func TestTriplesForSubjectAndPredicateLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -762,7 +762,7 @@ func TestTriplesForPredicateAndObject(t *testing.T) { } } -func TestTriplesForPredicateAndObjectLatestTemporal(t *testing.T) { +func TestTriplesForPredicateAndObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { @@ -827,7 +827,7 @@ func TestTriples(t *testing.T) { } } -func TestTriplesLastestTemporal(t *testing.T) { +func TestTriplesLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") if err := g.AddTriples(ctx, ts); err != nil { From e0003e2d746edcfd05c284e9e033d7887ceec91f Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 17 Sep 2020 23:33:29 -0300 Subject: [PATCH 10/26] Fix problem with partial predicate UUIDs in the volatile driver and make FILTER for latest return multiple triples if they share the same predicate and same latest anchor --- storage/memory/memory.go | 46 ++++++++++++++++++++++------------- storage/memory/memory_test.go | 18 +++++++------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/storage/memory/memory.go b/storage/memory/memory.go index 0558c28f..2ede6f70 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -275,14 +275,18 @@ func (c *checker) CheckAndUpdate(p *predicate.Predicate) bool { } // latestFilter executes the latest filter operation over memoryTriples following filterOptions. -func latestFilter(memoryTriples map[string]*triple.Triple, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { +func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { if filterOptions.Field != "predicate" && filterOptions.Field != "object" { return nil, fmt.Errorf(`invalid field %q for "latest" filter operation, can accept only "predicate" or "object"`, filterOptions.Field) } lastTA := make(map[string]*time.Time) - trps := make(map[string]*triple.Triple) + trps := make(map[string]map[string]*triple.Triple) for _, t := range memoryTriples { + if pQuery != nil && pQuery.String() != t.Predicate().String() { + continue + } + var p *predicate.Predicate if filterOptions.Field == "predicate" { p = t.Predicate() @@ -301,19 +305,27 @@ func latestFilter(memoryTriples map[string]*triple.Triple, filterOptions *storag return nil, err } if lta := lastTA[ppUUID]; lta == nil || ta.Sub(*lta) > 0 { - trps[ppUUID] = t + trps[ppUUID] = map[string]*triple.Triple{t.UUID().String(): t} lastTA[ppUUID] = ta + } else if ta.Sub(*lta) == 0 { + trps[ppUUID][t.UUID().String()] = t } } - return trps, nil + trpsByUUID := make(map[string]*triple.Triple) + for _, m := range trps { + for tUUID, t := range m { + trpsByUUID[tUUID] = t + } + } + return trpsByUUID, nil } // executeFilter executes the proper filter operation over memoryTriples following the specifications given in filterOptions. -func executeFilter(memoryTriples map[string]*triple.Triple, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { +func executeFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { switch filterOptions.Operation { case "latest": - return latestFilter(memoryTriples, filterOptions) + return latestFilter(memoryTriples, pQuery, filterOptions) default: return nil, fmt.Errorf("filter operation %q not supported in the driver", filterOptions.Operation) } @@ -347,7 +359,7 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxSP[spIdx], lo.FilterOptions) + trps, err := executeFilter(m.idxSP[spIdx], p, lo.FilterOptions) if err != nil { return err } @@ -396,7 +408,7 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxPO[poIdx], lo.FilterOptions) + trps, err := executeFilter(m.idxPO[poIdx], p, lo.FilterOptions) if err != nil { return err } @@ -445,7 +457,7 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxSO[soIdx], lo.FilterOptions) + trps, err := executeFilter(m.idxSO[soIdx], nil, lo.FilterOptions) if err != nil { return err } @@ -492,7 +504,7 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxS[sUUID], lo.FilterOptions) + trps, err := executeFilter(m.idxS[sUUID], nil, lo.FilterOptions) if err != nil { return err } @@ -539,7 +551,7 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxO[oUUID], lo.FilterOptions) + trps, err := executeFilter(m.idxO[oUUID], nil, lo.FilterOptions) if err != nil { return err } @@ -586,7 +598,7 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxS[sUUID], lo.FilterOptions) + trps, err := executeFilter(m.idxS[sUUID], nil, lo.FilterOptions) if err != nil { return err } @@ -633,7 +645,7 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxP[pUUID], lo.FilterOptions) + trps, err := executeFilter(m.idxP[pUUID], p, lo.FilterOptions) if err != nil { return err } @@ -680,7 +692,7 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxO[oUUID], lo.FilterOptions) + trps, err := executeFilter(m.idxO[oUUID], nil, lo.FilterOptions) if err != nil { return err } @@ -729,7 +741,7 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxSP[spIdx], lo.FilterOptions) + trps, err := executeFilter(m.idxSP[spIdx], p, lo.FilterOptions) if err != nil { return err } @@ -778,7 +790,7 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idxPO[poIdx], lo.FilterOptions) + trps, err := executeFilter(m.idxPO[poIdx], p, lo.FilterOptions) if err != nil { return err } @@ -833,7 +845,7 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c }() } if lo.FilterOptions != nil { - trps, err := executeFilter(m.idx, lo.FilterOptions) + trps, err := executeFilter(m.idx, nil, lo.FilterOptions) if err != nil { return err } diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index b0c431e0..059533c7 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -593,12 +593,12 @@ func TestTriplesForPredicateLatestAnchor(t *testing.T) { cnt := 0 for rts := range trpls { cnt++ - if !reflect.DeepEqual(rts.Predicate().UUID(), ts[len(ts)-1].Predicate().UUID()) { - t.Errorf("g.PredicatesForObject(%s) failed to return a valid predicate; returned %s instead", ts[0].Object(), rts.Predicate()) + if !reflect.DeepEqual(rts.Predicate().UUID(), ts[0].Predicate().UUID()) { + t.Errorf("g.TriplesForPredicate(%s) = %s for LatestAnchor; want %s", ts[0].Predicate(), rts.Predicate(), ts[0].Predicate()) } } if cnt != 1 { - t.Errorf("g.triplesForPredicate(%s) failed to retrieve 3 predicates, got %d instead", ts[0].Predicate(), cnt) + t.Errorf("g.triplesForPredicate(%s) retrieved %d predicates; want 1", ts[0].Predicate(), cnt) } } @@ -731,12 +731,12 @@ func TestTriplesForSubjectAndPredicateLatestAnchor(t *testing.T) { cnt := 0 for rts := range trpls { cnt++ - if !reflect.DeepEqual(rts.Predicate().UUID(), ts[len(ts)-1].Predicate().UUID()) { - t.Errorf("g.PredicatesForObject(%s) failed to return a valid predicate; returned %s instead", ts[0].Object(), rts.Predicate()) + if !reflect.DeepEqual(rts.Predicate().UUID(), ts[0].Predicate().UUID()) { + t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s) = %s for LatestAnchor; want %s", ts[0].Subject(), ts[0].Predicate(), rts.Predicate(), ts[0].Predicate()) } } if cnt != 1 { - t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s) failed to retrieve 3 predicates, got %d instead", ts[0].Subject(), ts[0].Predicate(), cnt) + t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s) retrieved %d predicates; want 1", ts[0].Subject(), ts[0].Predicate(), cnt) } } @@ -779,12 +779,12 @@ func TestTriplesForPredicateAndObjectLatestAnchor(t *testing.T) { cnt := 0 for rts := range trpls { cnt++ - if !reflect.DeepEqual(rts.Predicate().UUID(), ts[len(ts)-1].Predicate().UUID()) { - t.Errorf("g.PredicatesForObject(%s) failed to return a valid predicate; returned %s instead", ts[0].Object(), rts.Predicate()) + if !reflect.DeepEqual(rts.Predicate().UUID(), ts[0].Predicate().UUID()) { + t.Errorf("g.TriplesForPredicateAndObject(%s, %s) = %s for LatestAnchor; want %s", ts[0].Predicate(), ts[0].Object(), rts.Predicate(), ts[0].Predicate()) } } if cnt != 1 { - t.Errorf("g.TriplesForPredicateAndObject(%s, %s) failed to retrieve 1 predicates, got %d instead", ts[0].Predicate(), ts[0].Object(), cnt) + t.Errorf("g.TriplesForPredicateAndObject(%s, %s) retrieved %d predicates; want 1", ts[0].Predicate(), ts[0].Object(), cnt) } } From 43106464518e2e413babc1831931ea238dd66111 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Tue, 15 Sep 2020 21:23:47 -0300 Subject: [PATCH 11/26] Add tests for the FILTER execution in the volatile driver (in "memory_test.go") --- storage/memory/memory_test.go | 617 ++++++++++++++++++++++++++++++++++ 1 file changed, 617 insertions(+) diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index 059533c7..ad783c6f 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -237,6 +237,19 @@ func getTestTemporalTriples(t *testing.T) []*triple.Triple { }) } +func getTestTriplesFilter(t *testing.T) []*triple.Triple { + return createTriples(t, []string{ + "/u\t\"meet\"@[2012-04-10T04:21:00Z]\t/u", + "/u\t\"meet\"@[2013-04-10T04:21:00Z]\t/u", + "/u\t\"meet\"@[2014-04-10T04:21:00Z]\t/u", + "/u\t\"meet\"@[2014-04-10T04:21:00Z]\t/u", + "/u\t\"parent_of\"@[]\t/u", + "/_\t\"_predicate\"@[]\t\"meet\"@[2020-04-10T04:21:00Z]", + "/_\t\"_predicate\"@[]\t\"meet\"@[2021-04-10T04:21:00Z]", + "/_\t\"_predicate\"@[]\t\"height_cm\"@[]", + }) +} + func TestAddRemoveTriples(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -303,6 +316,65 @@ func TestObjectsLatestAnchor(t *testing.T) { } } +func TestObjectsFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + s *node.Node + p *predicate.Predicate + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), + want: map[string]int{"/u": 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), + want: map[string]int{"/u": 1, "/u": 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), + p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), + want: map[string]int{`"meet"@[2021-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + os := make(chan *triple.Object, 100) + s := entry.s + p := entry.p + if err := g.Objects(ctx, s, p, entry.lo, os); err != nil { + t.Fatalf("g.Objects(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) + } + for o := range os { + oStr := o.String() + if _, ok := entry.want[oStr]; !ok { + t.Fatalf("g.Objects(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, oStr) + } + entry.want[oStr] = entry.want[oStr] - 1 + if entry.want[oStr] == 0 { + delete(entry.want, oStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.Objects(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) + } + } +} + func TestSubjects(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -356,6 +428,65 @@ func TestSubjectsLatestAnchor(t *testing.T) { } } +func TestSubjectsFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + p *predicate.Predicate + o *triple.Object + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{"/u": 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{"/u": 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), + o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), + want: map[string]int{"/_": 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + ss := make(chan *node.Node, 100) + p := entry.p + o := entry.o + if err := g.Subjects(ctx, p, o, entry.lo, ss); err != nil { + t.Fatalf("g.Subjects(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) + } + for s := range ss { + sStr := s.String() + if _, ok := entry.want[sStr]; !ok { + t.Fatalf("g.Subjects(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, sStr) + } + entry.want[sStr] = entry.want[sStr] - 1 + if entry.want[sStr] == 0 { + delete(entry.want, sStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.Subjects(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) + } + } +} + func TestPredicatesForSubjectAndObject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -380,6 +511,7 @@ func TestPredicatesForSubjectAndObject(t *testing.T) { t.Errorf("g.PredicatesForSubjectAndObject(%s, %s) failed to retrieve 1 predicate, got %d instead", ts[0].Subject(), ts[0].Object(), cnt) } } + func TestPredicatesForSubjectAndObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -406,6 +538,65 @@ func TestPredicatesForSubjectAndObjectLatestAnchor(t *testing.T) { } } +func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + s *node.Node + o *triple.Object + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), + want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), + o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), + want: map[string]int{`"_predicate"@[]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + s := entry.s + o := entry.o + if err := g.PredicatesForSubjectAndObject(ctx, s, o, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) = %v; want nil", s, o, entry.lo, err) + } + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) retrieved unexpected %s", s, o, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForSubjectAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", s, o, entry.lo, entry.want) + } + } +} + func TestPredicatesForSubject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -457,6 +648,55 @@ func TestPredicatesForSubjectLatestAnchor(t *testing.T) { } } +func TestPredicatesForSubjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + s *node.Node + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 2}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), + want: map[string]int{`"_predicate"@[]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + s := entry.s + if err := g.PredicatesForSubject(ctx, s, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) + } + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) + } + } +} + func TestPredicatesForObject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -481,6 +721,7 @@ func TestPredicatesForObject(t *testing.T) { t.Errorf("g.PredicatesForObject(%s) failed to retrieve 1 predicate, got %d instead", ts[0].Object(), cnt) } } + func TestPredicatesForObjectLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -507,6 +748,60 @@ func TestPredicatesForObjectLatestAnchor(t *testing.T) { } } +func TestPredicatesForObjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + o *triple.Object + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), + want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), + want: map[string]int{`"_predicate"@[]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + o := entry.o + if err := g.PredicatesForObject(ctx, o, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForObject(%s, %s) = %v; want nil", o, entry.lo, err) + } + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) + } + } +} + func TestTriplesForSubject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -555,6 +850,55 @@ func TestTriplesForSubjectLatestAnchor(t *testing.T) { } } +func TestTriplesForSubjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + s *node.Node + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), + want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + s := entry.s + if err := g.TriplesForSubject(ctx, s, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) + } + } +} + func TestTriplesForPredicate(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -576,6 +920,7 @@ func TestTriplesForPredicate(t *testing.T) { t.Errorf("g.triplesForPredicate(%s) failed to retrieve 3 predicates, got %d instead", ts[0].Predicate(), cnt) } } + func TestTriplesForPredicateLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -602,6 +947,60 @@ func TestTriplesForPredicateLatestAnchor(t *testing.T) { } } +func TestTriplesForPredicateFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + p *predicate.Predicate + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), + want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), + want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + p := entry.p + if err := g.TriplesForPredicate(ctx, p, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForPredicate(%s, %s) = %v; want nil", p, entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForPredicate(%s, %s) retrieved unexpected %s", p, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForPredicate(%s, %s) failed to retrieve some expected elements: %v", p, entry.lo, entry.want) + } + } +} + func TestTriplesForObject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -650,6 +1049,60 @@ func TestTriplesForObjectLatestAnchor(t *testing.T) { } } +func TestTriplesForObjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + o *triple.Object + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), + want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + o := entry.o + if err := g.TriplesForObject(ctx, o, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForObject(%s, %s) = %v; want nil", o, entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) + } + } +} + func TestTriplesForObjectWithLimit(t *testing.T) { ts := createTriples(t, []string{ "/u\t\"kissed\"@[2015-01-01T00:00:00-09:00]\t/u", @@ -714,6 +1167,7 @@ func TestTriplesForSubjectAndPredicate(t *testing.T) { t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s) failed to retrieve 3 predicates, got %d instead", ts[0].Subject(), ts[0].Predicate(), cnt) } } + func TestTriplesForSubjectAndPredicateLatestAnchor(t *testing.T) { ts, ctx := getTestTemporalTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -740,6 +1194,65 @@ func TestTriplesForSubjectAndPredicateLatestAnchor(t *testing.T) { } } +func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + s *node.Node + p *predicate.Predicate + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), + want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), + p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), + p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), + want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + s := entry.s + p := entry.p + if err := g.TriplesForSubjectAndPredicate(ctx, s, p, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) + } + } +} + func TestTriplesForPredicateAndObject(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -788,6 +1301,65 @@ func TestTriplesForPredicateAndObjectLatestAnchor(t *testing.T) { } } +func TestTriplesForPredicateAndObjectFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + p *predicate.Predicate + o *triple.Object + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), + o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), + o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), + want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + p := entry.p + o := entry.o + if err := g.TriplesForPredicateAndObject(ctx, p, o, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForPredicateAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) + } + } +} + func TestExists(t *testing.T) { ts, ctx := getTestTriples(t), context.Background() g, _ := NewStore().NewGraph(ctx, "test") @@ -852,3 +1424,48 @@ func TestTriplesLatestAnchor(t *testing.T) { t.Errorf("g.TriplesForPredicateAndObject(%s, %s) failed to retrieve 1 predicates, got %d instead", ts[0].Predicate(), ts[0].Object(), cnt) } } + +func TestTriplesFilter(t *testing.T) { + ts, ctx := getTestTriplesFilter(t), context.Background() + g, _ := NewStore().NewGraph(ctx, "test") + if err := g.AddTriples(ctx, ts); err != nil { + t.Fatalf("g.AddTriples(_) failed to add test triples with error: %v", err) + } + + testTable := []struct { + lo *storage.LookupOptions + want map[string]int + }{ + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, + }, + { + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, + }, + } + + for _, entry := range testTable { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + if err := g.Triples(ctx, entry.lo, trpls); err != nil { + t.Fatalf("g.Triples(%s) = %v; want nil", entry.lo, err) + } + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.Triples(%s) retrieved unexpected %s", entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } + } + if len(entry.want) != 0 { + t.Errorf("g.Triples(%s) failed to retrieve some expected elements: %v", entry.lo, entry.want) + } + } +} From b572365b3f2c2f1b72096fc1cf8e37451760ebcf Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 24 Sep 2020 14:59:55 -0300 Subject: [PATCH 12/26] Allow user to specify for which bindings in the clause each filter can be applied to, improving error checking, and also move verification for supported filter functions to the planner level --- bql/planner/planner.go | 122 +++++++++++++++++++++++++---------------- bql/semantic/hooks.go | 6 -- 2 files changed, 75 insertions(+), 53 deletions(-) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index cf61e98d..d62f0291 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -264,10 +264,9 @@ type queryPlan struct { store storage.Store // Prepared plan information. bndgs []string - bndgsMap map[string]int grfsNames []string grfs []storage.Graph - cls []*semantic.GraphClause + clauses []*semantic.GraphClause filters []*semantic.FilterClause tbl *table.Table chanSize int @@ -293,9 +292,8 @@ func newQueryPlan(ctx context.Context, store storage.Store, stm *semantic.Statem stm: stm, store: store, bndgs: bs, - bndgsMap: stm.BindingsMap(), grfsNames: stm.InputGraphNames(), - cls: stm.GraphPatternClauses(), + clauses: stm.GraphPatternClauses(), filters: stm.FilterClauses(), tbl: t, chanSize: chanSize, @@ -643,52 +641,86 @@ func (p *queryPlan) filterOnExistence(ctx context.Context, cls *semantic.GraphCl return grp.Wait() } -// organizeFiltersByBinding takes the filters received as input and organize them in a map -// on which the keys are the bindings of each filter. -func organizeFiltersByBinding(filters []*semantic.FilterClause, bndgsMap map[string]int) (map[string]*semantic.FilterClause, error) { - filtersByBinding := map[string]*semantic.FilterClause{} - for _, f := range filters { - if _, ok := bndgsMap[f.Binding]; !ok { - return nil, fmt.Errorf("binding %q referenced by a filter clause does not exist", f.Binding) - } - if _, ok := filtersByBinding[f.Binding]; ok { - return nil, fmt.Errorf("multiple filters for the same binding are not supported at the moment") +// organizeClausesByBinding takes the graph clauses received as input and organize them in a map +// on which the keys are the bindings of these clauses. +func organizeClausesByBinding(clauses []*semantic.GraphClause) map[string][]*semantic.GraphClause { + clausesByBinding := map[string][]*semantic.GraphClause{} + for _, cls := range clauses { + for b := range cls.BindingsMap() { + clausesByBinding[b] = append(clausesByBinding[b], cls) } - filtersByBinding[f.Binding] = f } - return filtersByBinding, nil + return clausesByBinding } -// addFilterOptions adds FilterOptions to lookup options if the given clause has bindings for which -// filters were defined. -func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filtersByBinding map[string]*semantic.FilterClause) error { - bindingsByField := map[string][]string{ - "predicate": {cls.PBinding, cls.PAlias}, - "object": {cls.OBinding, cls.OAlias}, +// compatibleBindingsInClauseForFilterOperation returns the bindings in the given graph clause that +// are compatible with the specified filter operation. +func compatibleBindingsInClauseForFilterOperation(cls *semantic.GraphClause, operation string) (bindingsByField map[string]map[string]bool, err error) { + supportedFiltersAndBindings := map[string]map[string]map[string]bool{ + "latest": { + "predicate": {cls.PBinding: true, cls.PAlias: true}, + "object": {cls.OBinding: true, cls.OAlias: true}, + }, } - for field, bs := range bindingsByField { - for _, b := range bs { - if b == "" { - continue + if _, ok := supportedFiltersAndBindings[operation]; !ok { + err = fmt.Errorf("filter function %q on filter clause is not supported", operation) + return + } + + bindingsByField = supportedFiltersAndBindings[operation] + return +} + +// organizeFilterOptionsByClause processes all the given filters and organize them in a map that has as keys the +// clauses to which they must be applied. +func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []*semantic.GraphClause) (map[*semantic.GraphClause]*storage.FilteringOptions, error) { + clausesByBinding := organizeClausesByBinding(clauses) + filterOptionsByClause := map[*semantic.GraphClause]*storage.FilteringOptions{} + + for _, f := range filters { + if _, ok := clausesByBinding[f.Binding]; !ok { + return nil, fmt.Errorf("binding %q referenced by filter clause %q does not exist in the graph pattern", f.Binding, f) + } + + for _, cls := range clausesByBinding[f.Binding] { + if _, ok := filterOptionsByClause[cls]; ok { + return nil, fmt.Errorf("multiple filters for the same graph clause or same binding are not supported at the moment") } - if _, ok := filtersByBinding[b]; !ok { - continue + + compatibleBindingsByField, err := compatibleBindingsInClauseForFilterOperation(cls, f.Operation) + if err != nil { + return nil, err } - if lo.FilterOptions != nil { - return fmt.Errorf("multiple filters for the same graph clause are not supported at the moment") + + filterBindingIsCompatible := false + for field, bndgs := range compatibleBindingsByField { + if bndgs[f.Binding] { + filterBindingIsCompatible = true + filterOptionsByClause[cls] = &storage.FilteringOptions{ + Operation: f.Operation, + Field: field, + Value: f.Value, + } + break + } } - filter := filtersByBinding[b] - lo.FilterOptions = &storage.FilteringOptions{ - Operation: filter.Operation, - Field: field, - Value: filter.Value, + if !filterBindingIsCompatible { + return nil, fmt.Errorf("binding %q occupies a position in graph clause %q that is incompatible with filter function %q", f.Binding, cls, f.Operation) } } } - return nil + return filterOptionsByClause, nil +} + +// addFilterOptions adds FilterOptions to lookup options if the given clause has bindings for which +// filters were defined (organized in filterOptionsByClause). +func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filterOptionsByClause map[*semantic.GraphClause]*storage.FilteringOptions) { + if _, ok := filterOptionsByClause[cls]; ok { + lo.FilterOptions = filterOptionsByClause[cls] + } } // resetFilterOptions resets FilterOptions in lookup options to nil. @@ -701,7 +733,7 @@ func resetFilterOptions(lo *storage.LookupOptions) { func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupOptions) error { tracer.Trace(p.tracer, func() *tracer.Arguments { var res []string - for i, cls := range p.cls { + for i, cls := range p.clauses { res = append(res, fmt.Sprintf("Clause %d to process: %v", i, cls)) } return &tracer.Arguments{ @@ -709,23 +741,19 @@ func (p *queryPlan) processGraphPattern(ctx context.Context, lo *storage.LookupO } }) - filtersByBinding, err := organizeFiltersByBinding(p.filters, p.bndgsMap) + filterOptionsByClause, err := organizeFilterOptionsByClause(p.filters, p.clauses) if err != nil { return err } - for i, c := range p.cls { - i, cls := i, *c + for i, cls := range p.clauses { tracer.Trace(p.tracer, func() *tracer.Arguments { return &tracer.Arguments{ - Msgs: []string{fmt.Sprintf("Processing clause %d: %v", i, &cls)}, + Msgs: []string{fmt.Sprintf("Processing clause %d: %v", i, cls)}, } }) - err = addFilterOptions(lo, &cls, filtersByBinding) - if err != nil { - return err - } - unresolvable, err := p.processClause(ctx, &cls, lo) + addFilterOptions(lo, cls, filterOptionsByClause) + unresolvable, err := p.processClause(ctx, cls, lo) resetFilterOptions(lo) if err != nil { return err @@ -943,7 +971,7 @@ func (p *queryPlan) String(ctx context.Context) string { b.WriteString("using store(\"") b.WriteString(p.store.Name(nil)) b.WriteString(fmt.Sprintf("\") graphs %v\nresolve\n", p.grfsNames)) - for _, c := range p.cls { + for _, c := range p.clauses { b.WriteString("\t") b.WriteString(c.String()) b.WriteString("\n") diff --git a/bql/semantic/hooks.go b/bql/semantic/hooks.go index 76b5b7dc..2e10c08a 100644 --- a/bql/semantic/hooks.go +++ b/bql/semantic/hooks.go @@ -675,9 +675,6 @@ func whereObjectClause() ElementHook { // if the filter clause is complete, populates the filters list of the statement. func whereFilterClause() ElementHook { var hook ElementHook - supportedFilterFunctions := map[string]bool{ - "latest": true, - } hook = func(st *Statement, ce ConsumedElement) (ElementHook, error) { if ce.IsSymbol() { @@ -694,9 +691,6 @@ func whereFilterClause() ElementHook { if currFilter.Operation != "" { return nil, fmt.Errorf("invalid filter function %q on filter clause since already set to %q", tkn.Text, currFilter.Operation) } - if !supportedFilterFunctions[tkn.Text] { - return nil, fmt.Errorf("filter function %q on filter clause is not supported", tkn.Text) - } currFilter.Operation = tkn.Text return hook, nil case lexer.ItemBinding: From 079a781569605b52da4d25423818246cea0469ad Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Fri, 25 Sep 2020 12:46:53 -0300 Subject: [PATCH 13/26] Use a closure to make planner more performant (especially with the addition of multiple filter functions in the future) --- bql/planner/planner.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index d62f0291..3b4e45a5 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -654,22 +654,22 @@ func organizeClausesByBinding(clauses []*semantic.GraphClause) map[string][]*sem return clausesByBinding } -// compatibleBindingsInClauseForFilterOperation returns the bindings in the given graph clause that -// are compatible with the specified filter operation. -func compatibleBindingsInClauseForFilterOperation(cls *semantic.GraphClause, operation string) (bindingsByField map[string]map[string]bool, err error) { - supportedFiltersAndBindings := map[string]map[string]map[string]bool{ - "latest": { - "predicate": {cls.PBinding: true, cls.PAlias: true}, - "object": {cls.OBinding: true, cls.OAlias: true}, - }, - } - - if _, ok := supportedFiltersAndBindings[operation]; !ok { +// compatibleBindingsInClauseForFilterOperation returns a function that, for each given clause, returns the bindings that are +// compatible with the specified filter operation. +func compatibleBindingsInClauseForFilterOperation(operation string) (compatibleBindingsInClause func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool), err error) { + switch operation { + case "latest": + compatibleBindingsInClause = func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool) { + bindingsByField = map[string]map[string]bool{ + "predicate": {cls.PBinding: true, cls.PAlias: true}, + "object": {cls.OBinding: true, cls.OAlias: true}, + } + return + } + default: err = fmt.Errorf("filter function %q on filter clause is not supported", operation) - return } - bindingsByField = supportedFiltersAndBindings[operation] return } @@ -684,16 +684,16 @@ func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []* return nil, fmt.Errorf("binding %q referenced by filter clause %q does not exist in the graph pattern", f.Binding, f) } + compatibleBindingsInClause, err := compatibleBindingsInClauseForFilterOperation(f.Operation) + if err != nil { + return nil, err + } for _, cls := range clausesByBinding[f.Binding] { if _, ok := filterOptionsByClause[cls]; ok { return nil, fmt.Errorf("multiple filters for the same graph clause or same binding are not supported at the moment") } - compatibleBindingsByField, err := compatibleBindingsInClauseForFilterOperation(cls, f.Operation) - if err != nil { - return nil, err - } - + compatibleBindingsByField := compatibleBindingsInClause(cls) filterBindingIsCompatible := false for field, bndgs := range compatibleBindingsByField { if bndgs[f.Binding] { From 901d807c9d21b33b7d63b58aed8ab4f9d2c5307a Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 11:01:40 -0300 Subject: [PATCH 14/26] Use an "enum" instead of a "string" for filter operations (less error-prone when implementing the driver) --- bql/planner/filter/filter.go | 44 ++++++++++++++++++++++++++++++++++++ bql/planner/planner.go | 7 +++--- bql/semantic/hooks.go | 13 +++++++---- bql/semantic/semantic.go | 3 ++- storage/storage.go | 7 +++++- 5 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 bql/planner/filter/filter.go diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go new file mode 100644 index 00000000..5b6ad592 --- /dev/null +++ b/bql/planner/filter/filter.go @@ -0,0 +1,44 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package filter isolates core FILTER related implementation. +package filter + +import ( + "fmt" +) + +// Operation represents a filter operation supported in BadWolf. +type Operation int + +// List of supported filter operations. +const ( + Latest Operation = iota + 1 +) + +// SupportedOperations maps suported filter operation strings to their correspondant Operation. +// Note that the string keys here must be in lowercase letters only (for compatibility with the WhereFilterClauseHook). +var SupportedOperations = map[string]Operation{ + "latest": Latest, +} + +// String returns the string representation of Operation. +func (op Operation) String() string { + switch op { + case Latest: + return "latest" + default: + return fmt.Sprintf(`not defined filter operation "%d"`, op) + } +} diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 3b4e45a5..5d391b23 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -28,6 +28,7 @@ import ( "sync" "github.com/google/badwolf/bql/lexer" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/bql/planner/tracer" "github.com/google/badwolf/bql/semantic" "github.com/google/badwolf/bql/table" @@ -656,9 +657,9 @@ func organizeClausesByBinding(clauses []*semantic.GraphClause) map[string][]*sem // compatibleBindingsInClauseForFilterOperation returns a function that, for each given clause, returns the bindings that are // compatible with the specified filter operation. -func compatibleBindingsInClauseForFilterOperation(operation string) (compatibleBindingsInClause func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool), err error) { +func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (compatibleBindingsInClause func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool), err error) { switch operation { - case "latest": + case filter.Latest: compatibleBindingsInClause = func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool) { bindingsByField = map[string]map[string]bool{ "predicate": {cls.PBinding: true, cls.PAlias: true}, @@ -667,7 +668,7 @@ func compatibleBindingsInClauseForFilterOperation(operation string) (compatibleB return } default: - err = fmt.Errorf("filter function %q on filter clause is not supported", operation) + err = fmt.Errorf("filter function %q has no bindings in clause specified for it (planner level)", operation) } return diff --git a/bql/semantic/hooks.go b/bql/semantic/hooks.go index 2e10c08a..9e72776b 100644 --- a/bql/semantic/hooks.go +++ b/bql/semantic/hooks.go @@ -21,6 +21,7 @@ import ( "time" "github.com/google/badwolf/bql/lexer" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/bql/table" "github.com/google/badwolf/triple" "github.com/google/badwolf/triple/literal" @@ -688,16 +689,20 @@ func whereFilterClause() ElementHook { if currFilter == nil { return nil, fmt.Errorf("could not add filter function %q to nil filter clause", tkn.Text) } - if currFilter.Operation != "" { + if currFilter.Operation != filter.Operation(0) { return nil, fmt.Errorf("invalid filter function %q on filter clause since already set to %q", tkn.Text, currFilter.Operation) } - currFilter.Operation = tkn.Text + lowercaseFilter := strings.ToLower(tkn.Text) + if _, ok := filter.SupportedOperations[lowercaseFilter]; !ok { + return nil, fmt.Errorf("filter function %q on filter clause is not supported", tkn.Text) + } + currFilter.Operation = filter.SupportedOperations[lowercaseFilter] return hook, nil case lexer.ItemBinding: if currFilter == nil { return nil, fmt.Errorf("could not add binding %q to nil filter clause", tkn.Text) } - if currFilter.Operation == "" { + if currFilter.Operation == filter.Operation(0) { return nil, fmt.Errorf("could not add binding %q to a filter clause that does not have a filter function previously set", tkn.Text) } if currFilter.Binding != "" { @@ -706,7 +711,7 @@ func whereFilterClause() ElementHook { currFilter.Binding = tkn.Text return hook, nil case lexer.ItemRPar: - if currFilter == nil || currFilter.Operation == "" || currFilter.Binding == "" { + if currFilter == nil || currFilter.Operation == filter.Operation(0) || currFilter.Binding == "" { return nil, fmt.Errorf("could not add invalid working filter %q to the statement filters list", currFilter) } st.AddWorkingFilterClause() diff --git a/bql/semantic/semantic.go b/bql/semantic/semantic.go index fe556d0a..ee2ef70d 100644 --- a/bql/semantic/semantic.go +++ b/bql/semantic/semantic.go @@ -27,6 +27,7 @@ import ( "time" "github.com/google/badwolf/bql/lexer" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/bql/table" "github.com/google/badwolf/storage" "github.com/google/badwolf/triple" @@ -150,7 +151,7 @@ type GraphClause struct { // will be applied to and Value, when specified, contains the second argument of the filter function (not applicable for all // Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). type FilterClause struct { - Operation string + Operation filter.Operation Binding string Value string } diff --git a/storage/storage.go b/storage/storage.go index db5e7656..139595c2 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -22,6 +22,7 @@ import ( "strconv" "time" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/triple" "github.com/google/badwolf/triple/node" "github.com/google/badwolf/triple/predicate" @@ -52,7 +53,11 @@ type LookupOptions struct { // Operation below refers to the filter function being applied (eg: "latest"), Field refers to the position of the graph clause it // will be applied to ("subject", "predicate" or "object") and Value, when specified, contains the second argument of the filter // function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). -type FilteringOptions struct{ Operation, Field, Value string } +type FilteringOptions struct { + Operation filter.Operation + Field string + Value string +} // String returns a readable version of the LookupOptions instance. func (l *LookupOptions) String() string { From a861175b3def5a1f68f165518e135fbc2bcd1c63 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 11:11:49 -0300 Subject: [PATCH 15/26] Update "memory.go" and "memory_test.go" since filter operations are "enum" now --- storage/memory/memory.go | 25 +++++++------- storage/memory/memory_test.go | 61 ++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/storage/memory/memory.go b/storage/memory/memory.go index 2ede6f70..c18c5fd8 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -22,6 +22,7 @@ import ( "sync" "time" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/storage" "github.com/google/badwolf/triple" "github.com/google/badwolf/triple/node" @@ -324,7 +325,7 @@ func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Pre // executeFilter executes the proper filter operation over memoryTriples following the specifications given in filterOptions. func executeFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { switch filterOptions.Operation { - case "latest": + case filter.Latest: return latestFilter(memoryTriples, pQuery, filterOptions) default: return nil, fmt.Errorf("filter operation %q not supported in the driver", filterOptions.Operation) @@ -350,7 +351,7 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -399,7 +400,7 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -448,7 +449,7 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -495,7 +496,7 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -542,7 +543,7 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -589,7 +590,7 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -636,7 +637,7 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -683,7 +684,7 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -732,7 +733,7 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -781,7 +782,7 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". @@ -836,7 +837,7 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } lo.FilterOptions = &storage.FilteringOptions{ - Operation: "latest", + Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index ad783c6f..bc53de88 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/storage" "github.com/google/badwolf/tools/testutil" "github.com/google/badwolf/triple" @@ -330,19 +331,19 @@ func TestObjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{"/u": 1, "/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`"meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -442,19 +443,19 @@ func TestSubjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{"/_": 1}, @@ -552,19 +553,19 @@ func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, @@ -661,12 +662,12 @@ func TestPredicatesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 2}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -761,17 +762,17 @@ func TestPredicatesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -863,12 +864,12 @@ func TestTriplesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -960,17 +961,17 @@ func TestTriplesForPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -1062,17 +1063,17 @@ func TestTriplesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, }, @@ -1208,19 +1209,19 @@ func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -1315,19 +1316,19 @@ func TestTriplesForPredicateAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, @@ -1437,11 +1438,11 @@ func TestTriplesFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: "latest", Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, } From ebd7b24ab2b7f895c0b633d51bf953734f68a309 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 11:17:45 -0300 Subject: [PATCH 16/26] Update "hooks_test.go" and "semantic_test.go" since filter operations are "enum" now --- bql/semantic/hooks_test.go | 5 +++-- bql/semantic/semantic_test.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bql/semantic/hooks_test.go b/bql/semantic/hooks_test.go index b9dd18fe..4d1a0a3d 100644 --- a/bql/semantic/hooks_test.go +++ b/bql/semantic/hooks_test.go @@ -21,6 +21,7 @@ import ( "time" "github.com/google/badwolf/bql/lexer" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/bql/table" "github.com/google/badwolf/storage" "github.com/google/badwolf/triple" @@ -223,7 +224,7 @@ func TestWhereFilterClauseHook(t *testing.T) { NewConsumedSymbol("FOO"), }, want: &FilterClause{ - Operation: "latest", + Operation: filter.Latest, Binding: "?p", }, }, @@ -253,7 +254,7 @@ func TestWhereFilterClauseHook(t *testing.T) { NewConsumedSymbol("FOO"), }, want: &FilterClause{ - Operation: "latest", + Operation: filter.Latest, Binding: "?o", }, }, diff --git a/bql/semantic/semantic_test.go b/bql/semantic/semantic_test.go index 5bcd16c4..0fe5e60a 100644 --- a/bql/semantic/semantic_test.go +++ b/bql/semantic/semantic_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/google/badwolf/bql/planner/filter" "github.com/google/badwolf/triple" "github.com/google/badwolf/triple/literal" "github.com/google/badwolf/triple/node" @@ -274,7 +275,7 @@ func TestFilterClauseManipulation(t *testing.T) { t.Run("add workingFilter success", func(t *testing.T) { wf := st.WorkingFilter() - *wf = FilterClause{Operation: "latest", Binding: "?p"} + *wf = FilterClause{Operation: filter.Latest, Binding: "?p"} st.AddWorkingFilterClause() if got, want := len(st.FilterClauses()), 1; got != want { t.Fatalf(`len(semantic.Statement.FilterClauses()) = %d for statement "%v"; want %d`, got, st, want) @@ -360,7 +361,7 @@ func TestIsEmptyFilterClause(t *testing.T) { want: true, }, { - in: &FilterClause{Operation: "latest", Binding: "?p"}, + in: &FilterClause{Operation: filter.Latest, Binding: "?p"}, want: false, }, } From 0f1dfc8e7e37949ad119648387f3bd6f83d42afb Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 16:16:29 -0300 Subject: [PATCH 17/26] Move "storage.FilteringOptions" to "filter.go" as "filter.StorageOptions" --- bql/planner/filter/filter.go | 10 ++++++ bql/planner/planner.go | 10 +++--- storage/memory/memory.go | 48 ++++++++++++++-------------- storage/memory/memory_test.go | 60 +++++++++++++++++------------------ storage/storage.go | 12 +------ 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go index 5b6ad592..5567e43b 100644 --- a/bql/planner/filter/filter.go +++ b/bql/planner/filter/filter.go @@ -33,6 +33,16 @@ var SupportedOperations = map[string]Operation{ "latest": Latest, } +// StorageOptions represent the storage level specifications for the filtering to be executed. +// Operation below refers to the filter function being applied (eg: "latest"), Field refers to the position of the graph clause it +// will be applied to ("subject", "predicate" or "object") and Value, when specified, contains the second argument of the filter +// function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). +type StorageOptions struct { + Operation Operation + Field string + Value string +} + // String returns the string representation of Operation. func (op Operation) String() string { switch op { diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 5d391b23..f511b205 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -676,9 +676,9 @@ func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (c // organizeFilterOptionsByClause processes all the given filters and organize them in a map that has as keys the // clauses to which they must be applied. -func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []*semantic.GraphClause) (map[*semantic.GraphClause]*storage.FilteringOptions, error) { +func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []*semantic.GraphClause) (map[*semantic.GraphClause]*filter.StorageOptions, error) { clausesByBinding := organizeClausesByBinding(clauses) - filterOptionsByClause := map[*semantic.GraphClause]*storage.FilteringOptions{} + filterOptionsByClause := map[*semantic.GraphClause]*filter.StorageOptions{} for _, f := range filters { if _, ok := clausesByBinding[f.Binding]; !ok { @@ -699,7 +699,7 @@ func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []* for field, bndgs := range compatibleBindingsByField { if bndgs[f.Binding] { filterBindingIsCompatible = true - filterOptionsByClause[cls] = &storage.FilteringOptions{ + filterOptionsByClause[cls] = &filter.StorageOptions{ Operation: f.Operation, Field: field, Value: f.Value, @@ -718,7 +718,7 @@ func organizeFilterOptionsByClause(filters []*semantic.FilterClause, clauses []* // addFilterOptions adds FilterOptions to lookup options if the given clause has bindings for which // filters were defined (organized in filterOptionsByClause). -func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filterOptionsByClause map[*semantic.GraphClause]*storage.FilteringOptions) { +func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filterOptionsByClause map[*semantic.GraphClause]*filter.StorageOptions) { if _, ok := filterOptionsByClause[cls]; ok { lo.FilterOptions = filterOptionsByClause[cls] } @@ -726,7 +726,7 @@ func addFilterOptions(lo *storage.LookupOptions, cls *semantic.GraphClause, filt // resetFilterOptions resets FilterOptions in lookup options to nil. func resetFilterOptions(lo *storage.LookupOptions) { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) } // processGraphPattern process the query graph pattern to retrieve the diff --git a/storage/memory/memory.go b/storage/memory/memory.go index c18c5fd8..bb76e817 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -276,7 +276,7 @@ func (c *checker) CheckAndUpdate(p *predicate.Predicate) bool { } // latestFilter executes the latest filter operation over memoryTriples following filterOptions. -func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { +func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *filter.StorageOptions) (map[string]*triple.Triple, error) { if filterOptions.Field != "predicate" && filterOptions.Field != "object" { return nil, fmt.Errorf(`invalid field %q for "latest" filter operation, can accept only "predicate" or "object"`, filterOptions.Field) } @@ -323,7 +323,7 @@ func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Pre } // executeFilter executes the proper filter operation over memoryTriples following the specifications given in filterOptions. -func executeFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *storage.FilteringOptions) (map[string]*triple.Triple, error) { +func executeFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *filter.StorageOptions) (map[string]*triple.Triple, error) { switch filterOptions.Operation { case filter.Latest: return latestFilter(memoryTriples, pQuery, filterOptions) @@ -350,13 +350,13 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -399,13 +399,13 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -448,13 +448,13 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -495,13 +495,13 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -542,13 +542,13 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -589,13 +589,13 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -636,13 +636,13 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -683,13 +683,13 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -732,13 +732,13 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -781,13 +781,13 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { @@ -836,13 +836,13 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c if lo.FilterOptions != nil { return fmt.Errorf("cannot have LatestAnchor and FilterOptions used at the same time inside lookup options") } - lo.FilterOptions = &storage.FilteringOptions{ + lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, Field: "predicate", } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { - lo.FilterOptions = (*storage.FilteringOptions)(nil) + lo.FilterOptions = (*filter.StorageOptions)(nil) }() } if lo.FilterOptions != nil { diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index bc53de88..56eeea76 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -331,19 +331,19 @@ func TestObjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{"/u": 1, "/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`"meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -443,19 +443,19 @@ func TestSubjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{"/_": 1}, @@ -553,19 +553,19 @@ func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, @@ -662,12 +662,12 @@ func TestPredicatesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 2}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -762,17 +762,17 @@ func TestPredicatesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -864,12 +864,12 @@ func TestTriplesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -961,17 +961,17 @@ func TestTriplesForPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -1063,17 +1063,17 @@ func TestTriplesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, }, @@ -1209,19 +1209,19 @@ func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -1316,19 +1316,19 @@ func TestTriplesForPredicateAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, @@ -1438,11 +1438,11 @@ func TestTriplesFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &storage.FilteringOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, } diff --git a/storage/storage.go b/storage/storage.go index 139595c2..c0e6908f 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -46,17 +46,7 @@ type LookupOptions struct { LatestAnchor bool // FilterOptions, if provided, represent the specifications for the filtering to be executed. - FilterOptions *FilteringOptions -} - -// FilteringOptions represent the storage level specifications for the filtering to be executed. -// Operation below refers to the filter function being applied (eg: "latest"), Field refers to the position of the graph clause it -// will be applied to ("subject", "predicate" or "object") and Value, when specified, contains the second argument of the filter -// function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). -type FilteringOptions struct { - Operation filter.Operation - Field string - Value string + FilterOptions *filter.StorageOptions } // String returns a readable version of the LookupOptions instance. From d5f8ff8d721581063298048ef699790ba7a3efa5 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 16:59:35 -0300 Subject: [PATCH 18/26] Add a String method for filter.StorageOptions --- bql/planner/filter/filter.go | 5 +++++ storage/storage.go | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go index 5567e43b..cb01dae7 100644 --- a/bql/planner/filter/filter.go +++ b/bql/planner/filter/filter.go @@ -52,3 +52,8 @@ func (op Operation) String() string { return fmt.Sprintf(`not defined filter operation "%d"`, op) } } + +// String returns the string representation of StorageOptions. +func (so *StorageOptions) String() string { + return fmt.Sprintf("%+v", *so) +} diff --git a/storage/storage.go b/storage/storage.go index c0e6908f..de903e63 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -66,9 +66,7 @@ func (l *LookupOptions) String() string { b.WriteString("nil") } b.WriteString(fmt.Sprintf(", LatestAnchor=%v", l.LatestAnchor)) - b.WriteString(", FilterOptions=") - b.WriteString(fmt.Sprintf("%+v", l.FilterOptions)) - b.WriteString(">") + b.WriteString(fmt.Sprintf(", FilterOptions=%s>", l.FilterOptions)) return b.String() } From d62e88b04508ea85daf5fea0369ffd8badb3b4ca Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 21:50:26 -0300 Subject: [PATCH 19/26] Use an "enum" instead of a "string" for filter fields (less error-prone when implementing the driver) --- bql/planner/filter/filter.go | 26 +++++++++++++++++++++++++- bql/planner/planner.go | 10 +++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go index cb01dae7..2bfeacc7 100644 --- a/bql/planner/filter/filter.go +++ b/bql/planner/filter/filter.go @@ -27,6 +27,16 @@ const ( Latest Operation = iota + 1 ) +// Field represents the position of the semantic.GraphClause that will be operated by the filter at storage level. +type Field int + +// List of filter fields. +const ( + SubjectField Field = iota + 1 + PredicateField + ObjectField +) + // SupportedOperations maps suported filter operation strings to their correspondant Operation. // Note that the string keys here must be in lowercase letters only (for compatibility with the WhereFilterClauseHook). var SupportedOperations = map[string]Operation{ @@ -39,7 +49,7 @@ var SupportedOperations = map[string]Operation{ // function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). type StorageOptions struct { Operation Operation - Field string + Field Field Value string } @@ -53,6 +63,20 @@ func (op Operation) String() string { } } +// String returns the string representation of Field. +func (f Field) String() string { + switch f { + case SubjectField: + return "subject field" + case PredicateField: + return "predicate field" + case ObjectField: + return "object field" + default: + return fmt.Sprintf(`not defined filter field "%d"`, f) + } +} + // String returns the string representation of StorageOptions. func (so *StorageOptions) String() string { return fmt.Sprintf("%+v", *so) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index f511b205..277e0323 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -657,13 +657,13 @@ func organizeClausesByBinding(clauses []*semantic.GraphClause) map[string][]*sem // compatibleBindingsInClauseForFilterOperation returns a function that, for each given clause, returns the bindings that are // compatible with the specified filter operation. -func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (compatibleBindingsInClause func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool), err error) { +func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (compatibleBindingsInClause func(cls *semantic.GraphClause) (bindingsByField map[filter.Field]map[string]bool), err error) { switch operation { case filter.Latest: - compatibleBindingsInClause = func(cls *semantic.GraphClause) (bindingsByField map[string]map[string]bool) { - bindingsByField = map[string]map[string]bool{ - "predicate": {cls.PBinding: true, cls.PAlias: true}, - "object": {cls.OBinding: true, cls.OAlias: true}, + compatibleBindingsInClause = func(cls *semantic.GraphClause) (bindingsByField map[filter.Field]map[string]bool) { + bindingsByField = map[filter.Field]map[string]bool{ + filter.PredicateField: {cls.PBinding: true, cls.PAlias: true}, + filter.ObjectField: {cls.OBinding: true, cls.OAlias: true}, } return } From 960ec7bfc44a9944dffb7fb0d5f947f6dab1ae85 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 1 Oct 2020 22:05:58 -0300 Subject: [PATCH 20/26] Update "memory.go" and "memory_test.go" since filter fields are "enum" now --- storage/memory/memory.go | 30 +++++++++--------- storage/memory/memory_test.go | 60 +++++++++++++++++------------------ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/storage/memory/memory.go b/storage/memory/memory.go index bb76e817..0fc88821 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -277,8 +277,8 @@ func (c *checker) CheckAndUpdate(p *predicate.Predicate) bool { // latestFilter executes the latest filter operation over memoryTriples following filterOptions. func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Predicate, filterOptions *filter.StorageOptions) (map[string]*triple.Triple, error) { - if filterOptions.Field != "predicate" && filterOptions.Field != "object" { - return nil, fmt.Errorf(`invalid field %q for "latest" filter operation, can accept only "predicate" or "object"`, filterOptions.Field) + if filterOptions.Field != filter.PredicateField && filterOptions.Field != filter.ObjectField { + return nil, fmt.Errorf("invalid field %q for %q filter operation, can accept only %q or %q", filterOptions.Field, filter.Latest, filter.PredicateField, filter.ObjectField) } lastTA := make(map[string]*time.Time) @@ -289,9 +289,9 @@ func latestFilter(memoryTriples map[string]*triple.Triple, pQuery *predicate.Pre } var p *predicate.Predicate - if filterOptions.Field == "predicate" { + if filterOptions.Field == filter.PredicateField { p = t.Predicate() - } else if pObj, err := t.Object().Predicate(); filterOptions.Field == "object" && err == nil { + } else if pObj, err := t.Object().Predicate(); filterOptions.Field == filter.ObjectField && err == nil { p = pObj } else { continue @@ -352,7 +352,7 @@ func (m *memory) Objects(ctx context.Context, s *node.Node, p *predicate.Predica } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -401,7 +401,7 @@ func (m *memory) Subjects(ctx context.Context, p *predicate.Predicate, o *triple } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -450,7 +450,7 @@ func (m *memory) PredicatesForSubjectAndObject(ctx context.Context, s *node.Node } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -497,7 +497,7 @@ func (m *memory) PredicatesForSubject(ctx context.Context, s *node.Node, lo *sto } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -544,7 +544,7 @@ func (m *memory) PredicatesForObject(ctx context.Context, o *triple.Object, lo * } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -591,7 +591,7 @@ func (m *memory) TriplesForSubject(ctx context.Context, s *node.Node, lo *storag } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -638,7 +638,7 @@ func (m *memory) TriplesForPredicate(ctx context.Context, p *predicate.Predicate } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -685,7 +685,7 @@ func (m *memory) TriplesForObject(ctx context.Context, o *triple.Object, lo *sto } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -734,7 +734,7 @@ func (m *memory) TriplesForSubjectAndPredicate(ctx context.Context, s *node.Node } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -783,7 +783,7 @@ func (m *memory) TriplesForPredicateAndObject(ctx context.Context, p *predicate. } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { @@ -838,7 +838,7 @@ func (m *memory) Triples(ctx context.Context, lo *storage.LookupOptions, trpls c } lo.FilterOptions = &filter.StorageOptions{ Operation: filter.Latest, - Field: "predicate", + Field: filter.PredicateField, } // To guarantee that "lo.FilterOptions" will be cleaned at the driver level, since it was artificially created at the driver level for "LatestAnchor". defer func() { diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index 56eeea76..0c7cf553 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -331,19 +331,19 @@ func TestObjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{"/u": 1, "/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`"meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -443,19 +443,19 @@ func TestSubjectsFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{"/_": 1}, @@ -553,19 +553,19 @@ func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, @@ -662,12 +662,12 @@ func TestPredicatesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 2}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -762,17 +762,17 @@ func TestPredicatesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, }, @@ -864,12 +864,12 @@ func TestTriplesForSubjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -961,17 +961,17 @@ func TestTriplesForPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, @@ -1063,17 +1063,17 @@ func TestTriplesForObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, }, @@ -1209,19 +1209,19 @@ func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -1316,19 +1316,19 @@ func TestTriplesForPredicateAndObjectFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, @@ -1438,11 +1438,11 @@ func TestTriplesFilter(t *testing.T) { want map[string]int }{ { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "predicate"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: "object"}}, + lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, } From caf9d854ee198c94531b6fe158850617ecd6e29b Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Mon, 5 Oct 2020 12:06:04 -0300 Subject: [PATCH 21/26] Add an "isEmpty" method for "filter.Operation" --- bql/planner/filter/filter.go | 5 +++++ bql/semantic/hooks.go | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go index 2bfeacc7..09a61b2a 100644 --- a/bql/planner/filter/filter.go +++ b/bql/planner/filter/filter.go @@ -63,6 +63,11 @@ func (op Operation) String() string { } } +// IsEmpty returns true if the Operation was not set yet. +func (op Operation) IsEmpty() bool { + return op == Operation(0) +} + // String returns the string representation of Field. func (f Field) String() string { switch f { diff --git a/bql/semantic/hooks.go b/bql/semantic/hooks.go index 9e72776b..044a7635 100644 --- a/bql/semantic/hooks.go +++ b/bql/semantic/hooks.go @@ -689,7 +689,7 @@ func whereFilterClause() ElementHook { if currFilter == nil { return nil, fmt.Errorf("could not add filter function %q to nil filter clause", tkn.Text) } - if currFilter.Operation != filter.Operation(0) { + if !currFilter.Operation.IsEmpty() { return nil, fmt.Errorf("invalid filter function %q on filter clause since already set to %q", tkn.Text, currFilter.Operation) } lowercaseFilter := strings.ToLower(tkn.Text) @@ -702,7 +702,7 @@ func whereFilterClause() ElementHook { if currFilter == nil { return nil, fmt.Errorf("could not add binding %q to nil filter clause", tkn.Text) } - if currFilter.Operation == filter.Operation(0) { + if currFilter.Operation.IsEmpty() { return nil, fmt.Errorf("could not add binding %q to a filter clause that does not have a filter function previously set", tkn.Text) } if currFilter.Binding != "" { @@ -711,7 +711,7 @@ func whereFilterClause() ElementHook { currFilter.Binding = tkn.Text return hook, nil case lexer.ItemRPar: - if currFilter == nil || currFilter.Operation == filter.Operation(0) || currFilter.Binding == "" { + if currFilter == nil || currFilter.Operation.IsEmpty() || currFilter.Binding == "" { return nil, fmt.Errorf("could not add invalid working filter %q to the statement filters list", currFilter) } st.AddWorkingFilterClause() From d480b9d04d7f6108b1475aa4cc98b9b6e2beec19 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Mon, 5 Oct 2020 15:06:08 -0300 Subject: [PATCH 22/26] Update comments for "semantic.FilterClause" and "filter.StorageOptions" --- bql/planner/filter/filter.go | 6 +++--- bql/semantic/semantic.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bql/planner/filter/filter.go b/bql/planner/filter/filter.go index 09a61b2a..3311757e 100644 --- a/bql/planner/filter/filter.go +++ b/bql/planner/filter/filter.go @@ -44,9 +44,9 @@ var SupportedOperations = map[string]Operation{ } // StorageOptions represent the storage level specifications for the filtering to be executed. -// Operation below refers to the filter function being applied (eg: "latest"), Field refers to the position of the graph clause it -// will be applied to ("subject", "predicate" or "object") and Value, when specified, contains the second argument of the filter -// function (not applicable for all Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). +// Operation below refers to the filter function being applied (eg: Latest), Field refers to the position of the graph clause it +// will be applied to (subject, predicate, or object) and Value, when specified, contains the second argument of the filter +// function (not applicable for all Operations - some like Latest do not use it while others like GreaterThan do, see Issue 129). type StorageOptions struct { Operation Operation Field Field diff --git a/bql/semantic/semantic.go b/bql/semantic/semantic.go index ee2ef70d..c753e1be 100644 --- a/bql/semantic/semantic.go +++ b/bql/semantic/semantic.go @@ -147,9 +147,9 @@ type GraphClause struct { } // FilterClause represents a FILTER clause inside WHERE. -// Operation below refers to the filter function being applied (eg: "latest"), Binding refers to the binding it +// Operation below refers to the filter function being applied (eg: Latest), Binding refers to the binding it // will be applied to and Value, when specified, contains the second argument of the filter function (not applicable for all -// Operations - some like "latest" do not use it while others like "greaterThan" do, see Issue 129). +// Operations - some like Latest do not use it while others like GreaterThan do, see Issue 129). type FilterClause struct { Operation filter.Operation Binding string From 5f7962a6d1ced409bc5953898f7d708ac9050cd4 Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Fri, 9 Oct 2020 19:08:29 -0300 Subject: [PATCH 23/26] Refactor "WhereFilterClauseHook" to make it clearer (separating code into auxiliar functions) --- bql/semantic/hooks.go | 69 +++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/bql/semantic/hooks.go b/bql/semantic/hooks.go index 044a7635..f11e52f3 100644 --- a/bql/semantic/hooks.go +++ b/bql/semantic/hooks.go @@ -672,6 +672,45 @@ func whereObjectClause() ElementHook { return hook } +// addOperationToWorkingFilter takes the filter operation in its string format and tries to add the +// correspondent filter.Operation to workingFilter. +func addOperationToWorkingFilter(op string, workingFilter *FilterClause) error { + if workingFilter == nil { + return fmt.Errorf("could not add filter function %q to nil filter clause (which is still nil probably because a call to st.ResetWorkingFilterClause was not made before start processing the first filter clause)", op) + } + if !workingFilter.Operation.IsEmpty() { + return fmt.Errorf("invalid filter function %q on filter clause since already set to %q", op, workingFilter.Operation) + } + lowercaseOp := strings.ToLower(op) + if _, ok := filter.SupportedOperations[lowercaseOp]; !ok { + return fmt.Errorf("filter function %q on filter clause is not supported", op) + } + + workingFilter.Operation = filter.SupportedOperations[lowercaseOp] + return nil +} + +// addBindingToWorkingFilter takes the given binding and tries to add it to workingFilter. +func addBindingToWorkingFilter(bndg string, workingFilter *FilterClause) error { + if workingFilter == nil { + return fmt.Errorf("could not add binding %q to nil filter clause (which is still nil probably because a call to st.ResetWorkingFilterClause was not made before start processing the first filter clause)", bndg) + } + if workingFilter.Operation.IsEmpty() { + return fmt.Errorf("could not add binding %q to a filter clause that does not have a filter function previously set", bndg) + } + if workingFilter.Binding != "" { + return fmt.Errorf("invalid binding %q on filter clause since already set to %q", bndg, workingFilter.Binding) + } + + workingFilter.Binding = bndg + return nil +} + +// isValidFilterClause returns true if the given filter clause is valid and complete. +func isValidFilterClause(f *FilterClause) bool { + return f != nil && !f.Operation.IsEmpty() && f.Binding != "" +} + // whereFilterClause returns an element hook that updates the working filter clause and, // if the filter clause is complete, populates the filters list of the statement. func whereFilterClause() ElementHook { @@ -683,36 +722,22 @@ func whereFilterClause() ElementHook { } tkn := ce.Token() - currFilter := st.WorkingFilter() switch tkn.Type { case lexer.ItemFilterFunction: - if currFilter == nil { - return nil, fmt.Errorf("could not add filter function %q to nil filter clause", tkn.Text) - } - if !currFilter.Operation.IsEmpty() { - return nil, fmt.Errorf("invalid filter function %q on filter clause since already set to %q", tkn.Text, currFilter.Operation) - } - lowercaseFilter := strings.ToLower(tkn.Text) - if _, ok := filter.SupportedOperations[lowercaseFilter]; !ok { - return nil, fmt.Errorf("filter function %q on filter clause is not supported", tkn.Text) + err := addOperationToWorkingFilter(tkn.Text, st.WorkingFilter()) + if err != nil { + return nil, err } - currFilter.Operation = filter.SupportedOperations[lowercaseFilter] return hook, nil case lexer.ItemBinding: - if currFilter == nil { - return nil, fmt.Errorf("could not add binding %q to nil filter clause", tkn.Text) - } - if currFilter.Operation.IsEmpty() { - return nil, fmt.Errorf("could not add binding %q to a filter clause that does not have a filter function previously set", tkn.Text) - } - if currFilter.Binding != "" { - return nil, fmt.Errorf("invalid binding %q on filter clause since already set to %q", tkn.Text, currFilter.Binding) + err := addBindingToWorkingFilter(tkn.Text, st.WorkingFilter()) + if err != nil { + return nil, err } - currFilter.Binding = tkn.Text return hook, nil case lexer.ItemRPar: - if currFilter == nil || currFilter.Operation.IsEmpty() || currFilter.Binding == "" { - return nil, fmt.Errorf("could not add invalid working filter %q to the statement filters list", currFilter) + if !isValidFilterClause(st.WorkingFilter()) { + return nil, fmt.Errorf("could not add invalid working filter %q to the statement filters list", st.WorkingFilter()) } st.AddWorkingFilterClause() } From 0c8c4228e6fdbb7aff21c915bedf5f009761851f Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Tue, 13 Oct 2020 16:37:37 -0300 Subject: [PATCH 24/26] Use "t.Run" for FILTER tests in "memory_test.go", and make it cleaner --- storage/memory/memory_test.go | 502 ++++++++++++++++++---------------- 1 file changed, 273 insertions(+), 229 deletions(-) diff --git a/storage/memory/memory_test.go b/storage/memory/memory_test.go index 0c7cf553..760a7c0b 100644 --- a/storage/memory/memory_test.go +++ b/storage/memory/memory_test.go @@ -325,24 +325,28 @@ func TestObjectsFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions s *node.Node p *predicate.Predicate want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{"/u": 1}, }, { + id: "FILTER latest predicate duplicate timestamp", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{"/u": 1, "/u": 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), @@ -351,28 +355,30 @@ func TestObjectsFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - os := make(chan *triple.Object, 100) - s := entry.s - p := entry.p - if err := g.Objects(ctx, s, p, entry.lo, os); err != nil { - t.Fatalf("g.Objects(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) - } - for o := range os { - oStr := o.String() - if _, ok := entry.want[oStr]; !ok { - t.Fatalf("g.Objects(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, oStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + os := make(chan *triple.Object, 100) + s := entry.s + p := entry.p + if err := g.Objects(ctx, s, p, entry.lo, os); err != nil { + t.Fatalf("g.Objects(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) } - entry.want[oStr] = entry.want[oStr] - 1 - if entry.want[oStr] == 0 { - delete(entry.want, oStr) + for o := range os { + oStr := o.String() + if _, ok := entry.want[oStr]; !ok { + t.Fatalf("g.Objects(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, oStr) + } + entry.want[oStr] = entry.want[oStr] - 1 + if entry.want[oStr] == 0 { + delete(entry.want, oStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.Objects(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.Objects(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) + } + }) } } @@ -437,24 +443,28 @@ func TestSubjectsFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions p *predicate.Predicate o *triple.Object want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { + id: "FILTER latest predicate duplicate timestamp", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{"/u": 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), @@ -463,28 +473,30 @@ func TestSubjectsFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - ss := make(chan *node.Node, 100) - p := entry.p - o := entry.o - if err := g.Subjects(ctx, p, o, entry.lo, ss); err != nil { - t.Fatalf("g.Subjects(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) - } - for s := range ss { - sStr := s.String() - if _, ok := entry.want[sStr]; !ok { - t.Fatalf("g.Subjects(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, sStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + ss := make(chan *node.Node, 100) + p := entry.p + o := entry.o + if err := g.Subjects(ctx, p, o, entry.lo, ss); err != nil { + t.Fatalf("g.Subjects(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) } - entry.want[sStr] = entry.want[sStr] - 1 - if entry.want[sStr] == 0 { - delete(entry.want, sStr) + for s := range ss { + sStr := s.String() + if _, ok := entry.want[sStr]; !ok { + t.Fatalf("g.Subjects(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, sStr) + } + entry.want[sStr] = entry.want[sStr] - 1 + if entry.want[sStr] == 0 { + delete(entry.want, sStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.Subjects(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.Subjects(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) + } + }) } } @@ -547,24 +559,21 @@ func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions s *node.Node o *triple.Object want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, - s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), - o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), - want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, - }, - { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), @@ -573,28 +582,30 @@ func TestPredicatesForSubjectAndObjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - pp := make(chan *predicate.Predicate, 100) - s := entry.s - o := entry.o - if err := g.PredicatesForSubjectAndObject(ctx, s, o, entry.lo, pp); err != nil { - t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) = %v; want nil", s, o, entry.lo, err) - } - for p := range pp { - pStr := p.String() - if _, ok := entry.want[pStr]; !ok { - t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) retrieved unexpected %s", s, o, entry.lo, pStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + s := entry.s + o := entry.o + if err := g.PredicatesForSubjectAndObject(ctx, s, o, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) = %v; want nil", s, o, entry.lo, err) } - entry.want[pStr] = entry.want[pStr] - 1 - if entry.want[pStr] == 0 { - delete(entry.want, pStr) + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForSubjectAndObject(%s, %s, %s) retrieved unexpected %s", s, o, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.PredicatesForSubjectAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", s, o, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForSubjectAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", s, o, entry.lo, entry.want) + } + }) } } @@ -657,16 +668,19 @@ func TestPredicatesForSubjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions s *node.Node want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 2}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`"_predicate"@[]`: 1}, @@ -674,27 +688,29 @@ func TestPredicatesForSubjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - pp := make(chan *predicate.Predicate, 100) - s := entry.s - if err := g.PredicatesForSubject(ctx, s, entry.lo, pp); err != nil { - t.Fatalf("g.PredicatesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) - } - for p := range pp { - pStr := p.String() - if _, ok := entry.want[pStr]; !ok { - t.Fatalf("g.PredicatesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, pStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + s := entry.s + if err := g.PredicatesForSubject(ctx, s, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) } - entry.want[pStr] = entry.want[pStr] - 1 - if entry.want[pStr] == 0 { - delete(entry.want, pStr) + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.PredicatesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) + } + }) } } @@ -757,21 +773,19 @@ func TestPredicatesForObjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions o *triple.Object want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, - o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), - want: map[string]int{`"meet"@[2014-04-10T04:21:00Z]`: 1}, - }, - { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`"_predicate"@[]`: 1}, @@ -779,27 +793,29 @@ func TestPredicatesForObjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - pp := make(chan *predicate.Predicate, 100) - o := entry.o - if err := g.PredicatesForObject(ctx, o, entry.lo, pp); err != nil { - t.Fatalf("g.PredicatesForObject(%s, %s) = %v; want nil", o, entry.lo, err) - } - for p := range pp { - pStr := p.String() - if _, ok := entry.want[pStr]; !ok { - t.Fatalf("g.PredicatesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, pStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + pp := make(chan *predicate.Predicate, 100) + o := entry.o + if err := g.PredicatesForObject(ctx, o, entry.lo, pp); err != nil { + t.Fatalf("g.PredicatesForObject(%s, %s) = %v; want nil", o, entry.lo, err) } - entry.want[pStr] = entry.want[pStr] - 1 - if entry.want[pStr] == 0 { - delete(entry.want, pStr) + for p := range pp { + pStr := p.String() + if _, ok := entry.want[pStr]; !ok { + t.Fatalf("g.PredicatesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, pStr) + } + entry.want[pStr] = entry.want[pStr] - 1 + if entry.want[pStr] == 0 { + delete(entry.want, pStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.PredicatesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.PredicatesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) + } + }) } } @@ -859,16 +875,19 @@ func TestTriplesForSubjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions s *node.Node want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -876,27 +895,29 @@ func TestTriplesForSubjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - s := entry.s - if err := g.TriplesForSubject(ctx, s, entry.lo, trpls); err != nil { - t.Fatalf("g.TriplesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.TriplesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + s := entry.s + if err := g.TriplesForSubject(ctx, s, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForSubject(%s, %s) = %v; want nil", s, entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForSubject(%s, %s) retrieved unexpected %s", s, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.TriplesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForSubject(%s, %s) failed to retrieve some expected elements: %v", s, entry.lo, entry.want) + } + }) } } @@ -956,21 +977,25 @@ func TestTriplesForPredicateFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions p *predicate.Predicate want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest predicate duplicate timestamp", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, @@ -978,27 +1003,29 @@ func TestTriplesForPredicateFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - p := entry.p - if err := g.TriplesForPredicate(ctx, p, entry.lo, trpls); err != nil { - t.Fatalf("g.TriplesForPredicate(%s, %s) = %v; want nil", p, entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.TriplesForPredicate(%s, %s) retrieved unexpected %s", p, entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + p := entry.p + if err := g.TriplesForPredicate(ctx, p, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForPredicate(%s, %s) = %v; want nil", p, entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForPredicate(%s, %s) retrieved unexpected %s", p, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.TriplesForPredicate(%s, %s) failed to retrieve some expected elements: %v", p, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForPredicate(%s, %s) failed to retrieve some expected elements: %v", p, entry.lo, entry.want) + } + }) } } @@ -1058,21 +1085,19 @@ func TestTriplesForObjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions o *triple.Object want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { - lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, - o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "bob")), - want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, - }, - { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), want: map[string]int{`/_ "_predicate"@[] "meet"@[2020-04-10T04:21:00Z]`: 1}, @@ -1080,27 +1105,29 @@ func TestTriplesForObjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - o := entry.o - if err := g.TriplesForObject(ctx, o, entry.lo, trpls); err != nil { - t.Fatalf("g.TriplesForObject(%s, %s) = %v; want nil", o, entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.TriplesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + o := entry.o + if err := g.TriplesForObject(ctx, o, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForObject(%s, %s) = %v; want nil", o, entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForObject(%s, %s) retrieved unexpected %s", o, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.TriplesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForObject(%s, %s) failed to retrieve some expected elements: %v", o, entry.lo, entry.want) + } + }) } } @@ -1203,24 +1230,28 @@ func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions s *node.Node p *predicate.Predicate want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest predicate duplicate timestamp", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, s: testutil.MustBuildNodeFromStrings(t, "/u", "john"), p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, s: testutil.MustBuildNodeFromStrings(t, "/_", "bn"), p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), @@ -1229,28 +1260,30 @@ func TestTriplesForSubjectAndPredicateFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - s := entry.s - p := entry.p - if err := g.TriplesForSubjectAndPredicate(ctx, s, p, entry.lo, trpls); err != nil { - t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + s := entry.s + p := entry.p + if err := g.TriplesForSubjectAndPredicate(ctx, s, p, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) = %v; want nil", s, p, entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForSubjectAndPredicate(%s, %s, %s) retrieved unexpected %s", s, p, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForSubjectAndPredicate(%s, %s, %s) failed to retrieve some expected elements: %v", s, p, entry.lo, entry.want) + } + }) } } @@ -1310,24 +1343,28 @@ func TestTriplesForPredicateAndObjectFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions p *predicate.Predicate o *triple.Object want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2012-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2012-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest predicate duplicate timestamp", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, p: testutil.MustBuildPredicate(t, `"meet"@[2014-04-10T04:21:00Z]`), o: triple.NewNodeObject(testutil.MustBuildNodeFromStrings(t, "/u", "mary")), want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, p: testutil.MustBuildPredicate(t, `"_predicate"@[]`), o: triple.NewPredicateObject(testutil.MustBuildPredicate(t, `"meet"@[2020-04-10T04:21:00Z]`)), @@ -1336,28 +1373,30 @@ func TestTriplesForPredicateAndObjectFilter(t *testing.T) { } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - p := entry.p - o := entry.o - if err := g.TriplesForPredicateAndObject(ctx, p, o, entry.lo, trpls); err != nil { - t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + p := entry.p + o := entry.o + if err := g.TriplesForPredicateAndObject(ctx, p, o, entry.lo, trpls); err != nil { + t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) = %v; want nil", p, o, entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.TriplesForPredicateAndObject(%s, %s, %s) retrieved unexpected %s", p, o, entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.TriplesForPredicateAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.TriplesForPredicateAndObject(%s, %s, %s) failed to retrieve some expected elements: %v", p, o, entry.lo, entry.want) + } + }) } } @@ -1434,39 +1473,44 @@ func TestTriplesFilter(t *testing.T) { } testTable := []struct { + id string lo *storage.LookupOptions want map[string]int }{ { + id: "FILTER latest predicate", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.PredicateField}}, want: map[string]int{`/u "meet"@[2014-04-10T04:21:00Z] /u`: 1, `/u "meet"@[2014-04-10T04:21:00Z] /u`: 1}, }, { + id: "FILTER latest object", lo: &storage.LookupOptions{FilterOptions: &filter.StorageOptions{Operation: filter.Latest, Field: filter.ObjectField}}, want: map[string]int{`/_ "_predicate"@[] "meet"@[2021-04-10T04:21:00Z]`: 1}, }, } for _, entry := range testTable { - // To avoid blocking on the test we use a buffered channel of size 100. On a real - // usage of the driver you would like to call the graph operation on a separated - // goroutine using a sync.WaitGroup to collect the error code eventually. - trpls := make(chan *triple.Triple, 100) - if err := g.Triples(ctx, entry.lo, trpls); err != nil { - t.Fatalf("g.Triples(%s) = %v; want nil", entry.lo, err) - } - for trpl := range trpls { - tStr := trpl.String() - if _, ok := entry.want[tStr]; !ok { - t.Fatalf("g.Triples(%s) retrieved unexpected %s", entry.lo, tStr) + t.Run(entry.id, func(t *testing.T) { + // To avoid blocking on the test we use a buffered channel of size 100. On a real + // usage of the driver you would like to call the graph operation on a separated + // goroutine using a sync.WaitGroup to collect the error code eventually. + trpls := make(chan *triple.Triple, 100) + if err := g.Triples(ctx, entry.lo, trpls); err != nil { + t.Fatalf("g.Triples(%s) = %v; want nil", entry.lo, err) } - entry.want[tStr] = entry.want[tStr] - 1 - if entry.want[tStr] == 0 { - delete(entry.want, tStr) + for trpl := range trpls { + tStr := trpl.String() + if _, ok := entry.want[tStr]; !ok { + t.Fatalf("g.Triples(%s) retrieved unexpected %s", entry.lo, tStr) + } + entry.want[tStr] = entry.want[tStr] - 1 + if entry.want[tStr] == 0 { + delete(entry.want, tStr) + } } - } - if len(entry.want) != 0 { - t.Errorf("g.Triples(%s) failed to retrieve some expected elements: %v", entry.lo, entry.want) - } + if len(entry.want) != 0 { + t.Errorf("g.Triples(%s) failed to retrieve some expected elements: %v", entry.lo, entry.want) + } + }) } } From 5fd5a95de0f1e09ad85da88999cf103224edc35c Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Wed, 14 Oct 2020 14:09:47 -0300 Subject: [PATCH 25/26] Do not use naked returns inside "compatibleBindingsInClauseForFilterOperation" anymore --- bql/planner/planner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 277e0323..908d1b9a 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -665,13 +665,13 @@ func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (c filter.PredicateField: {cls.PBinding: true, cls.PAlias: true}, filter.ObjectField: {cls.OBinding: true, cls.OAlias: true}, } - return + return bindingsByField } default: - err = fmt.Errorf("filter function %q has no bindings in clause specified for it (planner level)", operation) + return nil, fmt.Errorf("filter function %q has no bindings in clause specified for it (planner level)", operation) } - return + return compatibleBindingsInClause, nil } // organizeFilterOptionsByClause processes all the given filters and organize them in a map that has as keys the From ea887b7122ae21b5bd8e4648084dff157b5d31de Mon Sep 17 00:00:00 2001 From: Roger Leite Lucena Date: Thu, 15 Oct 2020 11:21:17 -0300 Subject: [PATCH 26/26] Return "compatibleBindingsInClause" directly for each switch case inside "compatibleBindingsInClauseForFilterOperation" --- bql/planner/planner.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bql/planner/planner.go b/bql/planner/planner.go index 908d1b9a..0fb74240 100644 --- a/bql/planner/planner.go +++ b/bql/planner/planner.go @@ -667,11 +667,10 @@ func compatibleBindingsInClauseForFilterOperation(operation filter.Operation) (c } return bindingsByField } + return compatibleBindingsInClause, nil default: return nil, fmt.Errorf("filter function %q has no bindings in clause specified for it (planner level)", operation) } - - return compatibleBindingsInClause, nil } // organizeFilterOptionsByClause processes all the given filters and organize them in a map that has as keys the