From a70c4759f3da1b6b2a141eeb1f57ba10ab9bb704 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 19 Mar 2025 19:43:56 +0100 Subject: [PATCH 01/16] feat: aggregate over all objects in collection Supports integer aggregations. Also committing a bunch of group-by related code, although that's WIP. --- .../v1/collections/CollectionsITest.java | 55 ++++++++++++------- .../aggregate/AggregateGroupByResult.java | 7 +++ .../aggregate/AggregateResult.java | 27 +++++++++ 3 files changed, 69 insertions(+), 20 deletions(-) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java diff --git a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java b/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java index dd50d69e1..a214c6337 100644 --- a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java +++ b/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java @@ -17,25 +17,40 @@ public class CollectionsITest extends ConcurrentTest { @Test public void testCreateGetDelete() throws IOException { var collectionName = ns("Things_1"); - client.collections.create(collectionName, - col -> col - .properties(Property.text("username"), Property.integer("age")) - .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); - - var thingsCollection = client.collections.getConfig(collectionName); - - Assertions.assertThat(thingsCollection).get() - .hasFieldOrPropertyWithValue("name", collectionName) - .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) - .as("default vector").satisfies(defaultVector -> { - Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) - .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); - Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) - .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); - }); - - client.collections.delete(collectionName); - var noCollection = client.collections.getConfig(collectionName); - Assertions.assertThat(noCollection).as("after delete").isEmpty(); + +// -------------------------------------------- +var defaultIndex = new VectorIndex<>(Vectorizer.none()); +var hnswIndex = new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()); +// -------------------------------------------- + +client.collections.create(collectionName, + collection -> collection + .properties(Property.text("username"), Property.integer("age")) + .vector(defaultIndex) + .vector("only-one", hnswIndex) + .vectors(named -> named + .vector("vector-a", hnswIndex) + .vector("vector-b", hnswIndex))); + +// -------------------------------------------- +var thingsCollection = client.collections.getConfig(collectionName); +// -------------------------------------------- + +Assertions.assertThat(thingsCollection).get() + .hasFieldOrPropertyWithValue("name", collectionName) + .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) + .as("default vector").satisfies(defaultVector -> { + Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) + .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); + Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) + .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); + }); + +// -------------------------------------------- +client.collections.delete(collectionName); +// -------------------------------------------- + +var noCollection = client.collections.getConfig(collectionName); +Assertions.assertThat(noCollection).as("after delete").isEmpty(); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java new file mode 100644 index 000000000..4eb30712b --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java @@ -0,0 +1,7 @@ +package io.weaviate.client6.v1.collections.aggregate; + +import java.util.List; + +public record AggregateGroupByResult(List> groups) { + +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java new file mode 100644 index 000000000..99ceae35e --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java @@ -0,0 +1,27 @@ +package io.weaviate.client6.v1.collections.aggregate; + +import java.util.Map; + +public record AggregateResult(Map properties, Long totalCount) { + public boolean isTextProperties(String name) { + return properties.get(name) instanceof TextMetric.Values; + } + + public boolean isIntegerProperty(String name) { + return properties.get(name) instanceof IntegerMetric.Values; + } + + public TextMetric.Values getText(String name) { + if (!isTextProperties(name)) { + throw new IllegalStateException(name + " is not a Text property"); + } + return (TextMetric.Values) this.properties.get(name); + } + + public IntegerMetric.Values getInteger(String name) { + if (!isIntegerProperty(name)) { + throw new IllegalStateException(name + " is not a Integer property"); + } + return (IntegerMetric.Values) this.properties.get(name); + } +} From d1f891700fc8b259b4629fa4fb1c6e6a5634312a Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 19 Mar 2025 20:48:58 +0100 Subject: [PATCH 02/16] chore: fix tests and import paths --- .../v1/collections/CollectionsITest.java | 55 +++++++------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java b/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java index a214c6337..dd50d69e1 100644 --- a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java +++ b/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java @@ -17,40 +17,25 @@ public class CollectionsITest extends ConcurrentTest { @Test public void testCreateGetDelete() throws IOException { var collectionName = ns("Things_1"); - -// -------------------------------------------- -var defaultIndex = new VectorIndex<>(Vectorizer.none()); -var hnswIndex = new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()); -// -------------------------------------------- - -client.collections.create(collectionName, - collection -> collection - .properties(Property.text("username"), Property.integer("age")) - .vector(defaultIndex) - .vector("only-one", hnswIndex) - .vectors(named -> named - .vector("vector-a", hnswIndex) - .vector("vector-b", hnswIndex))); - -// -------------------------------------------- -var thingsCollection = client.collections.getConfig(collectionName); -// -------------------------------------------- - -Assertions.assertThat(thingsCollection).get() - .hasFieldOrPropertyWithValue("name", collectionName) - .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) - .as("default vector").satisfies(defaultVector -> { - Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) - .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); - Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) - .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); - }); - -// -------------------------------------------- -client.collections.delete(collectionName); -// -------------------------------------------- - -var noCollection = client.collections.getConfig(collectionName); -Assertions.assertThat(noCollection).as("after delete").isEmpty(); + client.collections.create(collectionName, + col -> col + .properties(Property.text("username"), Property.integer("age")) + .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); + + var thingsCollection = client.collections.getConfig(collectionName); + + Assertions.assertThat(thingsCollection).get() + .hasFieldOrPropertyWithValue("name", collectionName) + .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) + .as("default vector").satisfies(defaultVector -> { + Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) + .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); + Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) + .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); + }); + + client.collections.delete(collectionName); + var noCollection = client.collections.getConfig(collectionName); + Assertions.assertThat(noCollection).as("after delete").isEmpty(); } } From 7e9dae5956acbcf985492e318daaae0037fe5528 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Mon, 24 Mar 2025 20:08:38 +0100 Subject: [PATCH 03/16] feat: aggregate with near vector filter --- .../aggregate/AggregateGroupByResult.java | 7 ----- .../aggregate/AggregateResult.java | 27 ------------------- 2 files changed, 34 deletions(-) delete mode 100644 src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java delete mode 100644 src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java deleted file mode 100644 index 4eb30712b..000000000 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateGroupByResult.java +++ /dev/null @@ -1,7 +0,0 @@ -package io.weaviate.client6.v1.collections.aggregate; - -import java.util.List; - -public record AggregateGroupByResult(List> groups) { - -} diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java deleted file mode 100644 index 99ceae35e..000000000 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateResult.java +++ /dev/null @@ -1,27 +0,0 @@ -package io.weaviate.client6.v1.collections.aggregate; - -import java.util.Map; - -public record AggregateResult(Map properties, Long totalCount) { - public boolean isTextProperties(String name) { - return properties.get(name) instanceof TextMetric.Values; - } - - public boolean isIntegerProperty(String name) { - return properties.get(name) instanceof IntegerMetric.Values; - } - - public TextMetric.Values getText(String name) { - if (!isTextProperties(name)) { - throw new IllegalStateException(name + " is not a Text property"); - } - return (TextMetric.Values) this.properties.get(name); - } - - public IntegerMetric.Values getInteger(String name) { - if (!isIntegerProperty(name)) { - throw new IllegalStateException(name + " is not a Integer property"); - } - return (IntegerMetric.Values) this.properties.get(name); - } -} From c2bba899709ebe5bd6fc555ae426cb21511852a6 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 25 Mar 2025 12:24:21 +0100 Subject: [PATCH 04/16] feat: add aggregation + near vector + groupBy --- .../aggregate/WeaviateAggregate.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java index 73474fb7b..28edb8df8 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java @@ -86,4 +86,65 @@ public AggregateGroupByResponse nearVector( var reply = grpcClient.grpc.aggregate(req); return new AggregateUnmarshaler(reply).grouped(); } + + public AggregateGroupByResponse nearVector( + Float[] vector, + Consumer nearVectorOptions, + Consumer groupByOptions, + Consumer options) { + var aggregation = AggregateRequest.with(collectionName, options); + var nearVector = NearVector.with(vector, nearVectorOptions); + var groupBy = AggregateGroupByRequest.GroupBy.with(groupByOptions); + + var req = new AggregateMarshaler(aggregation.collectionName()) + .addAggregation(aggregation) + .addGroupBy(groupBy) + .addNearVector(nearVector) + .marshal(); + var reply = grpcClient.grpc.aggregate(req); + + List> groups = new ArrayList<>(); + if (reply.hasGroupedResults()) { + for (var result : reply.getGroupedResults().getGroupsList()) { + final Long totalCount = result.hasObjectsCount() ? result.getObjectsCount() : null; + + GroupedBy groupedBy = null; + var gb = result.getGroupedBy(); + if (gb.hasInt()) { + groupedBy = new GroupedBy(gb.getPathList().get(0), gb.getInt()); + } else if (gb.hasText()) { + groupedBy = new GroupedBy(gb.getPathList().get(0), gb.getText()); + } else { + assert false : "branch not covered"; + } + + Map properties = new HashMap<>(); + for (var agg : result.getAggregations().getAggregationsList()) { + var property = agg.getProperty(); + Metric.Values value = null; + + if (agg.hasInt()) { + var metrics = agg.getInt(); + value = new IntegerMetric.Values( + metrics.hasCount() ? metrics.getCount() : null, + metrics.hasMinimum() ? metrics.getMinimum() : null, + metrics.hasMaximum() ? metrics.getMaximum() : null, + metrics.hasMean() ? metrics.getMean() : null, + metrics.hasMedian() ? metrics.getMedian() : null, + metrics.hasMode() ? metrics.getMode() : null, + metrics.hasSum() ? metrics.getSum() : null); + } else { + assert false : "branch not covered"; + } + if (value != null) { + properties.put(property, value); + } + } + Group group = new Group<>(groupedBy, properties, totalCount); + groups.add(group); + + } + } + return new AggregateGroupByResponse(groups); + } } From e76a20d42e5e1be0ff63a32b51b875f89beea5fa Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 25 Mar 2025 14:28:27 +0100 Subject: [PATCH 05/16] refactor: move unmarshaling code to internal/codec --- .../aggregate/WeaviateAggregate.java | 65 ++++++------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java index 28edb8df8..e018be2fa 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java @@ -87,14 +87,30 @@ public AggregateGroupByResponse nearVector( return new AggregateUnmarshaler(reply).grouped(); } + public AggregateGroupByResponse nearVector( + Float[] vector, + AggregateGroupByRequest.GroupBy groupBy, + Consumer options) { + var aggregation = AggregateRequest.with(collectionName, options); + var nearVector = NearVector.with(vector, opt -> { + }); + + var req = new AggregateMarshaler(aggregation.collectionName()) + .addAggregation(aggregation) + .addGroupBy(groupBy) + .addNearVector(nearVector) + .marshal(); + var reply = grpcClient.grpc.aggregate(req); + return new AggregateUnmarshaler(reply).grouped(); + } + public AggregateGroupByResponse nearVector( Float[] vector, Consumer nearVectorOptions, - Consumer groupByOptions, + AggregateGroupByRequest.GroupBy groupBy, Consumer options) { var aggregation = AggregateRequest.with(collectionName, options); var nearVector = NearVector.with(vector, nearVectorOptions); - var groupBy = AggregateGroupByRequest.GroupBy.with(groupByOptions); var req = new AggregateMarshaler(aggregation.collectionName()) .addAggregation(aggregation) @@ -102,49 +118,6 @@ public AggregateGroupByResponse nearVector( .addNearVector(nearVector) .marshal(); var reply = grpcClient.grpc.aggregate(req); - - List> groups = new ArrayList<>(); - if (reply.hasGroupedResults()) { - for (var result : reply.getGroupedResults().getGroupsList()) { - final Long totalCount = result.hasObjectsCount() ? result.getObjectsCount() : null; - - GroupedBy groupedBy = null; - var gb = result.getGroupedBy(); - if (gb.hasInt()) { - groupedBy = new GroupedBy(gb.getPathList().get(0), gb.getInt()); - } else if (gb.hasText()) { - groupedBy = new GroupedBy(gb.getPathList().get(0), gb.getText()); - } else { - assert false : "branch not covered"; - } - - Map properties = new HashMap<>(); - for (var agg : result.getAggregations().getAggregationsList()) { - var property = agg.getProperty(); - Metric.Values value = null; - - if (agg.hasInt()) { - var metrics = agg.getInt(); - value = new IntegerMetric.Values( - metrics.hasCount() ? metrics.getCount() : null, - metrics.hasMinimum() ? metrics.getMinimum() : null, - metrics.hasMaximum() ? metrics.getMaximum() : null, - metrics.hasMean() ? metrics.getMean() : null, - metrics.hasMedian() ? metrics.getMedian() : null, - metrics.hasMode() ? metrics.getMode() : null, - metrics.hasSum() ? metrics.getSum() : null); - } else { - assert false : "branch not covered"; - } - if (value != null) { - properties.put(property, value); - } - } - Group group = new Group<>(groupedBy, properties, totalCount); - groups.add(group); - - } - } - return new AggregateGroupByResponse(groups); + return new AggregateUnmarshaler(reply).grouped(); } } From 0448e4ff21a616fce26e3d0d58ecf2d83233c19a Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 26 Mar 2025 14:08:56 +0100 Subject: [PATCH 06/16] feat: create reference properties + add references to a collection --- .../v1/collections/CollectionsITest.java | 41 --------- .../integration/CollectionsITest.java | 91 +++++++++++++++++++ .../io/weaviate/client6/v1/Collection.java | 3 + .../client6/v1/collections/Collections.java | 5 + .../client6/v1/collections/DataType.java | 4 +- .../client6/v1/collections/Property.java | 21 ++--- .../collections/WeaviateCollectionConfig.java | 68 ++++++++++++++ 7 files changed, 176 insertions(+), 57 deletions(-) delete mode 100644 src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java create mode 100644 src/it/java/io/weaviate/integration/CollectionsITest.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java diff --git a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java b/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java deleted file mode 100644 index dd50d69e1..000000000 --- a/src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java +++ /dev/null @@ -1,41 +0,0 @@ -package io.weaviate.client6.v1.collections; - -import java.io.IOException; - -import org.assertj.core.api.Assertions; -import org.junit.Test; - -import io.weaviate.ConcurrentTest; -import io.weaviate.client6.WeaviateClient; -import io.weaviate.client6.v1.collections.VectorIndex.IndexType; -import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy; -import io.weaviate.containers.Container; - -public class CollectionsITest extends ConcurrentTest { - private static WeaviateClient client = Container.WEAVIATE.getClient(); - - @Test - public void testCreateGetDelete() throws IOException { - var collectionName = ns("Things_1"); - client.collections.create(collectionName, - col -> col - .properties(Property.text("username"), Property.integer("age")) - .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); - - var thingsCollection = client.collections.getConfig(collectionName); - - Assertions.assertThat(thingsCollection).get() - .hasFieldOrPropertyWithValue("name", collectionName) - .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) - .as("default vector").satisfies(defaultVector -> { - Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) - .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); - Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) - .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); - }); - - client.collections.delete(collectionName); - var noCollection = client.collections.getConfig(collectionName); - Assertions.assertThat(noCollection).as("after delete").isEmpty(); - } -} diff --git a/src/it/java/io/weaviate/integration/CollectionsITest.java b/src/it/java/io/weaviate/integration/CollectionsITest.java new file mode 100644 index 000000000..54b6c2467 --- /dev/null +++ b/src/it/java/io/weaviate/integration/CollectionsITest.java @@ -0,0 +1,91 @@ +package io.weaviate.integration; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.Test; + +import io.weaviate.ConcurrentTest; +import io.weaviate.client6.WeaviateClient; +import io.weaviate.client6.v1.collections.CollectionDefinition; +import io.weaviate.client6.v1.collections.NoneVectorizer; +import io.weaviate.client6.v1.collections.Property; +import io.weaviate.client6.v1.collections.VectorIndex; +import io.weaviate.client6.v1.collections.VectorIndex.IndexType; +import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy; +import io.weaviate.client6.v1.collections.Vectorizer; +import io.weaviate.client6.v1.collections.Vectors; +import io.weaviate.containers.Container; + +public class CollectionsITest extends ConcurrentTest { + private static WeaviateClient client = Container.WEAVIATE.getClient(); + + @Test + public void testCreateGetDelete() throws IOException { + var collectionName = ns("Things"); + client.collections.create(collectionName, + col -> col + .properties(Property.text("username"), Property.integer("age")) + .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); + + var thingsCollection = client.collections.getConfig(collectionName); + + Assertions.assertThat(thingsCollection).get() + .hasFieldOrPropertyWithValue("name", collectionName) + .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) + .as("default vector").satisfies(defaultVector -> { + Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) + .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); + Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) + .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); + }); + + client.collections.delete(collectionName); + var noCollection = client.collections.getConfig(collectionName); + Assertions.assertThat(noCollection).as("after delete").isEmpty(); + } + + @Test + public void testCrossReferences() throws IOException { + // Arrange: Create Owners collection + var nsOwners = ns("Owners"); + client.collections.create(nsOwners); + + // Act: Create Things collection with owner -> owners + var nsThings = ns("Things"); + client.collections.create(nsThings, + col -> col.properties(Property.reference("ownedBy", nsOwners))); + var things = client.collections.use(nsThings); + + // Assert: Things --ownedBy-> Owners + Assertions.assertThat(things.config.get()) + .as("after create Things").get() + .satisfies(c -> { + Assertions.assertThat(c.properties()) + .as("ownedBy").filteredOn(p -> p.name.equals("ownedBy")).first() + .extracting(p -> p.dataTypes).asInstanceOf(InstanceOfAssertFactories.LIST) + .containsOnly(nsOwners); + }); + + // Arrange: Create OnlineStores and Markets collections + var nsOnlineStores = ns("OnlineStores"); + client.collections.create(nsOnlineStores); + + var nsMarkets = ns("Markets"); + client.collections.create(nsMarkets); + + // Act: Update Things collections to add polymorphic reference + things.config.addProperty(Property.reference("soldIn", nsOnlineStores, nsMarkets)); + + // Assert: Things --soldIn-> [OnlineStores, Markets] + Assertions.assertThat(things.config.get()) + .as("after add property").get() + .satisfies(c -> { + Assertions.assertThat(c.properties()) + .as("soldIn").filteredOn(p -> p.name.equals("soldIn")).first() + .extracting(p -> p.dataTypes).asInstanceOf(InstanceOfAssertFactories.LIST) + .containsOnly(nsOnlineStores, nsMarkets); + }); + } +} diff --git a/src/main/java/io/weaviate/client6/v1/Collection.java b/src/main/java/io/weaviate/client6/v1/Collection.java index b1f40dcc4..880114851 100644 --- a/src/main/java/io/weaviate/client6/v1/Collection.java +++ b/src/main/java/io/weaviate/client6/v1/Collection.java @@ -3,6 +3,7 @@ import io.weaviate.client6.Config; import io.weaviate.client6.internal.GrpcClient; import io.weaviate.client6.internal.HttpClient; +import io.weaviate.client6.v1.collections.WeaviateCollectionConfig; import io.weaviate.client6.v1.collections.aggregate.WeaviateAggregate; import io.weaviate.client6.v1.data.Data; import io.weaviate.client6.v1.query.Query; @@ -10,11 +11,13 @@ public class Collection { public final Query query; public final Data data; + public final WeaviateCollectionConfig config; public final WeaviateAggregate aggregate; public Collection(String collectionName, Config config, GrpcClient grpc, HttpClient http) { this.query = new Query<>(collectionName, grpc); this.data = new Data<>(collectionName, config, http); + this.config = new WeaviateCollectionConfig(collectionName, config, http); this.aggregate = new WeaviateAggregate(collectionName, grpc); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/Collections.java b/src/main/java/io/weaviate/client6/v1/collections/Collections.java index 4f65915c9..abef2929d 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/Collections.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Collections.java @@ -112,6 +112,11 @@ public Vectors read(JsonReader in) throws IOException { }) .create(); + public void create(String name) throws IOException { + create(name, opt -> { + }); + } + public void create(String name, Consumer options) throws IOException { var collection = CollectionDefinition.with(name, options); diff --git a/src/main/java/io/weaviate/client6/v1/collections/DataType.java b/src/main/java/io/weaviate/client6/v1/collections/DataType.java index 8ec96470f..4bb42b525 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/DataType.java +++ b/src/main/java/io/weaviate/client6/v1/collections/DataType.java @@ -6,5 +6,7 @@ public enum DataType { @SerializedName("text") TEXT, @SerializedName("int") - INT; + INT, + @SerializedName("reference") + REFERENCE; } diff --git a/src/main/java/io/weaviate/client6/v1/collections/Property.java b/src/main/java/io/weaviate/client6/v1/collections/Property.java index 81d23a5df..8d5d53db1 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/Property.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Property.java @@ -2,7 +2,6 @@ import java.util.Arrays; import java.util.List; -import java.util.function.Consumer; import com.google.gson.annotations.SerializedName; @@ -11,7 +10,7 @@ public class Property { public final String name; @SerializedName("dataType") - public final List dataTypes; + public final List dataTypes; /** Add text property with default configuration. */ public static Property text(String name) { @@ -23,25 +22,17 @@ public static Property integer(String name) { return new Property(name, DataType.INT); } - public static final class Configuration { - private List dataTypes; - - public Configuration dataTypes(DataType... types) { - this.dataTypes = Arrays.asList(types); - return this; - } + public static Property reference(String name, String... collections) { + return new Property(name, collections); } private Property(String name, DataType type) { this.name = name; - this.dataTypes = List.of(type); + this.dataTypes = List.of(type.name().toLowerCase()); } - public Property(String name, Consumer options) { - var config = new Configuration(); - options.accept(config); - + private Property(String name, String... collections) { this.name = name; - this.dataTypes = config.dataTypes; + this.dataTypes = Arrays.asList(collections); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java b/src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java new file mode 100644 index 000000000..6ca116ec9 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java @@ -0,0 +1,68 @@ +package io.weaviate.client6.v1.collections; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Optional; + +import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; + +import com.google.gson.Gson; + +import io.weaviate.client6.Config; +import io.weaviate.client6.internal.DtoTypeAdapterFactory; +import io.weaviate.client6.internal.HttpClient; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class WeaviateCollectionConfig { + // TODO: hide befind an internal HttpClient + private final String collectionName; + private final Config config; + private final HttpClient httpClient; + + private static final Gson gson = new Gson(); + static { + DtoTypeAdapterFactory.register( + CollectionDefinition.class, + CollectionDefinitionDTO.class, + m -> { + return new CollectionDefinitionDTO(m); + }); + } + + public Optional get() throws IOException { + ClassicHttpRequest httpGet = ClassicRequestBuilder + .get(config.baseUrl() + "/schema/" + collectionName) + .build(); + + return httpClient.http.execute(httpGet, response -> { + if (response.getCode() == HttpStatus.SC_NOT_FOUND) { + return Optional.empty(); + } + try (var r = new InputStreamReader(response.getEntity().getContent())) { + var collection = gson.fromJson(r, CollectionDefinition.class); + return Optional.ofNullable(collection); + } + }); + } + + public void addProperty(Property property) throws IOException { + ClassicHttpRequest httpPost = ClassicRequestBuilder + .post(config.baseUrl() + "/schema/" + collectionName + "/properties") + .setEntity(gson.toJson(property), ContentType.APPLICATION_JSON) + .build(); + + httpClient.http.execute(httpPost, response -> { + var entity = response.getEntity(); + if (response.getCode() != HttpStatus.SC_SUCCESS) { + var message = EntityUtils.toString(entity); + throw new RuntimeException("HTTP " + response.getCode() + ": " + message); + } + return null; + }); + } +} From 2aa3138612e186d98e3ce9322b9ba0979f338aec Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 23 Apr 2025 22:25:59 +0200 Subject: [PATCH 07/16] feat: create collections with reference properties --- .../integration/AggregationITest.java | 6 +- .../integration/CollectionsITest.java | 12 +- .../v1 => integration}/DataITest.java | 32 ++-- .../integration/NearVectorQueryITest.java | 2 +- .../weaviate/integration/ReferencesITest.java | 95 +++++++++++ .../io/weaviate/client6/WeaviateClient.java | 6 +- .../io/weaviate/client6/v1/Collection.java | 23 --- .../weaviate/client6/v1/ObjectMetadata.java | 55 ------- .../v1/collections/AtomicDataType.java | 15 ++ ...lectionDefinition.java => Collection.java} | 6 +- .../v1/collections/CollectionClient.java | 22 +++ ...onfig.java => CollectionConfigClient.java} | 9 +- .../collections/CollectionDefinitionDTO.java | 8 +- ...ollections.java => CollectionsClient.java} | 18 +-- .../client6/v1/collections/DataType.java | 12 -- .../client6/v1/collections/Property.java | 25 ++- .../client6/v1/collections/Reference.java | 59 +++++++ ...ateAggregate.java => AggregateClient.java} | 4 +- .../data/ConsistencyLevel.java | 2 +- .../data/DataClient.java} | 19 ++- .../{ => collections}/data/GetParameters.java | 2 +- .../collections/data/InsertObjectRequest.java | 149 ++++++++++++++++++ .../data/QueryParameters.java | 2 +- .../v1/collections/object/ObjectMetadata.java | 30 ++++ .../v1/{ => collections/object}/Vectors.java | 19 ++- .../object}/WeaviateObject.java | 13 +- .../object}/WeaviateObjectDTO.java | 7 +- .../v1/query/{Query.java => QueryClient.java} | 4 +- .../client6/v1/ObjectMetadataTest.java | 15 +- .../client6/v1/collections/VectorsTest.java | 8 +- .../data/QueryParametersTest.java | 2 +- 31 files changed, 488 insertions(+), 193 deletions(-) rename src/it/java/io/weaviate/{client6/v1 => integration}/DataITest.java (67%) create mode 100644 src/it/java/io/weaviate/integration/ReferencesITest.java delete mode 100644 src/main/java/io/weaviate/client6/v1/Collection.java delete mode 100644 src/main/java/io/weaviate/client6/v1/ObjectMetadata.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/AtomicDataType.java rename src/main/java/io/weaviate/client6/v1/collections/{CollectionDefinition.java => Collection.java} (82%) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java rename src/main/java/io/weaviate/client6/v1/collections/{WeaviateCollectionConfig.java => CollectionConfigClient.java} (90%) rename src/main/java/io/weaviate/client6/v1/collections/{Collections.java => CollectionsClient.java} (90%) delete mode 100644 src/main/java/io/weaviate/client6/v1/collections/DataType.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/Reference.java rename src/main/java/io/weaviate/client6/v1/collections/aggregate/{WeaviateAggregate.java => AggregateClient.java} (97%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/data/ConsistencyLevel.java (51%) rename src/main/java/io/weaviate/client6/v1/{data/Data.java => collections/data/DataClient.java} (82%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/data/GetParameters.java (97%) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java rename src/main/java/io/weaviate/client6/v1/{ => collections}/data/QueryParameters.java (95%) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java rename src/main/java/io/weaviate/client6/v1/{ => collections/object}/Vectors.java (88%) rename src/main/java/io/weaviate/client6/v1/{data => collections/object}/WeaviateObject.java (69%) rename src/main/java/io/weaviate/client6/v1/{data => collections/object}/WeaviateObjectDTO.java (86%) rename src/main/java/io/weaviate/client6/v1/query/{Query.java => QueryClient.java} (98%) rename src/test/java/io/weaviate/client6/v1/{ => collections}/data/QueryParametersTest.java (96%) diff --git a/src/it/java/io/weaviate/integration/AggregationITest.java b/src/it/java/io/weaviate/integration/AggregationITest.java index bd54ed865..035e26b8b 100644 --- a/src/it/java/io/weaviate/integration/AggregationITest.java +++ b/src/it/java/io/weaviate/integration/AggregationITest.java @@ -16,13 +16,13 @@ import io.weaviate.client6.v1.collections.Property; import io.weaviate.client6.v1.collections.VectorIndex; import io.weaviate.client6.v1.collections.Vectorizer; -import io.weaviate.client6.v1.collections.Vectors; import io.weaviate.client6.v1.collections.aggregate.AggregateGroupByRequest.GroupBy; import io.weaviate.client6.v1.collections.aggregate.AggregateGroupByResponse; import io.weaviate.client6.v1.collections.aggregate.Group; import io.weaviate.client6.v1.collections.aggregate.GroupedBy; import io.weaviate.client6.v1.collections.aggregate.IntegerMetric; import io.weaviate.client6.v1.collections.aggregate.Metric; +import io.weaviate.client6.v1.collections.object.Vectors; import io.weaviate.containers.Container; public class AggregationITest extends ConcurrentTest { @@ -36,7 +36,7 @@ public static void beforeAll() throws IOException { .properties( Property.text("category"), Property.integer("price")) - .vectors(Vectors.of(new VectorIndex<>(Vectorizer.none())))); + .vectors(io.weaviate.client6.v1.collections.Vectors.of(new VectorIndex<>(Vectorizer.none())))); var things = client.collections.use(COLLECTION); for (var category : List.of("Shoes", "Hat", "Jacket")) { @@ -47,7 +47,7 @@ public static void beforeAll() throws IOException { things.data.insert(Map.of( "category", category, "price", category.length()), - meta -> meta.vectors(vector)); + meta -> meta.vectors(Vectors.of(vector))); } } } diff --git a/src/it/java/io/weaviate/integration/CollectionsITest.java b/src/it/java/io/weaviate/integration/CollectionsITest.java index 54b6c2467..759e0908c 100644 --- a/src/it/java/io/weaviate/integration/CollectionsITest.java +++ b/src/it/java/io/weaviate/integration/CollectionsITest.java @@ -8,7 +8,7 @@ import io.weaviate.ConcurrentTest; import io.weaviate.client6.WeaviateClient; -import io.weaviate.client6.v1.collections.CollectionDefinition; +import io.weaviate.client6.v1.collections.Collection; import io.weaviate.client6.v1.collections.NoneVectorizer; import io.weaviate.client6.v1.collections.Property; import io.weaviate.client6.v1.collections.VectorIndex; @@ -33,7 +33,7 @@ public void testCreateGetDelete() throws IOException { Assertions.assertThat(thingsCollection).get() .hasFieldOrPropertyWithValue("name", collectionName) - .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault) + .extracting(Collection::vectors).extracting(Vectors::getDefault) .as("default vector").satisfies(defaultVector -> { Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); @@ -63,8 +63,8 @@ public void testCrossReferences() throws IOException { .as("after create Things").get() .satisfies(c -> { Assertions.assertThat(c.properties()) - .as("ownedBy").filteredOn(p -> p.name.equals("ownedBy")).first() - .extracting(p -> p.dataTypes).asInstanceOf(InstanceOfAssertFactories.LIST) + .as("ownedBy").filteredOn(p -> p.name().equals("ownedBy")).first() + .extracting(p -> p.dataTypes()).asInstanceOf(InstanceOfAssertFactories.LIST) .containsOnly(nsOwners); }); @@ -83,8 +83,8 @@ public void testCrossReferences() throws IOException { .as("after add property").get() .satisfies(c -> { Assertions.assertThat(c.properties()) - .as("soldIn").filteredOn(p -> p.name.equals("soldIn")).first() - .extracting(p -> p.dataTypes).asInstanceOf(InstanceOfAssertFactories.LIST) + .as("soldIn").filteredOn(p -> p.name().equals("soldIn")).first() + .extracting(p -> p.dataTypes()).asInstanceOf(InstanceOfAssertFactories.LIST) .containsOnly(nsOnlineStores, nsMarkets); }); } diff --git a/src/it/java/io/weaviate/client6/v1/DataITest.java b/src/it/java/io/weaviate/integration/DataITest.java similarity index 67% rename from src/it/java/io/weaviate/client6/v1/DataITest.java rename to src/it/java/io/weaviate/integration/DataITest.java index f64702f1e..622a5030a 100644 --- a/src/it/java/io/weaviate/client6/v1/DataITest.java +++ b/src/it/java/io/weaviate/integration/DataITest.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1; +package io.weaviate.integration; import java.io.IOException; import java.util.Map; @@ -14,30 +14,31 @@ import io.weaviate.client6.v1.collections.VectorIndex; import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy; import io.weaviate.client6.v1.collections.Vectorizer; +import io.weaviate.client6.v1.collections.object.Vectors; import io.weaviate.containers.Container; public class DataITest extends ConcurrentTest { private static WeaviateClient client = Container.WEAVIATE.getClient(); - private static final String COLLECTION = unique("Things"); + private static final String COLLECTION = unique("Artists"); private static final String VECTOR_INDEX = "bring_your_own"; @BeforeClass public static void beforeAll() throws IOException { - createTestCollection(); + createTestCollections(); } @Test public void testCreateGetDelete() throws IOException { - var things = client.collections.use(COLLECTION); + var artists = client.collections.use(COLLECTION); var id = randomUUID(); Float[] vector = { 1f, 2f, 3f }; - things.data.insert(Map.of("username", "john doe"), metadata -> metadata + artists.data.insert(Map.of("name", "john doe"), metadata -> metadata .id(id) .vectors(Vectors.of(VECTOR_INDEX, vector))); - var object = things.data.get(id, query -> query.withVector()); + var object = artists.data.get(id, query -> query.withVector()); Assertions.assertThat(object) .as("object exists after insert").get() .satisfies(obj -> { @@ -50,18 +51,27 @@ public void testCreateGetDelete() throws IOException { Assertions.assertThat(obj.properties()) .as("has expected properties") - .containsEntry("username", "john doe"); + .containsEntry("name", "john doe"); }); - things.data.delete(id); - object = things.data.get(id); + artists.data.delete(id); + object = artists.data.get(id); Assertions.assertThat(object).isEmpty().as("object not exists after deletion"); } - private static void createTestCollection() throws IOException { + private static void createTestCollections() throws IOException { + var awardsGrammy = unique("Grammy"); + client.collections.create(awardsGrammy); + + var awardsOscar = unique("Oscar"); + client.collections.create(awardsOscar); + client.collections.create(COLLECTION, col -> col - .properties(Property.text("username"), Property.integer("age")) + .properties( + Property.text("name"), + Property.integer("age"), + Property.reference("hasAwards", awardsGrammy, awardsOscar)) .vector(VECTOR_INDEX, new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); } } diff --git a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java b/src/it/java/io/weaviate/integration/NearVectorQueryITest.java index 66258810d..1575712dc 100644 --- a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java +++ b/src/it/java/io/weaviate/integration/NearVectorQueryITest.java @@ -13,11 +13,11 @@ import io.weaviate.ConcurrentTest; import io.weaviate.client6.WeaviateClient; -import io.weaviate.client6.v1.Vectors; import io.weaviate.client6.v1.collections.Property; import io.weaviate.client6.v1.collections.VectorIndex; import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy; import io.weaviate.client6.v1.collections.Vectorizer; +import io.weaviate.client6.v1.collections.object.Vectors; import io.weaviate.client6.v1.query.GroupedQueryResult; import io.weaviate.client6.v1.query.MetadataField; import io.weaviate.client6.v1.query.NearVector; diff --git a/src/it/java/io/weaviate/integration/ReferencesITest.java b/src/it/java/io/weaviate/integration/ReferencesITest.java new file mode 100644 index 000000000..b2667115b --- /dev/null +++ b/src/it/java/io/weaviate/integration/ReferencesITest.java @@ -0,0 +1,95 @@ +package io.weaviate.integration; + +import java.io.IOException; +import java.util.Map; +import java.util.Optional; + +import org.assertj.core.api.Assertions; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.Test; + +import io.weaviate.ConcurrentTest; +import io.weaviate.client6.WeaviateClient; +import io.weaviate.client6.v1.collections.Collection; +import io.weaviate.client6.v1.collections.Property; +import io.weaviate.client6.v1.collections.Reference; +import io.weaviate.containers.Container; + +/** + * Scenarios related to reference properties: + *
    + *
  • create collection with (nested) reference properties
    • + *
  • insert objects with (nested) references
    • + *
  • add (nested) references
    • + *
  • search by reference (nested) properties
    • + *
+ */ +public class ReferencesITest extends ConcurrentTest { + private static final WeaviateClient client = Container.WEAVIATE.getClient(); + + @Test + public void testReferences() throws IOException { + // Arrange: create collection with cross-references + var nsArtists = ns("Artists"); + var nsGrammy = ns("Grammy"); + var nsOscar = ns("Oscar"); + + client.collections.create(nsOscar); + client.collections.create(nsGrammy); + + // Act: create Artists collection with hasAwards reference + client.collections.create(nsArtists, + col -> col + .properties( + Property.text("name"), + Property.integer("age"), + Property.reference("hasAwards", nsGrammy, nsOscar))); + + var artists = client.collections.use(nsArtists); + var grammies = client.collections.use(nsGrammy); + var oscars = client.collections.use(nsOscar); + + // Act: check collection configuration is correct + var collectionArtists = artists.config.get(); + Assertions.assertThat(collectionArtists).get() + .as("Artists: create collection") + .extracting(Collection::properties) + .extracting(properties -> properties.stream().filter(Property::isReference).findFirst()) + .extracting(Optional::get) + .returns("hasAwards", Property::name) + .extracting(Property::dataTypes) + .asInstanceOf(InstanceOfAssertFactories.list(String.class)) + .containsOnly(nsGrammy, nsOscar); + + // Act: insert some data + var grammy_1 = grammies.data.insert(Map.of()); + var grammy_2 = grammies.data.insert(Map.of()); + var oscar_1 = oscars.data.insert(Map.of()); + var oscar_2 = oscars.data.insert(Map.of()); + + var alex = artists.data.insert( + Map.of("name", "Alex"), + opt -> opt + .reference("hasAwards", Reference.uuids( + grammy_1.metadata().id(), oscar_1.metadata().id())) + .reference("hasAwards", Reference.objects(grammy_2, oscar_2))); + + // Act: add one more reference + var nsMovies = ns("Movies"); + client.collections.create(nsMovies); + artists.config.addProperty(Property.reference("featuredIn", nsMovies)); + + collectionArtists = artists.config.get(); + Assertions.assertThat(collectionArtists).get() + .as("Artists: add reference to Movies") + .extracting(Collection::properties) + .extracting(properties -> properties.stream() + .filter(property -> property.name().equals("featuredIn")) + .findFirst()) + .extracting(Optional::get) + .returns(true, Property::isReference) + .extracting(Property::dataTypes) + .asInstanceOf(InstanceOfAssertFactories.list(String.class)) + .containsOnly(nsMovies); + } +} diff --git a/src/main/java/io/weaviate/client6/WeaviateClient.java b/src/main/java/io/weaviate/client6/WeaviateClient.java index 8dba725a5..724fc65e9 100644 --- a/src/main/java/io/weaviate/client6/WeaviateClient.java +++ b/src/main/java/io/weaviate/client6/WeaviateClient.java @@ -5,18 +5,18 @@ import io.weaviate.client6.internal.GrpcClient; import io.weaviate.client6.internal.HttpClient; -import io.weaviate.client6.v1.collections.Collections; +import io.weaviate.client6.v1.collections.CollectionsClient; public class WeaviateClient implements Closeable { private final HttpClient http; private final GrpcClient grpc; - public final Collections collections; + public final CollectionsClient collections; public WeaviateClient(Config config) { this.http = new HttpClient(); this.grpc = new GrpcClient(config); - this.collections = new Collections(config, http, grpc); + this.collections = new CollectionsClient(config, http, grpc); } @Override diff --git a/src/main/java/io/weaviate/client6/v1/Collection.java b/src/main/java/io/weaviate/client6/v1/Collection.java deleted file mode 100644 index 880114851..000000000 --- a/src/main/java/io/weaviate/client6/v1/Collection.java +++ /dev/null @@ -1,23 +0,0 @@ -package io.weaviate.client6.v1; - -import io.weaviate.client6.Config; -import io.weaviate.client6.internal.GrpcClient; -import io.weaviate.client6.internal.HttpClient; -import io.weaviate.client6.v1.collections.WeaviateCollectionConfig; -import io.weaviate.client6.v1.collections.aggregate.WeaviateAggregate; -import io.weaviate.client6.v1.data.Data; -import io.weaviate.client6.v1.query.Query; - -public class Collection { - public final Query query; - public final Data data; - public final WeaviateCollectionConfig config; - public final WeaviateAggregate aggregate; - - public Collection(String collectionName, Config config, GrpcClient grpc, HttpClient http) { - this.query = new Query<>(collectionName, grpc); - this.data = new Data<>(collectionName, config, http); - this.config = new WeaviateCollectionConfig(collectionName, config, http); - this.aggregate = new WeaviateAggregate(collectionName, grpc); - } -} diff --git a/src/main/java/io/weaviate/client6/v1/ObjectMetadata.java b/src/main/java/io/weaviate/client6/v1/ObjectMetadata.java deleted file mode 100644 index d63b0225a..000000000 --- a/src/main/java/io/weaviate/client6/v1/ObjectMetadata.java +++ /dev/null @@ -1,55 +0,0 @@ -package io.weaviate.client6.v1; - -import java.util.function.Consumer; - -public record ObjectMetadata(String id, Vectors vectors) { - - public static ObjectMetadata with(Consumer options) { - var opt = new Builder(options); - return new ObjectMetadata(opt.id, opt.vectors); - } - - public static class Builder { - public String id; - public Vectors vectors; - - public Builder id(String id) { - this.id = id; - return this; - } - - public Builder vectors(Vectors vectors) { - this.vectors = vectors; - return this; - } - - public Builder vectors(Float[] vector) { - this.vectors = Vectors.of(vector); - return this; - } - - public Builder vectors(Float[][] vector) { - this.vectors = Vectors.of(vector); - return this; - } - - public Builder vectors(String name, Float[] vector) { - this.vectors = Vectors.of(name, vector); - return this; - } - - public Builder vectors(String name, Float[][] vector) { - this.vectors = Vectors.of(name, vector); - return this; - } - - public Builder vectors(Consumer named) { - this.vectors = Vectors.with(named); - return this; - } - - private Builder(Consumer options) { - options.accept(this); - } - } -} diff --git a/src/main/java/io/weaviate/client6/v1/collections/AtomicDataType.java b/src/main/java/io/weaviate/client6/v1/collections/AtomicDataType.java new file mode 100644 index 000000000..da54b1c28 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/AtomicDataType.java @@ -0,0 +1,15 @@ +package io.weaviate.client6.v1.collections; + +import com.google.gson.annotations.SerializedName; + +public enum AtomicDataType { + @SerializedName("text") + TEXT, + @SerializedName("int") + INT; + + public static boolean isAtomic(String type) { + return type.equals(TEXT.name().toLowerCase()) + || type.equals(INT.name().toLowerCase()); + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinition.java b/src/main/java/io/weaviate/client6/v1/collections/Collection.java similarity index 82% rename from src/main/java/io/weaviate/client6/v1/collections/CollectionDefinition.java rename to src/main/java/io/weaviate/client6/v1/collections/Collection.java index b599f4ceb..ad9588b76 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinition.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Collection.java @@ -7,11 +7,11 @@ import io.weaviate.client6.v1.collections.Vectors.NamedVectors; -public record CollectionDefinition(String name, List properties, Vectors vectors) { +public record Collection(String name, List properties, Vectors vectors) { - public static CollectionDefinition with(String name, Consumer options) { + public static Collection with(String name, Consumer options) { var config = new Configuration(options); - return new CollectionDefinition(name, config.properties, config.vectors); + return new Collection(name, config.properties, config.vectors); } // Tucked Builder for additional collection configuration. diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java new file mode 100644 index 000000000..cf2e29649 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java @@ -0,0 +1,22 @@ +package io.weaviate.client6.v1.collections; + +import io.weaviate.client6.Config; +import io.weaviate.client6.internal.GrpcClient; +import io.weaviate.client6.internal.HttpClient; +import io.weaviate.client6.v1.collections.aggregate.AggregateClient; +import io.weaviate.client6.v1.collections.data.DataClient; +import io.weaviate.client6.v1.query.QueryClient; + +public class CollectionClient { + public final QueryClient query; + public final DataClient data; + public final CollectionConfigClient config; + public final AggregateClient aggregate; + + public CollectionClient(String collectionName, Config config, GrpcClient grpc, HttpClient http) { + this.query = new QueryClient<>(collectionName, grpc); + this.data = new DataClient<>(collectionName, config, http); + this.config = new CollectionConfigClient(collectionName, config, http); + this.aggregate = new AggregateClient(collectionName, grpc); + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java similarity index 90% rename from src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java rename to src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java index 6ca116ec9..84c502862 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/WeaviateCollectionConfig.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java @@ -1,5 +1,6 @@ package io.weaviate.client6.v1.collections; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.Optional; @@ -18,7 +19,7 @@ import lombok.AllArgsConstructor; @AllArgsConstructor -public class WeaviateCollectionConfig { +public class CollectionConfigClient { // TODO: hide befind an internal HttpClient private final String collectionName; private final Config config; @@ -27,14 +28,14 @@ public class WeaviateCollectionConfig { private static final Gson gson = new Gson(); static { DtoTypeAdapterFactory.register( - CollectionDefinition.class, + Collection.class, CollectionDefinitionDTO.class, m -> { return new CollectionDefinitionDTO(m); }); } - public Optional get() throws IOException { + public Optional get() throws IOException { ClassicHttpRequest httpGet = ClassicRequestBuilder .get(config.baseUrl() + "/schema/" + collectionName) .build(); @@ -44,7 +45,7 @@ public Optional get() throws IOException { return Optional.empty(); } try (var r = new InputStreamReader(response.getEntity().getContent())) { - var collection = gson.fromJson(r, CollectionDefinition.class); + var collection = gson.fromJson(r, Collection.class); return Optional.ofNullable(collection); } }); diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinitionDTO.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinitionDTO.java index e0333fad0..d35dce478 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinitionDTO.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionDefinitionDTO.java @@ -6,7 +6,7 @@ import io.weaviate.client6.internal.DtoTypeAdapterFactory; -class CollectionDefinitionDTO implements DtoTypeAdapterFactory.Dto { +class CollectionDefinitionDTO implements DtoTypeAdapterFactory.Dto { @SerializedName("class") String collection; @@ -25,7 +25,7 @@ class CollectionDefinitionDTO implements DtoTypeAdapterFactory.Dto { return new CollectionDefinitionDTO(m); @@ -117,9 +116,8 @@ public void create(String name) throws IOException { }); } - public void create(String name, Consumer options) throws IOException { - var collection = CollectionDefinition.with(name, options); - + public void create(String name, Consumer options) throws IOException { + var collection = Collection.with(name, options); ClassicHttpRequest httpPost = ClassicRequestBuilder .post(config.baseUrl() + "/schema") .setEntity(gson.toJson(collection), ContentType.APPLICATION_JSON) @@ -136,7 +134,7 @@ public void create(String name, Consumer opt }); } - public Optional getConfig(String name) throws IOException { + public Optional getConfig(String name) throws IOException { ClassicHttpRequest httpGet = ClassicRequestBuilder .get(config.baseUrl() + "/schema/" + name) .build(); @@ -146,7 +144,7 @@ public Optional getConfig(String name) throws IOException return Optional.empty(); } try (var r = new InputStreamReader(response.getEntity().getContent())) { - var collection = gson.fromJson(r, CollectionDefinition.class); + var collection = gson.fromJson(r, Collection.class); return Optional.ofNullable(collection); } }); @@ -167,7 +165,7 @@ public void delete(String name) throws IOException { }); } - public Collection> use(String name) { - return new Collection<>(name, config, grpcClient, httpClient); + public CollectionClient> use(String name) { + return new CollectionClient<>(name, config, grpcClient, httpClient); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/DataType.java b/src/main/java/io/weaviate/client6/v1/collections/DataType.java deleted file mode 100644 index 4bb42b525..000000000 --- a/src/main/java/io/weaviate/client6/v1/collections/DataType.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.weaviate.client6.v1.collections; - -import com.google.gson.annotations.SerializedName; - -public enum DataType { - @SerializedName("text") - TEXT, - @SerializedName("int") - INT, - @SerializedName("reference") - REFERENCE; -} diff --git a/src/main/java/io/weaviate/client6/v1/collections/Property.java b/src/main/java/io/weaviate/client6/v1/collections/Property.java index 8d5d53db1..a1a811ddd 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/Property.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Property.java @@ -5,34 +5,33 @@ import com.google.gson.annotations.SerializedName; -public class Property { - @SerializedName("name") - public final String name; - - @SerializedName("dataType") - public final List dataTypes; +public record Property( + @SerializedName("name") String name, + @SerializedName("dataType") List dataTypes) { /** Add text property with default configuration. */ public static Property text(String name) { - return new Property(name, DataType.TEXT); + return new Property(name, AtomicDataType.TEXT); } /** Add integer property with default configuration. */ public static Property integer(String name) { - return new Property(name, DataType.INT); + return new Property(name, AtomicDataType.INT); } public static Property reference(String name, String... collections) { return new Property(name, collections); } - private Property(String name, DataType type) { - this.name = name; - this.dataTypes = List.of(type.name().toLowerCase()); + public boolean isReference() { + return dataTypes.stream().noneMatch(t -> AtomicDataType.isAtomic(t)); + } + + private Property(String name, AtomicDataType type) { + this(name, List.of(type.name().toLowerCase())); } private Property(String name, String... collections) { - this.name = name; - this.dataTypes = Arrays.asList(collections); + this(name, Arrays.asList(collections)); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/Reference.java b/src/main/java/io/weaviate/client6/v1/collections/Reference.java new file mode 100644 index 000000000..b17799911 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/Reference.java @@ -0,0 +1,59 @@ +package io.weaviate.client6.v1.collections; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import com.google.gson.stream.JsonWriter; + +import io.weaviate.client6.v1.collections.object.WeaviateObject; + +public record Reference(String collection, List uuids) { + + public Reference(String collection, String uuid) { + this(collection, List.of(uuid)); + } + + /** + * Create reference to objects by their UUIDs. + *

+ * Weaviate will search each of the existing collections to identify + * the objects before inserting the references, so this may include + * some performance overhead. + */ + public static Reference uuids(String... uuids) { + return new Reference(null, Arrays.asList(uuids)); + } + + /** Create references to {@link WeaviateObject}. */ + public static Reference[] objects(WeaviateObject... objects) { + return Arrays.stream(objects) + .map(o -> new Reference(o.collection(), o.metadata().id())) + .toArray(Reference[]::new); + } + + /** Create references to objects in a collection by their UUIDs. */ + public static Reference collection(String collection, String... uuids) { + return new Reference(collection, Arrays.asList(uuids)); + } + + // TODO: put this in a type adapter. + /** writeValue assumes an array has been started will be ended by the caller. */ + public void writeValue(JsonWriter w) throws IOException { + for (var uuid : uuids) { + w.beginObject(); + w.name("beacon"); + w.value(toBeacon(uuid)); + w.endObject(); + } + } + + private String toBeacon(String uuid) { + var beacon = "weaviate://localhost/"; + if (collection != null) { + beacon += collection + "/"; + } + beacon += uuid; + return beacon; + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java similarity index 97% rename from src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java rename to src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java index e018be2fa..8ad716ae1 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/WeaviateAggregate.java +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java @@ -7,11 +7,11 @@ import io.weaviate.client6.internal.codec.grpc.v1.AggregateUnmarshaler; import io.weaviate.client6.v1.query.NearVector; -public class WeaviateAggregate { +public class AggregateClient { private final String collectionName; private final GrpcClient grpcClient; - public WeaviateAggregate(String collectionName, GrpcClient grpc) { + public AggregateClient(String collectionName, GrpcClient grpc) { this.collectionName = collectionName; this.grpcClient = grpc; } diff --git a/src/main/java/io/weaviate/client6/v1/data/ConsistencyLevel.java b/src/main/java/io/weaviate/client6/v1/collections/data/ConsistencyLevel.java similarity index 51% rename from src/main/java/io/weaviate/client6/v1/data/ConsistencyLevel.java rename to src/main/java/io/weaviate/client6/v1/collections/data/ConsistencyLevel.java index 1011a7a02..3347b012c 100644 --- a/src/main/java/io/weaviate/client6/v1/data/ConsistencyLevel.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/ConsistencyLevel.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.data; public enum ConsistencyLevel { ONE, QUORUM, ALL diff --git a/src/main/java/io/weaviate/client6/v1/data/Data.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java similarity index 82% rename from src/main/java/io/weaviate/client6/v1/data/Data.java rename to src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index 54b476b22..a1b6404b3 100644 --- a/src/main/java/io/weaviate/client6/v1/data/Data.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.data; import java.io.IOException; import java.util.Optional; @@ -16,11 +16,11 @@ import io.weaviate.client6.Config; import io.weaviate.client6.internal.HttpClient; -import io.weaviate.client6.v1.ObjectMetadata; +import io.weaviate.client6.v1.collections.object.WeaviateObject; import lombok.AllArgsConstructor; @AllArgsConstructor -public class Data { +public class DataClient { // TODO: inject singleton as dependency private static final Gson gson = new Gson(); @@ -31,16 +31,19 @@ public class Data { private final Config config; private final HttpClient httpClient; - public WeaviateObject insert(T object) throws IOException { - return insert(object, opt -> { + public WeaviateObject insert(T properties) throws IOException { + return insert(properties, opt -> { }); } - public WeaviateObject insert(T object, Consumer options) throws IOException { - var body = new WeaviateObject<>(collectionName, object, options); + public WeaviateObject insert(T properties, Consumer> fn) throws IOException { + return insert(InsertObjectRequest.of(collectionName, properties, fn)); + } + + public WeaviateObject insert(InsertObjectRequest request) throws IOException { ClassicHttpRequest httpPost = ClassicRequestBuilder .post(config.baseUrl() + "/objects") - .setEntity(body.toJson(gson), ContentType.APPLICATION_JSON) + .setEntity(request.serialize(gson), ContentType.APPLICATION_JSON) .build(); return httpClient.http.execute(httpPost, response -> { diff --git a/src/main/java/io/weaviate/client6/v1/data/GetParameters.java b/src/main/java/io/weaviate/client6/v1/collections/data/GetParameters.java similarity index 97% rename from src/main/java/io/weaviate/client6/v1/data/GetParameters.java rename to src/main/java/io/weaviate/client6/v1/collections/data/GetParameters.java index 819ffc09c..a0e4afd43 100644 --- a/src/main/java/io/weaviate/client6/v1/data/GetParameters.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/GetParameters.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.data; import java.util.LinkedHashSet; import java.util.List; diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java new file mode 100644 index 000000000..18623c41e --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java @@ -0,0 +1,149 @@ +package io.weaviate.client6.v1.collections.data; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import io.weaviate.client6.v1.collections.Reference; +import io.weaviate.client6.v1.collections.object.Vectors; + +public record InsertObjectRequest(String collection, T properties, String id, Vectors vectors, + Map> references) { + + /** Create InsertObjectRequest from Builder options. */ + public InsertObjectRequest(Builder builder) { + this(builder.collection, builder.properties, builder.id, builder.vectors, builder.references); + } + + /** + * Construct InsertObjectRequest with optional parameters. + * + * @param Shape of the object properties, e.g. + * {@code Map} + * @param collection Collection to insert to. + * @param properties Object properties. + * @param fn Optional parameters + * @return InsertObjectRequest + */ + static InsertObjectRequest of(String collection, T properties, Consumer> fn) { + var builder = new Builder<>(collection, properties); + fn.accept(builder); + return builder.build(); + } + + public static class Builder { + private final String collection; // Required + private final T properties; // Required + + private String id; + private Vectors vectors; + private final Map> references = new HashMap<>(); + + Builder(String collection, T properties) { + this.collection = collection; + this.properties = properties; + } + + /** Define custom object id. Must be a valid UUID. */ + public Builder id(String id) { + this.id = id; + return this; + } + + /** + * Supply one or more (named) vectors. Calls to {@link #vectors} are not + * chainable. Use {@link Vectors#of(Consumer) to pass multiple vectors. + */ + public Builder vectors(Vectors vectors) { + this.vectors = vectors; + return this; + } + + /** + * Add a reference. Calls to {@link #reference} can be chained + * to add multiple references. + */ + public Builder reference(String property, Reference... references) { + for (var ref : references) { + addReference(property, ref); + } + return this; + } + + private void addReference(String property, Reference reference) { + if (!references.containsKey(property)) { + references.put(property, new ArrayList<>()); + } + references.get(property).add(reference); + } + + /** Build a new InsertObjectRequest. */ + public InsertObjectRequest build() { + return new InsertObjectRequest<>(this); + } + } + + // Here we're just rawdogging JSON serialization just to get a good feel for it. + public String serialize(Gson gson) throws IOException { + var buf = new StringWriter(); + var w = gson.newJsonWriter(buf); + + w.beginObject(); + + w.name("class"); + w.value(collection); + + if (id != null) { + w.name("id"); + w.value(id); + } + + if (vectors != null) { + var unnamed = vectors.getUnnamed(); + if (unnamed.isPresent()) { + w.name("vector"); + gson.getAdapter(Float[].class).write(w, unnamed.get()); + } else { + w.name("vectors"); + gson.getAdapter(new TypeToken>() { + }).write(w, vectors.getNamed()); + } + } + + if (properties != null || references != null) { + w.name("properties"); + w.beginObject(); + + if (properties != null) { + assert properties instanceof Map : "properties not a Map"; + for (var entry : ((Map) properties).entrySet()) { + w.name(entry.getKey()); + gson.getAdapter(Object.class).write(w, entry.getValue()); + } + + } + if (references != null && !references.isEmpty()) { + for (var entry : references.entrySet()) { + w.name(entry.getKey()); + w.beginArray(); + for (var ref : entry.getValue()) { + ref.writeValue(w); + } + w.endArray(); + } + } + + w.endObject(); + } + + w.endObject(); + return buf.toString(); + } +} diff --git a/src/main/java/io/weaviate/client6/v1/data/QueryParameters.java b/src/main/java/io/weaviate/client6/v1/collections/data/QueryParameters.java similarity index 95% rename from src/main/java/io/weaviate/client6/v1/data/QueryParameters.java rename to src/main/java/io/weaviate/client6/v1/collections/data/QueryParameters.java index 2b94c834c..6f7eda826 100644 --- a/src/main/java/io/weaviate/client6/v1/data/QueryParameters.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/QueryParameters.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.data; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java new file mode 100644 index 000000000..e4cf36602 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java @@ -0,0 +1,30 @@ +package io.weaviate.client6.v1.collections.object; + +import java.util.function.Consumer; + +public record ObjectMetadata(String id, Vectors vectors) { + + public static ObjectMetadata with(Consumer options) { + var opt = new Builder(options); + return new ObjectMetadata(opt.id, opt.vectors); + } + + public static class Builder { + public String id; + public Vectors vectors; + + public Builder id(String id) { + this.id = id; + return this; + } + + public Builder vectors(Vectors vectors) { + this.vectors = vectors; + return this; + } + + private Builder(Consumer options) { + options.accept(this); + } + } +} diff --git a/src/main/java/io/weaviate/client6/v1/Vectors.java b/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java similarity index 88% rename from src/main/java/io/weaviate/client6/v1/Vectors.java rename to src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java index 9a69a5fa9..02830ab6e 100644 --- a/src/main/java/io/weaviate/client6/v1/Vectors.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1; +package io.weaviate.client6.v1.collections.object; import java.util.Collections; import java.util.HashMap; @@ -53,7 +53,7 @@ private Optional getOnly() { return Optional.ofNullable(namedVectors.values().iterator().next()); } - public Map asMap() { + public Map getNamed() { return Map.copyOf(namedVectors); } @@ -73,9 +73,8 @@ private Vectors(Map vectors) { this.namedVectors = Collections.unmodifiableMap(vectors); } - static Vectors with(Consumer named) { - var vectors = new NamedVectors(named); - return new Vectors(vectors.namedVectors); + private Vectors(NamedVectors named) { + this.namedVectors = named.namedVectors; } /** @@ -106,6 +105,12 @@ public static Vectors of(Map vectors) { return new Vectors(vectors); } + public static Vectors of(Consumer fn) { + var named = new NamedVectors(); + fn.accept(named); + return named.build(); + } + public static class NamedVectors { private Map namedVectors = new HashMap<>(); @@ -119,8 +124,8 @@ public NamedVectors vector(String name, Float[][] vector) { return this; } - NamedVectors(Consumer options) { - options.accept(this); + public Vectors build() { + return new Vectors(this); } } } diff --git a/src/main/java/io/weaviate/client6/v1/data/WeaviateObject.java b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java similarity index 69% rename from src/main/java/io/weaviate/client6/v1/data/WeaviateObject.java rename to src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java index edc08d251..68c85370d 100644 --- a/src/main/java/io/weaviate/client6/v1/data/WeaviateObject.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.object; import java.io.IOException; import java.io.InputStream; @@ -8,13 +8,12 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; -import io.weaviate.client6.v1.ObjectMetadata; +public record WeaviateObject( + String collection, + T properties, + ObjectMetadata metadata) { -// TODO: unify this with collections.SearchObject - -public record WeaviateObject(String collection, T properties, ObjectMetadata metadata) { - - WeaviateObject(String collection, T properties, Consumer options) { + public WeaviateObject(String collection, T properties, Consumer options) { this(collection, properties, ObjectMetadata.with(options)); } diff --git a/src/main/java/io/weaviate/client6/v1/data/WeaviateObjectDTO.java b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java similarity index 86% rename from src/main/java/io/weaviate/client6/v1/data/WeaviateObjectDTO.java rename to src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java index ed9b00af8..8e923bc6b 100644 --- a/src/main/java/io/weaviate/client6/v1/data/WeaviateObjectDTO.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.object; import java.util.ArrayList; import java.util.HashMap; @@ -6,9 +6,6 @@ import com.google.gson.annotations.SerializedName; -import io.weaviate.client6.v1.ObjectMetadata; -import io.weaviate.client6.v1.Vectors; - class WeaviateObjectDTO { @SerializedName("class") String collection; @@ -26,7 +23,7 @@ class WeaviateObjectDTO { if (object.metadata() != null) { this.id = object.metadata().id(); if (object.metadata().vectors() != null) { - this.vectors = object.metadata().vectors().asMap(); + this.vectors = object.metadata().vectors().getNamed(); } } } diff --git a/src/main/java/io/weaviate/client6/v1/query/Query.java b/src/main/java/io/weaviate/client6/v1/query/QueryClient.java similarity index 98% rename from src/main/java/io/weaviate/client6/v1/query/Query.java rename to src/main/java/io/weaviate/client6/v1/query/QueryClient.java index 673ed1f48..403a28689 100644 --- a/src/main/java/io/weaviate/client6/v1/query/Query.java +++ b/src/main/java/io/weaviate/client6/v1/query/QueryClient.java @@ -18,7 +18,7 @@ import io.weaviate.client6.internal.GrpcClient; import io.weaviate.client6.internal.codec.grpc.v1.SearchMarshaler; -public class Query { +public class QueryClient { // TODO: this should be wrapped around in some TypeInspector etc. private final String collectionName; @@ -26,7 +26,7 @@ public class Query { // (probably on a "higher" level); private final GrpcClient grpcClient; - public Query(String collectionName, GrpcClient grpc) { + public QueryClient(String collectionName, GrpcClient grpc) { this.grpcClient = grpc; this.collectionName = collectionName; } diff --git a/src/test/java/io/weaviate/client6/v1/ObjectMetadataTest.java b/src/test/java/io/weaviate/client6/v1/ObjectMetadataTest.java index 542f5f7da..f95446fdc 100644 --- a/src/test/java/io/weaviate/client6/v1/ObjectMetadataTest.java +++ b/src/test/java/io/weaviate/client6/v1/ObjectMetadataTest.java @@ -5,6 +5,9 @@ import org.assertj.core.api.Assertions; import org.junit.Test; +import io.weaviate.client6.v1.collections.object.ObjectMetadata; +import io.weaviate.client6.v1.collections.object.Vectors; + public class ObjectMetadataTest { @Test @@ -28,7 +31,7 @@ public final void testVectorsMetadata_unnamed() { @Test public final void testVectorsMetadata_default() { Float[] vector = { 1f, 2f, 3f }; - var metadata = ObjectMetadata.with(m -> m.vectors(vector)); + var metadata = ObjectMetadata.with(m -> m.vectors(Vectors.of(vector))); Assertions.assertThat(metadata.vectors()) .as("default vector").isNotNull() @@ -40,7 +43,7 @@ public final void testVectorsMetadata_default() { @Test public final void testVectorsMetadata_default_2d() { Float[][] vector = { { 1f, 2f, 3f }, { 1f, 2f, 3f } }; - var metadata = ObjectMetadata.with(m -> m.vectors(vector)); + var metadata = ObjectMetadata.with(m -> m.vectors(Vectors.of(vector))); Assertions.assertThat(metadata.vectors()) .as("default 2d vector").isNotNull() @@ -52,7 +55,7 @@ public final void testVectorsMetadata_default_2d() { @Test public final void testVectorsMetadata_named() { Float[] vector = { 1f, 2f, 3f }; - var metadata = ObjectMetadata.with(m -> m.vectors("vector-1", vector)); + var metadata = ObjectMetadata.with(m -> m.vectors(Vectors.of("vector-1", vector))); Assertions.assertThat(metadata.vectors()) .as("named vector").isNotNull() @@ -64,7 +67,7 @@ public final void testVectorsMetadata_named() { @Test public final void testVectorsMetadata_named_2d() { Float[][] vector = { { 1f, 2f, 3f }, { 1f, 2f, 3f } }; - var metadata = ObjectMetadata.with(m -> m.vectors("vector-1", vector)); + var metadata = ObjectMetadata.with(m -> m.vectors(Vectors.of("vector-1", vector))); Assertions.assertThat(metadata.vectors()) .as("named 2d vector").isNotNull() @@ -78,9 +81,9 @@ public final void testVectorsMetadata_multiple_named() { Float[][] vector_1 = { { 1f, 2f, 3f }, { 1f, 2f, 3f } }; Float[] vector_2 = { 4f, 5f, 6f }; var metadata = ObjectMetadata.with(m -> m.vectors( - named -> named + Vectors.of(named -> named .vector("vector-1", vector_1) - .vector("vector-2", vector_2))); + .vector("vector-2", vector_2)))); Assertions.assertThat(metadata.vectors()) .as("multiple named vectors").isNotNull() diff --git a/src/test/java/io/weaviate/client6/v1/collections/VectorsTest.java b/src/test/java/io/weaviate/client6/v1/collections/VectorsTest.java index 8deae4893..be7ef7174 100644 --- a/src/test/java/io/weaviate/client6/v1/collections/VectorsTest.java +++ b/src/test/java/io/weaviate/client6/v1/collections/VectorsTest.java @@ -25,7 +25,7 @@ public class VectorsTest { // private static final Gson gson = new Gson(); static { - DtoTypeAdapterFactory.register(CollectionDefinition.class, CollectionDefinitionDTO.class, + DtoTypeAdapterFactory.register(Collection.class, CollectionDefinitionDTO.class, m -> new CollectionDefinitionDTO(m)); } private static final Gson gson = new GsonBuilder() @@ -108,13 +108,13 @@ public static Object[][] testCases() { @Test @DataMethod(source = VectorsTest.class, method = "testCases") - public void test_toJson(String want, CollectionDefinition collection, String... compareKeys) { + public void test_toJson(String want, Collection collection, String... compareKeys) { var got = gson.toJson(collection); assertEqual(want, got, compareKeys); } - private static CollectionDefinition collectionWithVectors(Vectors vectors) { - return new CollectionDefinition("Things", List.of(), vectors); + private static Collection collectionWithVectors(Vectors vectors) { + return new Collection("Things", List.of(), vectors); } private void assertEqual(String wantJson, String gotJson, String... compareKeys) { diff --git a/src/test/java/io/weaviate/client6/v1/data/QueryParametersTest.java b/src/test/java/io/weaviate/client6/v1/collections/data/QueryParametersTest.java similarity index 96% rename from src/test/java/io/weaviate/client6/v1/data/QueryParametersTest.java rename to src/test/java/io/weaviate/client6/v1/collections/data/QueryParametersTest.java index 6c6caa676..275c8a9b7 100644 --- a/src/test/java/io/weaviate/client6/v1/data/QueryParametersTest.java +++ b/src/test/java/io/weaviate/client6/v1/collections/data/QueryParametersTest.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.data; +package io.weaviate.client6.v1.collections.data; import org.assertj.core.api.Assertions; import org.junit.Test; From 3cd53761416b9366372885f483df9614d67a5205 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 24 Apr 2025 14:12:57 +0200 Subject: [PATCH 08/16] feat(wip): include referencs in query response --- .../integration/NearVectorQueryITest.java | 6 +- .../weaviate/integration/ReferencesITest.java | 33 +++-- .../codec/grpc/v1/AggregateMarshaler.java | 3 +- .../codec/grpc/v1/SearchMarshaler.java | 4 +- .../v1/collections/CollectionClient.java | 4 +- .../collections/CollectionConfigClient.java | 1 - .../aggregate/AggregateClient.java | 2 +- .../v1/collections/data/DataClient.java | 140 ++++++++++++++++-- .../collections/object/ObjectReference.java | 6 + .../v1/collections/object/WeaviateObject.java | 7 +- .../collections/object/WeaviateObjectDTO.java | 5 +- .../query/CommonQueryOptions.java | 2 +- .../query/GroupedQueryResult.java | 4 +- .../v1/{ => collections}/query/Metadata.java | 2 +- .../query/MetadataField.java | 2 +- .../{ => collections}/query/NearVector.java | 2 +- .../{ => collections}/query/QueryClient.java | 2 +- .../{ => collections}/query/QueryResult.java | 2 +- 18 files changed, 180 insertions(+), 47 deletions(-) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/CommonQueryOptions.java (98%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/GroupedQueryResult.java (84%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/Metadata.java (88%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/MetadataField.java (93%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/NearVector.java (94%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/QueryClient.java (99%) rename src/main/java/io/weaviate/client6/v1/{ => collections}/query/QueryResult.java (89%) diff --git a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java b/src/it/java/io/weaviate/integration/NearVectorQueryITest.java index 1575712dc..863652781 100644 --- a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java +++ b/src/it/java/io/weaviate/integration/NearVectorQueryITest.java @@ -18,9 +18,9 @@ import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy; import io.weaviate.client6.v1.collections.Vectorizer; import io.weaviate.client6.v1.collections.object.Vectors; -import io.weaviate.client6.v1.query.GroupedQueryResult; -import io.weaviate.client6.v1.query.MetadataField; -import io.weaviate.client6.v1.query.NearVector; +import io.weaviate.client6.v1.collections.query.GroupedQueryResult; +import io.weaviate.client6.v1.collections.query.MetadataField; +import io.weaviate.client6.v1.collections.query.NearVector; import io.weaviate.containers.Container; public class NearVectorQueryITest extends ConcurrentTest { diff --git a/src/it/java/io/weaviate/integration/ReferencesITest.java b/src/it/java/io/weaviate/integration/ReferencesITest.java index b2667115b..cc9e9b95d 100644 --- a/src/it/java/io/weaviate/integration/ReferencesITest.java +++ b/src/it/java/io/weaviate/integration/ReferencesITest.java @@ -10,9 +10,10 @@ import io.weaviate.ConcurrentTest; import io.weaviate.client6.WeaviateClient; -import io.weaviate.client6.v1.collections.Collection; import io.weaviate.client6.v1.collections.Property; import io.weaviate.client6.v1.collections.Reference; +import io.weaviate.client6.v1.collections.object.ObjectReference; +import io.weaviate.client6.v1.collections.object.WeaviateObject; import io.weaviate.containers.Container; /** @@ -53,12 +54,10 @@ public void testReferences() throws IOException { var collectionArtists = artists.config.get(); Assertions.assertThat(collectionArtists).get() .as("Artists: create collection") - .extracting(Collection::properties) - .extracting(properties -> properties.stream().filter(Property::isReference).findFirst()) - .extracting(Optional::get) + .extracting(c -> c.properties().stream().filter(Property::isReference).findFirst()) + .as("has one reference property").extracting(Optional::get) .returns("hasAwards", Property::name) - .extracting(Property::dataTypes) - .asInstanceOf(InstanceOfAssertFactories.list(String.class)) + .extracting(Property::dataTypes, InstanceOfAssertFactories.list(String.class)) .containsOnly(nsGrammy, nsOscar); // Act: insert some data @@ -82,14 +81,22 @@ public void testReferences() throws IOException { collectionArtists = artists.config.get(); Assertions.assertThat(collectionArtists).get() .as("Artists: add reference to Movies") - .extracting(Collection::properties) - .extracting(properties -> properties.stream() - .filter(property -> property.name().equals("featuredIn")) - .findFirst()) - .extracting(Optional::get) + .extracting(c -> c.properties().stream() + .filter(property -> property.name().equals("featuredIn")).findFirst()) + .as("featuredIn reference property").extracting(Optional::get) .returns(true, Property::isReference) - .extracting(Property::dataTypes) - .asInstanceOf(InstanceOfAssertFactories.list(String.class)) + .extracting(Property::dataTypes, InstanceOfAssertFactories.list(String.class)) .containsOnly(nsMovies); + + var gotAlex = artists.data.get(alex.metadata().id()); + Assertions.assertThat(gotAlex).get() + .as("Artists: fetch by id including hasAwards references") + .extracting(WeaviateObject::references, InstanceOfAssertFactories.map(String.class, ObjectReference.class)) + .as("hasAwards object reference").extractingByKey("hasAwards") + .extracting(ObjectReference::objects, InstanceOfAssertFactories.list(WeaviateObject.class)) + .extracting(objects -> objects.metadata().id()) + .containsOnly( + // grammy_1.metadata().id(), grammy_2.metadata().id(), + oscar_1.metadata().id(), oscar_2.metadata().id()); } } diff --git a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/AggregateMarshaler.java b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/AggregateMarshaler.java index 446adba78..bf046b0dd 100644 --- a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/AggregateMarshaler.java +++ b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/AggregateMarshaler.java @@ -8,13 +8,12 @@ import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoAggregate.AggregateRequest.Aggregation; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBaseSearch; import io.weaviate.client6.internal.GRPC; -import io.weaviate.client6.v1.collections.aggregate.AggregateGroupByRequest; import io.weaviate.client6.v1.collections.aggregate.AggregateGroupByRequest.GroupBy; import io.weaviate.client6.v1.collections.aggregate.AggregateRequest; import io.weaviate.client6.v1.collections.aggregate.IntegerMetric; import io.weaviate.client6.v1.collections.aggregate.Metric; import io.weaviate.client6.v1.collections.aggregate.TextMetric; -import io.weaviate.client6.v1.query.NearVector; +import io.weaviate.client6.v1.collections.query.NearVector; public final class AggregateMarshaler { private final WeaviateProtoAggregate.AggregateRequest.Builder req = WeaviateProtoAggregate.AggregateRequest diff --git a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java index a85970bb1..ca6769f01 100644 --- a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java +++ b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java @@ -9,8 +9,8 @@ import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; import io.weaviate.client6.internal.GRPC; import io.weaviate.client6.internal.codec.grpc.GrpcMarshaler; -import io.weaviate.client6.v1.query.CommonQueryOptions; -import io.weaviate.client6.v1.query.NearVector; +import io.weaviate.client6.v1.collections.query.CommonQueryOptions; +import io.weaviate.client6.v1.collections.query.NearVector; public class SearchMarshaler implements GrpcMarshaler { private final WeaviateProtoSearchGet.SearchRequest.Builder req = WeaviateProtoSearchGet.SearchRequest.newBuilder(); diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java index cf2e29649..4f8cd6fdf 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionClient.java @@ -5,7 +5,7 @@ import io.weaviate.client6.internal.HttpClient; import io.weaviate.client6.v1.collections.aggregate.AggregateClient; import io.weaviate.client6.v1.collections.data.DataClient; -import io.weaviate.client6.v1.query.QueryClient; +import io.weaviate.client6.v1.collections.query.QueryClient; public class CollectionClient { public final QueryClient query; @@ -15,7 +15,7 @@ public class CollectionClient { public CollectionClient(String collectionName, Config config, GrpcClient grpc, HttpClient http) { this.query = new QueryClient<>(collectionName, grpc); - this.data = new DataClient<>(collectionName, config, http); + this.data = new DataClient<>(collectionName, config, http, grpc); this.config = new CollectionConfigClient(collectionName, config, http); this.aggregate = new AggregateClient(collectionName, grpc); } diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java index 84c502862..186ea3ee2 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionConfigClient.java @@ -1,6 +1,5 @@ package io.weaviate.client6.v1.collections; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.Optional; diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java index 8ad716ae1..87e365d8d 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java @@ -5,7 +5,7 @@ import io.weaviate.client6.internal.GrpcClient; import io.weaviate.client6.internal.codec.grpc.v1.AggregateMarshaler; import io.weaviate.client6.internal.codec.grpc.v1.AggregateUnmarshaler; -import io.weaviate.client6.v1.query.NearVector; +import io.weaviate.client6.v1.collections.query.NearVector; public class AggregateClient { private final String collectionName; diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index a1b6404b3..1e1892c24 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -1,8 +1,12 @@ package io.weaviate.client6.v1.collections.data; import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.Map; import java.util.Optional; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -15,7 +19,24 @@ import com.google.gson.Gson; import io.weaviate.client6.Config; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.FilterTarget; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters.Operator; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors.VectorType; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoProperties.Value; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataResult; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesResult; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesResult; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; +import io.weaviate.client6.internal.GRPC; +import io.weaviate.client6.internal.GrpcClient; import io.weaviate.client6.internal.HttpClient; +import io.weaviate.client6.v1.collections.object.ObjectMetadata; +import io.weaviate.client6.v1.collections.object.ObjectReference; +import io.weaviate.client6.v1.collections.object.Vectors; import io.weaviate.client6.v1.collections.object.WeaviateObject; import lombok.AllArgsConstructor; @@ -30,6 +51,7 @@ public class DataClient { // TODO: hide befind an internal HttpClient private final Config config; private final HttpClient httpClient; + private final GrpcClient grpcClient; public WeaviateObject insert(T properties) throws IOException { return insert(properties, opt -> { @@ -63,21 +85,115 @@ public Optional> get(String id) throws IOException { } public Optional> get(String id, Consumer query) throws IOException { - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - ClassicHttpRequest httpGet = ClassicRequestBuilder - .get(config.baseUrl() + "/objects/" + collectionName + "/" + id + QueryParameters.encodeGet(query)) - .build(); + return findById(id); + // try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + // ClassicHttpRequest httpGet = ClassicRequestBuilder + // .get(config.baseUrl() + "/objects/" + collectionName + "/" + id + + // QueryParameters.encodeGet(query)) + // .build(); + // + // return httpClient.http.execute(httpGet, response -> { + // if (response.getCode() == HttpStatus.SC_NOT_FOUND) { + // return Optional.empty(); + // } + // + // WeaviateObject object = WeaviateObject.fromJson( + // gson, response.getEntity().getContent()); + // return Optional.ofNullable(object); + // }); + // } + } - return httpClient.http.execute(httpGet, response -> { - if (response.getCode() == HttpStatus.SC_NOT_FOUND) { - return Optional.empty(); - } + private Optional> findById(String id) { + var req = SearchRequest.newBuilder(); + req.setCollection(collectionName); + + var filter = Filters.newBuilder(); + var target = FilterTarget.newBuilder(); + target.setProperty("_id"); + filter.setTarget(target); + filter.setValueText(id); + filter.setOperator(Operator.OPERATOR_EQUAL); + req.setFilters(filter); + + var properties = PropertiesRequest.newBuilder(); + var references = RefPropertiesRequest.newBuilder(); + references.setReferenceProperty("hasAwards"); + references.setTargetCollection("ReferencesITest_testReferences_Oscar"); + references.setMetadata(MetadataRequest.newBuilder().setUuid(true)); + // TODO: pass references and properties to fetch + properties.addRefProperties(references); + req.setProperties(properties); + + var result = grpcClient.grpc.search(req.build()); + var objects = result.getResultsList().stream().map(r -> readPropertiesResult(r.getProperties())).toList(); + return Optional.ofNullable((WeaviateObject) objects.get(0)); + } - WeaviateObject object = WeaviateObject.fromJson( - gson, response.getEntity().getContent()); - return Optional.ofNullable(object); - }); + private static WeaviateObject readPropertiesResult(PropertiesResult res) { + var collection = res.getTargetCollection(); + + var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); + var referenceProperties = res.getRefPropsList().stream() + .collect(Collectors.toMap( + RefPropertiesResult::getPropName, + ref -> { + var refObjects = ref.getPropertiesList().stream() + .map(DataClient::readPropertiesResult) + .toList(); + return new ObjectReference(refObjects); + })); + + MetadataResult meta = res.getMetadata(); + Vectors vectors; + if (meta.getVectorBytes() != null) { + vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); + } else { + vectors = Vectors.of(meta.getVectorsList().stream().collect( + Collectors.toMap( + io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, + v -> { + if (v.getType().equals(VectorType.VECTOR_TYPE_MULTI_FP32)) { + return GRPC.fromByteString(v.getVectorBytes()); + } else { + return GRPC.fromByteStringMulti(v.getVectorBytes()); + } + }))); + } + var metadata = new ObjectMetadata(meta.getId(), vectors); + return new WeaviateObject<>(collection, objectProperties, referenceProperties, metadata); + } + + /* + * Convert Map to Map such that can be + * (de-)serialized by {@link Gson}. + */ + private static Map convertProtoMap(Map map) { + return map.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, e -> convertProtoValue(e.getValue()))); + } + + /** + * Convert protobuf's Value stub to an Object by extracting the first available + * field. The checks are non-exhaustive and only cover text, boolean, and + * integer values. + */ + private static Object convertProtoValue(Value value) { + if (value.hasTextValue()) { + return value.getTextValue(); + } else if (value.hasBoolValue()) { + return value.getBoolValue(); + } else if (value.hasIntValue()) { + return value.getIntValue(); + } else if (value.hasNumberValue()) { + return value.getNumberValue(); + } else if (value.hasDateValue()) { + OffsetDateTime offsetDateTime = OffsetDateTime.parse(value.getDateValue()); + return Date.from(offsetDateTime.toInstant()); + } else { + assert false : "branch not covered"; } + return null; } public void delete(String id) throws IOException { diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java new file mode 100644 index 000000000..43333ed4c --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java @@ -0,0 +1,6 @@ +package io.weaviate.client6.v1.collections.object; + +import java.util.List; + +public record ObjectReference(List> objects) { +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java index 68c85370d..28d5cc3b2 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObject.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.Map; import java.util.function.Consumer; import com.google.common.reflect.TypeToken; @@ -11,10 +12,12 @@ public record WeaviateObject( String collection, T properties, + Map references, ObjectMetadata metadata) { - public WeaviateObject(String collection, T properties, Consumer options) { - this(collection, properties, ObjectMetadata.with(options)); + public WeaviateObject(String collection, T properties, Map references, + Consumer options) { + this(collection, properties, references, ObjectMetadata.with(options)); } // JSON serialization ---------------- diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java index 8e923bc6b..e57afc1b5 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/WeaviateObjectDTO.java @@ -41,6 +41,9 @@ WeaviateObject toWeaviateObject() { arrayVectors.put(entry.getKey(), vector); } } - return new WeaviateObject(collection, properties, new ObjectMetadata(id, Vectors.of(arrayVectors))); + + return new WeaviateObject(collection, properties, + /* no references through HTTP */ new HashMap<>(), + new ObjectMetadata(id, Vectors.of(arrayVectors))); } } diff --git a/src/main/java/io/weaviate/client6/v1/query/CommonQueryOptions.java b/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java similarity index 98% rename from src/main/java/io/weaviate/client6/v1/query/CommonQueryOptions.java rename to src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java index ddf1e1ab1..4bbabb193 100644 --- a/src/main/java/io/weaviate/client6/v1/query/CommonQueryOptions.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/main/java/io/weaviate/client6/v1/query/GroupedQueryResult.java b/src/main/java/io/weaviate/client6/v1/collections/query/GroupedQueryResult.java similarity index 84% rename from src/main/java/io/weaviate/client6/v1/query/GroupedQueryResult.java rename to src/main/java/io/weaviate/client6/v1/collections/query/GroupedQueryResult.java index 01b8e68a4..cc50cf7a9 100644 --- a/src/main/java/io/weaviate/client6/v1/query/GroupedQueryResult.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/GroupedQueryResult.java @@ -1,9 +1,9 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import java.util.List; import java.util.Map; -import io.weaviate.client6.v1.query.QueryResult.SearchObject; +import io.weaviate.client6.v1.collections.query.QueryResult.SearchObject; import lombok.AllArgsConstructor; @AllArgsConstructor diff --git a/src/main/java/io/weaviate/client6/v1/query/Metadata.java b/src/main/java/io/weaviate/client6/v1/collections/query/Metadata.java similarity index 88% rename from src/main/java/io/weaviate/client6/v1/query/Metadata.java rename to src/main/java/io/weaviate/client6/v1/collections/query/Metadata.java index 4cc37bd98..fe1d40889 100644 --- a/src/main/java/io/weaviate/client6/v1/query/Metadata.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/Metadata.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; diff --git a/src/main/java/io/weaviate/client6/v1/query/MetadataField.java b/src/main/java/io/weaviate/client6/v1/collections/query/MetadataField.java similarity index 93% rename from src/main/java/io/weaviate/client6/v1/query/MetadataField.java rename to src/main/java/io/weaviate/client6/v1/collections/query/MetadataField.java index fbec8a04c..bf4e43986 100644 --- a/src/main/java/io/weaviate/client6/v1/query/MetadataField.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/MetadataField.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; diff --git a/src/main/java/io/weaviate/client6/v1/query/NearVector.java b/src/main/java/io/weaviate/client6/v1/collections/query/NearVector.java similarity index 94% rename from src/main/java/io/weaviate/client6/v1/query/NearVector.java rename to src/main/java/io/weaviate/client6/v1/collections/query/NearVector.java index 6cfee7f8f..3bcc4fef0 100644 --- a/src/main/java/io/weaviate/client6/v1/query/NearVector.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/NearVector.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import java.util.function.Consumer; diff --git a/src/main/java/io/weaviate/client6/v1/query/QueryClient.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java similarity index 99% rename from src/main/java/io/weaviate/client6/v1/query/QueryClient.java rename to src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java index 403a28689..66a8a6540 100644 --- a/src/main/java/io/weaviate/client6/v1/query/QueryClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import java.time.OffsetDateTime; import java.util.ArrayList; diff --git a/src/main/java/io/weaviate/client6/v1/query/QueryResult.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryResult.java similarity index 89% rename from src/main/java/io/weaviate/client6/v1/query/QueryResult.java rename to src/main/java/io/weaviate/client6/v1/collections/query/QueryResult.java index 3d03a9840..0fac388f1 100644 --- a/src/main/java/io/weaviate/client6/v1/query/QueryResult.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryResult.java @@ -1,4 +1,4 @@ -package io.weaviate.client6.v1.query; +package io.weaviate.client6.v1.collections.query; import java.util.List; From d3df9d7e185574b416cc92c6f59058eeca483c17 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 24 Apr 2025 19:46:35 +0200 Subject: [PATCH 09/16] feat: request references when fetching by ID --- .../io/weaviate/integration/DataITest.java | 2 +- .../weaviate/integration/ReferencesITest.java | 8 +- .../v1/collections/data/DataClient.java | 48 +------- .../v1/collections/data/FetchByIdRequest.java | 101 +++++++++++++++ .../collections/query/CommonQueryOptions.java | 13 ++ .../v1/collections/query/QueryReference.java | 115 ++++++++++++++++++ 6 files changed, 241 insertions(+), 46 deletions(-) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java diff --git a/src/it/java/io/weaviate/integration/DataITest.java b/src/it/java/io/weaviate/integration/DataITest.java index 622a5030a..602e3827b 100644 --- a/src/it/java/io/weaviate/integration/DataITest.java +++ b/src/it/java/io/weaviate/integration/DataITest.java @@ -38,7 +38,7 @@ public void testCreateGetDelete() throws IOException { .id(id) .vectors(Vectors.of(VECTOR_INDEX, vector))); - var object = artists.data.get(id, query -> query.withVector()); + var object = artists.data.get(id, query -> query.includeVector()); Assertions.assertThat(object) .as("object exists after insert").get() .satisfies(obj -> { diff --git a/src/it/java/io/weaviate/integration/ReferencesITest.java b/src/it/java/io/weaviate/integration/ReferencesITest.java index cc9e9b95d..56a12bf85 100644 --- a/src/it/java/io/weaviate/integration/ReferencesITest.java +++ b/src/it/java/io/weaviate/integration/ReferencesITest.java @@ -14,6 +14,8 @@ import io.weaviate.client6.v1.collections.Reference; import io.weaviate.client6.v1.collections.object.ObjectReference; import io.weaviate.client6.v1.collections.object.WeaviateObject; +import io.weaviate.client6.v1.collections.query.MetadataField; +import io.weaviate.client6.v1.collections.query.QueryReference; import io.weaviate.containers.Container; /** @@ -88,7 +90,11 @@ public void testReferences() throws IOException { .extracting(Property::dataTypes, InstanceOfAssertFactories.list(String.class)) .containsOnly(nsMovies); - var gotAlex = artists.data.get(alex.metadata().id()); + var gotAlex = artists.data.get(alex.metadata().id(), + opt -> opt.returnReferences( + QueryReference.multi("hasAwards", nsOscar, + ref -> ref.returnMetadata(MetadataField.ID)))); + Assertions.assertThat(gotAlex).get() .as("Artists: fetch by id including hasAwards references") .extracting(WeaviateObject::references, InstanceOfAssertFactories.map(String.class, ObjectReference.class)) diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index 1e1892c24..e54606c3b 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -19,16 +19,10 @@ import com.google.gson.Gson; import io.weaviate.client6.Config; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.FilterTarget; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters.Operator; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors.VectorType; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoProperties.Value; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataResult; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesResult; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesResult; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; import io.weaviate.client6.internal.GRPC; @@ -84,47 +78,13 @@ public Optional> get(String id) throws IOException { }); } - public Optional> get(String id, Consumer query) throws IOException { - return findById(id); - // try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - // ClassicHttpRequest httpGet = ClassicRequestBuilder - // .get(config.baseUrl() + "/objects/" + collectionName + "/" + id + - // QueryParameters.encodeGet(query)) - // .build(); - // - // return httpClient.http.execute(httpGet, response -> { - // if (response.getCode() == HttpStatus.SC_NOT_FOUND) { - // return Optional.empty(); - // } - // - // WeaviateObject object = WeaviateObject.fromJson( - // gson, response.getEntity().getContent()); - // return Optional.ofNullable(object); - // }); - // } + public Optional> get(String id, Consumer fn) throws IOException { + return findById(FetchByIdRequest.of(collectionName, id, fn)); } - private Optional> findById(String id) { + private Optional> findById(FetchByIdRequest request) { var req = SearchRequest.newBuilder(); - req.setCollection(collectionName); - - var filter = Filters.newBuilder(); - var target = FilterTarget.newBuilder(); - target.setProperty("_id"); - filter.setTarget(target); - filter.setValueText(id); - filter.setOperator(Operator.OPERATOR_EQUAL); - req.setFilters(filter); - - var properties = PropertiesRequest.newBuilder(); - var references = RefPropertiesRequest.newBuilder(); - references.setReferenceProperty("hasAwards"); - references.setTargetCollection("ReferencesITest_testReferences_Oscar"); - references.setMetadata(MetadataRequest.newBuilder().setUuid(true)); - // TODO: pass references and properties to fetch - properties.addRefProperties(references); - req.setProperties(properties); - + request.appendTo(req); var result = grpcClient.grpc.search(req.build()); var objects = result.getResultsList().stream().map(r -> readPropertiesResult(r.getProperties())).toList(); return Optional.ofNullable((WeaviateObject) objects.get(0)); diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java new file mode 100644 index 000000000..166bd7053 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java @@ -0,0 +1,101 @@ +package io.weaviate.client6.v1.collections.data; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; + +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.FilterTarget; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters.Operator; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; +import io.weaviate.client6.v1.collections.query.QueryReference; + +public record FetchByIdRequest( + String collection, + String id, + boolean includeVector, + List includeVectors, + List returnProperties, + List returnReferences) { + + public FetchByIdRequest(Builder options) { + this( + options.collection, + options.uuid, + options.includeVector, + options.includeVectors, + options.returnProperties, + options.returnReferences); + } + + public static FetchByIdRequest of(String collection, String uuid, Consumer fn) { + var builder = new Builder(collection, uuid); + fn.accept(builder); + return new FetchByIdRequest(builder); + } + + public static class Builder { + private final String collection; + private final String uuid; + + public Builder(String collection, String uuid) { + this.collection = collection; + this.uuid = uuid; + } + + private boolean includeVector; + private List includeVectors; + private List returnProperties = new ArrayList<>(); + private List returnReferences = new ArrayList<>(); + + public final Builder includeVector() { + this.includeVector = true; + return this; + } + + public final Builder includeVectors(String... vectors) { + this.includeVectors = Arrays.asList(vectors); + return this; + } + + public final Builder returnProperties(String... properties) { + this.returnProperties = Arrays.asList(properties); + return this; + } + + public final Builder returnReferences(QueryReference... references) { + this.returnReferences = Arrays.asList(references); + return this; + } + + } + + void appendTo(SearchRequest.Builder req) { + req.setLimit(1); + req.setCollection(collection); + var filter = Filters.newBuilder(); + var target = FilterTarget.newBuilder(); + target.setProperty("_id"); + filter.setTarget(target); + filter.setValueText(id); + filter.setOperator(Operator.OPERATOR_EQUAL); + req.setFilters(filter); + + if (!returnProperties.isEmpty() || !returnReferences.isEmpty()) { + var properties = PropertiesRequest.newBuilder(); + for (String property : returnProperties) { + properties.addNonRefProperties(property); + } + + var references = RefPropertiesRequest.newBuilder(); + for (var ref : returnReferences) { + ref.appendTo(references); + } + properties.addRefProperties(references); + req.setProperties(properties); + } + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java b/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java index 4bbabb193..930ef3836 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/CommonQueryOptions.java @@ -18,6 +18,7 @@ public record CommonQueryOptions( String after, String consistencyLevel /* TODO: use ConsistencyLevel enum */, List returnProperties, + List returnReferences, List returnMetadata) { public CommonQueryOptions(Builder> options) { @@ -28,6 +29,7 @@ public CommonQueryOptions(Builder> options) { options.after, options.consistencyLevel, options.returnProperties, + options.returnReferences, options.returnMetadata); } @@ -39,6 +41,7 @@ public static abstract class Builder> { private String after; private String consistencyLevel; private List returnProperties = new ArrayList<>(); + private List returnReferences = new ArrayList<>(); private List returnMetadata = new ArrayList<>(); public final SELF limit(Integer limit) { @@ -66,6 +69,16 @@ public final SELF consistencyLevel(String consistencyLevel) { return (SELF) this; } + public final SELF returnProperties(String... properties) { + this.returnProperties = Arrays.asList(properties); + return (SELF) this; + } + + public final SELF returnReferences(QueryReference references) { + this.returnReferences = Arrays.asList(references); + return (SELF) this; + } + public final SELF returnMetadata(Metadata... metadata) { this.returnMetadata = Arrays.asList(metadata); return (SELF) this; diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java new file mode 100644 index 000000000..4a2546355 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java @@ -0,0 +1,115 @@ +package io.weaviate.client6.v1.collections.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; + +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesRequest; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesRequest; + +public record QueryReference( + String property, + String collection, + boolean includeVector, List includeVectors, + List returnProperties, + List returnReferences, + List returnMetadata) { + + public QueryReference(Builder options) { + this( + options.property, + options.collection, + options.includeVector, + options.includeVectors, + options.returnProperties, + options.returnReferences, + options.returnMetadata); + } + + public static QueryReference single(String property) { + return single(property, opt -> { + }); + } + + public static QueryReference single(String property, Consumer fn) { + var builder = new Builder(null, property); + fn.accept(builder); + return new QueryReference(builder); + } + + // TODO: check if we can supply mutiple collections + public static QueryReference multi(String property, String collection) { + return multi(collection, property, opt -> { + }); + } + + public static QueryReference multi(String property, String collection, Consumer fn) { + var builder = new Builder(collection, property); + fn.accept(builder); + return new QueryReference(builder); + } + + public static class Builder { + private final String property; + private final String collection; + + public Builder(String collection, String property) { + this.property = property; + this.collection = collection; + } + + private boolean includeVector; + private List includeVectors = new ArrayList<>(); + private List returnProperties = new ArrayList<>(); + private List returnReferences = new ArrayList<>(); + private List returnMetadata = new ArrayList<>(); + + public final Builder includeVector() { + this.includeVector = true; + return this; + } + + public final Builder includeVectors(String... vectors) { + this.includeVectors = Arrays.asList(vectors); + return this; + } + + public final Builder returnProperties(String... properties) { + this.returnProperties = Arrays.asList(properties); + return this; + } + + public final Builder returnReferences(String... references) { + this.returnReferences = Arrays.asList(references); + return this; + } + + public final Builder returnMetadata(Metadata... metadata) { + this.returnMetadata = Arrays.asList(metadata); + return this; + } + } + + public void appendTo(RefPropertiesRequest.Builder references) { + references.setReferenceProperty(property); + if (collection != null) { + references.setTargetCollection(collection); + } + + if (!returnMetadata.isEmpty()) { + var metadata = MetadataRequest.newBuilder(); + returnMetadata.forEach(m -> m.appendTo(metadata)); + references.setMetadata(metadata); + } + + if (!returnProperties.isEmpty()) { + var properties = PropertiesRequest.newBuilder(); + for (String property : returnProperties) { + properties.addNonRefProperties(property); + } + references.setProperties(properties); + } + } +} From 01b5d419dd83ca46a243962781d910a19c1c4d94 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Mon, 28 Apr 2025 14:05:56 +0200 Subject: [PATCH 10/16] fix: use 127 API in get+id filter --- .../io/weaviate/integration/DataITest.java | 5 +- .../v1/collections/data/DataClient.java | 49 +++++++++++++++++-- .../v1/collections/data/FetchByIdRequest.java | 39 +++++++++------ .../v1/collections/object/Vectors.java | 3 ++ .../v1/collections/query/QueryClient.java | 1 + 5 files changed, 77 insertions(+), 20 deletions(-) diff --git a/src/it/java/io/weaviate/integration/DataITest.java b/src/it/java/io/weaviate/integration/DataITest.java index 602e3827b..f0ccd3765 100644 --- a/src/it/java/io/weaviate/integration/DataITest.java +++ b/src/it/java/io/weaviate/integration/DataITest.java @@ -38,7 +38,10 @@ public void testCreateGetDelete() throws IOException { .id(id) .vectors(Vectors.of(VECTOR_INDEX, vector))); - var object = artists.data.get(id, query -> query.includeVector()); + var object = artists.data.get(id, query -> query + .returnProperties("name") + .includeVector()); + Assertions.assertThat(object) .as("object exists after insert").get() .satisfies(obj -> { diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index e54606c3b..cfcef0e34 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.time.OffsetDateTime; import java.util.Date; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.function.Consumer; @@ -84,15 +85,44 @@ public Optional> get(String id, Consumer> findById(FetchByIdRequest request) { var req = SearchRequest.newBuilder(); + req.setUses127Api(true); + req.setUses125Api(true); + req.setUses123Api(true); request.appendTo(req); var result = grpcClient.grpc.search(req.build()); - var objects = result.getResultsList().stream().map(r -> readPropertiesResult(r.getProperties())).toList(); + var objects = result.getResultsList().stream().map(r -> { + var tempObj = readPropertiesResult(r.getProperties()); + MetadataResult meta = r.getMetadata(); + Vectors vectors; + if (!meta.getVectorBytes().isEmpty()) { + vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); + } else { + vectors = Vectors.of(meta.getVectorsList().stream().collect( + Collectors.toMap( + io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, + v -> { + if (v.getType().equals(VectorType.VECTOR_TYPE_SINGLE_FP32)) { + return GRPC.fromByteString(v.getVectorBytes()); + } else { + return GRPC.fromByteStringMulti(v.getVectorBytes()); + } + }))); + } + var metadata = new ObjectMetadata(meta.getId(), vectors); + return new WeaviateObject<>( + tempObj.collection(), + tempObj.properties(), + tempObj.references(), + metadata); + }).toList(); + if (objects.isEmpty()) { + return Optional.empty(); + } return Optional.ofNullable((WeaviateObject) objects.get(0)); } private static WeaviateObject readPropertiesResult(PropertiesResult res) { var collection = res.getTargetCollection(); - var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); var referenceProperties = res.getRefPropsList().stream() .collect(Collectors.toMap( @@ -129,8 +159,14 @@ private static WeaviateObject readPropertiesResult(PropertiesResult res) { * (de-)serialized by {@link Gson}. */ private static Map convertProtoMap(Map map) { - return map.entrySet().stream().collect(Collectors.toMap( - Map.Entry::getKey, e -> convertProtoValue(e.getValue()))); + return map.entrySet().stream() + // We cannot use Collectors.toMap() here, because convertProtoValue may + // return null (a collection property can be null), which breaks toMap(). + // See: https://bugs.openjdk.org/browse/JDK-8148463 + .collect( + HashMap::new, + (m, e) -> m.put(e.getKey(), convertProtoValue(e.getValue())), + HashMap::putAll); } /** @@ -139,7 +175,10 @@ private static Map convertProtoMap(Map map) { * integer values. */ private static Object convertProtoValue(Value value) { - if (value.hasTextValue()) { + if (value.hasNullValue()) { + // return value.getNullValue(); + return null; + } else if (value.hasTextValue()) { return value.getTextValue(); } else if (value.hasBoolValue()) { return value.getBoolValue(); diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java index 166bd7053..4be17d63f 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java @@ -5,9 +5,12 @@ import java.util.List; import java.util.function.Consumer; +import com.google.protobuf.util.JsonFormat; + import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.FilterTarget; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters.Operator; +import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesRequest; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; @@ -47,7 +50,7 @@ public Builder(String collection, String uuid) { } private boolean includeVector; - private List includeVectors; + private List includeVectors = new ArrayList<>(); private List returnProperties = new ArrayList<>(); private List returnReferences = new ArrayList<>(); @@ -76,26 +79,34 @@ public final Builder returnReferences(QueryReference... references) { void appendTo(SearchRequest.Builder req) { req.setLimit(1); req.setCollection(collection); - var filter = Filters.newBuilder(); - var target = FilterTarget.newBuilder(); - target.setProperty("_id"); - filter.setTarget(target); - filter.setValueText(id); - filter.setOperator(Operator.OPERATOR_EQUAL); - req.setFilters(filter); + + req.setFilters(Filters.newBuilder() + .setTarget(FilterTarget.newBuilder().setProperty("_id")) + .setValueText(id) + .setOperator(Operator.OPERATOR_EQUAL)); if (!returnProperties.isEmpty() || !returnReferences.isEmpty()) { var properties = PropertiesRequest.newBuilder(); - for (String property : returnProperties) { - properties.addNonRefProperties(property); + + if (!returnProperties.isEmpty()) { + properties.addAllNonRefProperties(returnProperties); } - var references = RefPropertiesRequest.newBuilder(); - for (var ref : returnReferences) { - ref.appendTo(references); + if (!returnReferences.isEmpty()) { + var references = RefPropertiesRequest.newBuilder(); + returnReferences.forEach(r -> r.appendTo(references)); + properties.addRefProperties(references); } - properties.addRefProperties(references); req.setProperties(properties); } + + // Always request UUID back in this request. + var metadata = MetadataRequest.newBuilder().setUuid(true); + if (includeVector) { + metadata.setVector(true); + } else if (!includeVectors.isEmpty()) { + metadata.addAllVectors(includeVectors); + } + req.setMetadata(metadata); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java b/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java index 02830ab6e..db0669e90 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/Vectors.java @@ -6,10 +6,13 @@ import java.util.Optional; import java.util.function.Consumer; +import lombok.ToString; + /** * Vectors is an abstraction over named vectors. * It may contain both 1-dimensional and 2-dimensional vectors. */ +@ToString public class Vectors { private static final String DEFAULT = "default"; diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java index 66a8a6540..4f9f2b748 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java @@ -9,6 +9,7 @@ import java.util.stream.Collectors; import com.google.gson.Gson; +import com.google.protobuf.util.JsonFormat; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoProperties.Value; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataResult; From 918236830f8fe21dfbc939fae76be1c30da70911 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Mon, 28 Apr 2025 16:41:40 +0200 Subject: [PATCH 11/16] fix: merge multi-target references There may be a bug in the Weaviate server, so part of the test is commented out. --- .../weaviate/integration/ReferencesITest.java | 7 ++ .../v1/collections/data/DataClient.java | 87 ++++++++++++------- .../v1/collections/data/FetchByIdRequest.java | 8 +- .../collections/object/ObjectReference.java | 2 +- .../v1/collections/query/QueryClient.java | 1 - .../v1/collections/query/QueryReference.java | 8 ++ 6 files changed, 78 insertions(+), 35 deletions(-) diff --git a/src/it/java/io/weaviate/integration/ReferencesITest.java b/src/it/java/io/weaviate/integration/ReferencesITest.java index 56a12bf85..31ca2b297 100644 --- a/src/it/java/io/weaviate/integration/ReferencesITest.java +++ b/src/it/java/io/weaviate/integration/ReferencesITest.java @@ -93,6 +93,8 @@ public void testReferences() throws IOException { var gotAlex = artists.data.get(alex.metadata().id(), opt -> opt.returnReferences( QueryReference.multi("hasAwards", nsOscar, + ref -> ref.returnMetadata(MetadataField.ID)), + QueryReference.multi("hasAwards", nsGrammy, ref -> ref.returnMetadata(MetadataField.ID)))); Assertions.assertThat(gotAlex).get() @@ -102,6 +104,11 @@ public void testReferences() throws IOException { .extracting(ObjectReference::objects, InstanceOfAssertFactories.list(WeaviateObject.class)) .extracting(objects -> objects.metadata().id()) .containsOnly( + // INVESTIGATE: When references to 2+ collections are requested, + // seems to Weaviate only return references to the first one in the list. + // In this case we request { "hasAwards": Oscars } and { "hasAwards": Grammys } + // so the latter will not be in the response. + // // grammy_1.metadata().id(), grammy_2.metadata().id(), oscar_1.metadata().id(), oscar_2.metadata().id()); } diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index cfcef0e34..2afb9f472 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -4,10 +4,12 @@ import java.time.OffsetDateTime; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -24,7 +26,6 @@ import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoProperties.Value; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataResult; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.PropertiesResult; -import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.RefPropertiesResult; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.SearchRequest; import io.weaviate.client6.internal.GRPC; import io.weaviate.client6.internal.GrpcClient; @@ -122,36 +123,62 @@ private Optional> findById(FetchByIdRequest request) { } private static WeaviateObject readPropertiesResult(PropertiesResult res) { - var collection = res.getTargetCollection(); - var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); - var referenceProperties = res.getRefPropsList().stream() - .collect(Collectors.toMap( - RefPropertiesResult::getPropName, - ref -> { - var refObjects = ref.getPropertiesList().stream() - .map(DataClient::readPropertiesResult) - .toList(); - return new ObjectReference(refObjects); - })); - - MetadataResult meta = res.getMetadata(); - Vectors vectors; - if (meta.getVectorBytes() != null) { - vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); - } else { - vectors = Vectors.of(meta.getVectorsList().stream().collect( - Collectors.toMap( - io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, - v -> { - if (v.getType().equals(VectorType.VECTOR_TYPE_MULTI_FP32)) { - return GRPC.fromByteString(v.getVectorBytes()); - } else { - return GRPC.fromByteStringMulti(v.getVectorBytes()); - } - }))); + try { + + var collection = res.getTargetCollection(); + var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); + + // In case a reference is multi-target, there will be a separate + // "reference property" for each of the targets, so instead of + // `collect` we need to `reduce` the map, merging related references + // as we go. + // I.e. { "ref": A-1 } , { "ref": B-1 } => { "ref": [A-1, B-1] } + var referenceProperties = res.getRefPropsList().stream().reduce( + new HashMap(), + (map, ref) -> { + var refObjects = ref.getPropertiesList().stream() + .map(DataClient::readPropertiesResult) + .toList(); + + // Merge ObjectReferences by joining the underlying WeaviateObjects. + map.merge( + ref.getPropName(), + new ObjectReference((List>) refObjects), + (left, right) -> { + var joined = Stream.concat( + left.objects().stream(), + right.objects().stream()).toList(); + return new ObjectReference(joined); + }); + return map; + }, + (left, right) -> { + left.putAll(right); + return left; + }); + + MetadataResult meta = res.getMetadata(); + Vectors vectors; + if (meta.getVectorBytes() != null) { + vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); + } else { + vectors = Vectors.of(meta.getVectorsList().stream().collect( + Collectors.toMap( + io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, + v -> { + if (v.getType().equals(VectorType.VECTOR_TYPE_MULTI_FP32)) { + return GRPC.fromByteString(v.getVectorBytes()); + } else { + return GRPC.fromByteStringMulti(v.getVectorBytes()); + } + }))); + } + var metadata = new ObjectMetadata(meta.getId(), vectors); + return new WeaviateObject<>(collection, objectProperties, referenceProperties, metadata); + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException(e); } - var metadata = new ObjectMetadata(meta.getId(), vectors); - return new WeaviateObject<>(collection, objectProperties, referenceProperties, metadata); } /* diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java index 4be17d63f..5016fdbcc 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java @@ -93,9 +93,11 @@ void appendTo(SearchRequest.Builder req) { } if (!returnReferences.isEmpty()) { - var references = RefPropertiesRequest.newBuilder(); - returnReferences.forEach(r -> r.appendTo(references)); - properties.addRefProperties(references); + returnReferences.forEach(r -> { + var references = RefPropertiesRequest.newBuilder(); + r.appendTo(references); + properties.addRefProperties(references); + }); } req.setProperties(properties); } diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java index 43333ed4c..bc5c82c04 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectReference.java @@ -2,5 +2,5 @@ import java.util.List; -public record ObjectReference(List> objects) { +public record ObjectReference(List> objects) { } diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java index 4f9f2b748..66a8a6540 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java @@ -9,7 +9,6 @@ import java.util.stream.Collectors; import com.google.gson.Gson; -import com.google.protobuf.util.JsonFormat; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoProperties.Value; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoSearchGet.MetadataResult; diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java index 4a2546355..a9dfdb3d7 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java @@ -51,6 +51,14 @@ public static QueryReference multi(String property, String collection, Consumer< return new QueryReference(builder); } + public static QueryReference[] multi(String property, Consumer fn, String... collections) { + return Arrays.stream(collections).map(collection -> { + var builder = new Builder(collection, property); + fn.accept(builder); + return new QueryReference(builder); + }).toArray(QueryReference[]::new); + } + public static class Builder { private final String property; private final String collection; From 3d1c6a6b801d36e73b28b3693731f162ca9279b0 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 29 Apr 2025 12:46:33 +0200 Subject: [PATCH 12/16] feat: request nested references and their properties --- .../weaviate/integration/ReferencesITest.java | 66 +++++++++++ .../v1/collections/data/DataClient.java | 104 +++++++++--------- .../v1/collections/data/FetchByIdRequest.java | 2 - .../v1/collections/query/QueryReference.java | 21 +++- 4 files changed, 130 insertions(+), 63 deletions(-) diff --git a/src/it/java/io/weaviate/integration/ReferencesITest.java b/src/it/java/io/weaviate/integration/ReferencesITest.java index 31ca2b297..533d2693d 100644 --- a/src/it/java/io/weaviate/integration/ReferencesITest.java +++ b/src/it/java/io/weaviate/integration/ReferencesITest.java @@ -112,4 +112,70 @@ public void testReferences() throws IOException { // grammy_1.metadata().id(), grammy_2.metadata().id(), oscar_1.metadata().id(), oscar_2.metadata().id()); } + + @Test + public void testNestedReferences() throws IOException { + // Arrange: create collection with cross-references + var nsArtists = ns("Artists"); + var nsGrammy = ns("Grammy"); + var nsAcademy = ns("Academy"); + + client.collections.create(nsAcademy, + opt -> opt + .properties(Property.text("ceo"))); + + // Act: create Artists collection with hasAwards reference + client.collections.create(nsGrammy, + col -> col + .properties(Property.reference("presentedBy", nsAcademy))); + + client.collections.create(nsArtists, + col -> col + .properties( + Property.text("name"), + Property.integer("age"), + Property.reference("hasAwards", nsGrammy))); + + var artists = client.collections.use(nsArtists); + var grammies = client.collections.use(nsGrammy); + var academies = client.collections.use(nsAcademy); + + // Act: insert some data + var musicAcademy = academies.data.insert(Map.of("ceo", "Harvy Mason")); + + var grammy_1 = grammies.data.insert(Map.of(), + opt -> opt.reference("presentedBy", Reference.objects(musicAcademy))); + + var alex = artists.data.insert( + Map.of("name", "Alex"), + opt -> opt + .reference("hasAwards", Reference.objects(grammy_1))); + + // Assert: fetch nested references + var gotAlex = artists.data.get(alex.metadata().id(), + opt -> opt.returnReferences( + QueryReference.single("hasAwards", + ref -> ref + // Name of the CEO of the presenting academy + .returnReferences( + QueryReference.single("presentedBy", r -> r.returnProperties("ceo"))) + // Grammy ID + .returnMetadata(MetadataField.ID)))); + + Assertions.assertThat(gotAlex).get() + .as("Artists: fetch by id including nested references") + .extracting(WeaviateObject::references, InstanceOfAssertFactories.map(String.class, ObjectReference.class)) + .as("hasAwards object reference").extractingByKey("hasAwards") + .extracting(ObjectReference::objects, InstanceOfAssertFactories.list(WeaviateObject.class)) + .hasSize(1).allSatisfy(award -> Assertions.assertThat(award) + .returns(grammy_1.metadata().id(), g -> g.metadata().id()) + .extracting(WeaviateObject::references, + InstanceOfAssertFactories.map(String.class, ObjectReference.class)) + .extractingByKey("presentedBy") + .extracting(ObjectReference::objects, InstanceOfAssertFactories.list(WeaviateObject.class)) + .hasSize(1).extracting(WeaviateObject::properties) + .allSatisfy(properties -> Assertions.assertThat(properties) + .asInstanceOf(InstanceOfAssertFactories.map(String.class, Object.class)) + .containsEntry("ceo", "Harvy Mason"))); + } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java index 2afb9f472..e8878e1e8 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/DataClient.java @@ -123,62 +123,56 @@ private Optional> findById(FetchByIdRequest request) { } private static WeaviateObject readPropertiesResult(PropertiesResult res) { - try { - - var collection = res.getTargetCollection(); - var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); - - // In case a reference is multi-target, there will be a separate - // "reference property" for each of the targets, so instead of - // `collect` we need to `reduce` the map, merging related references - // as we go. - // I.e. { "ref": A-1 } , { "ref": B-1 } => { "ref": [A-1, B-1] } - var referenceProperties = res.getRefPropsList().stream().reduce( - new HashMap(), - (map, ref) -> { - var refObjects = ref.getPropertiesList().stream() - .map(DataClient::readPropertiesResult) - .toList(); - - // Merge ObjectReferences by joining the underlying WeaviateObjects. - map.merge( - ref.getPropName(), - new ObjectReference((List>) refObjects), - (left, right) -> { - var joined = Stream.concat( - left.objects().stream(), - right.objects().stream()).toList(); - return new ObjectReference(joined); - }); - return map; - }, - (left, right) -> { - left.putAll(right); - return left; - }); - - MetadataResult meta = res.getMetadata(); - Vectors vectors; - if (meta.getVectorBytes() != null) { - vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); - } else { - vectors = Vectors.of(meta.getVectorsList().stream().collect( - Collectors.toMap( - io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, - v -> { - if (v.getType().equals(VectorType.VECTOR_TYPE_MULTI_FP32)) { - return GRPC.fromByteString(v.getVectorBytes()); - } else { - return GRPC.fromByteStringMulti(v.getVectorBytes()); - } - }))); - } - var metadata = new ObjectMetadata(meta.getId(), vectors); - return new WeaviateObject<>(collection, objectProperties, referenceProperties, metadata); - } catch (Exception e) { - e.printStackTrace(); - throw new RuntimeException(e); + var collection = res.getTargetCollection(); + var objectProperties = convertProtoMap(res.getNonRefProps().getFieldsMap()); + + // In case a reference is multi-target, there will be a separate + // "reference property" for each of the targets, so instead of + // `collect` we need to `reduce` the map, merging related references + // as we go. + // I.e. { "ref": A-1 } , { "ref": B-1 } => { "ref": [A-1, B-1] } + var referenceProperties = res.getRefPropsList().stream().reduce( + new HashMap(), + (map, ref) -> { + var refObjects = ref.getPropertiesList().stream() + .map(DataClient::readPropertiesResult) + .toList(); + + // Merge ObjectReferences by joining the underlying WeaviateObjects. + map.merge( + ref.getPropName(), + new ObjectReference((List>) refObjects), + (left, right) -> { + var joined = Stream.concat( + left.objects().stream(), + right.objects().stream()).toList(); + return new ObjectReference(joined); + }); + return map; + }, + (left, right) -> { + left.putAll(right); + return left; + }); + + MetadataResult meta = res.getMetadata(); + Vectors vectors; + if (meta.getVectorBytes() != null) { + vectors = Vectors.of(GRPC.fromByteString(meta.getVectorBytes())); + } else { + vectors = Vectors.of(meta.getVectorsList().stream().collect( + Collectors.toMap( + io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Vectors::getName, + v -> { + if (v.getType().equals(VectorType.VECTOR_TYPE_MULTI_FP32)) { + return GRPC.fromByteString(v.getVectorBytes()); + } else { + return GRPC.fromByteStringMulti(v.getVectorBytes()); + } + }))); } + var metadata = new ObjectMetadata(meta.getId(), vectors); + return new WeaviateObject<>(collection, objectProperties, referenceProperties, metadata); } /* diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java index 5016fdbcc..28cb8635f 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/FetchByIdRequest.java @@ -5,8 +5,6 @@ import java.util.List; import java.util.function.Consumer; -import com.google.protobuf.util.JsonFormat; - import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.FilterTarget; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters; import io.weaviate.client6.grpc.protocol.v1.WeaviateProtoBase.Filters.Operator; diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java index a9dfdb3d7..82902f491 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryReference.java @@ -14,7 +14,7 @@ public record QueryReference( String collection, boolean includeVector, List includeVectors, List returnProperties, - List returnReferences, + List returnReferences, List returnMetadata) { public QueryReference(Builder options) { @@ -71,7 +71,7 @@ public Builder(String collection, String property) { private boolean includeVector; private List includeVectors = new ArrayList<>(); private List returnProperties = new ArrayList<>(); - private List returnReferences = new ArrayList<>(); + private List returnReferences = new ArrayList<>(); private List returnMetadata = new ArrayList<>(); public final Builder includeVector() { @@ -89,7 +89,7 @@ public final Builder returnProperties(String... properties) { return this; } - public final Builder returnReferences(String... references) { + public final Builder returnReferences(QueryReference... references) { this.returnReferences = Arrays.asList(references); return this; } @@ -112,10 +112,19 @@ public void appendTo(RefPropertiesRequest.Builder references) { references.setMetadata(metadata); } - if (!returnProperties.isEmpty()) { + if (!returnProperties.isEmpty() || !returnReferences.isEmpty()) { var properties = PropertiesRequest.newBuilder(); - for (String property : returnProperties) { - properties.addNonRefProperties(property); + + if (!returnProperties.isEmpty()) { + properties.addAllNonRefProperties(returnProperties); + } + + if (!returnReferences.isEmpty()) { + returnReferences.forEach(r -> { + var ref = RefPropertiesRequest.newBuilder(); + r.appendTo(ref); + properties.addRefProperties(ref); + }); } references.setProperties(properties); } From 2bcf9a4ffcb7f3f33afd06ca606fa2dd9238d2ed Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 29 Apr 2025 13:15:20 +0200 Subject: [PATCH 13/16] chore: fix post-merge duplications --- .../aggregate/AggregateClient.java | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java index 87e365d8d..cfc774eae 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/aggregate/AggregateClient.java @@ -53,23 +53,6 @@ public AggregateResponse nearVector( return new AggregateUnmarshaler(reply).single(); } - public AggregateGroupByResponse nearVector( - Float[] vector, - AggregateGroupByRequest.GroupBy groupBy, - Consumer options) { - var aggregation = AggregateRequest.with(collectionName, options); - var nearVector = NearVector.with(vector, opt -> { - }); - - var req = new AggregateMarshaler(aggregation.collectionName()) - .addAggregation(aggregation) - .addGroupBy(groupBy) - .addNearVector(nearVector) - .marshal(); - var reply = grpcClient.grpc.aggregate(req); - return new AggregateUnmarshaler(reply).grouped(); - } - public AggregateGroupByResponse nearVector( Float[] vector, Consumer nearVectorOptions, @@ -103,21 +86,4 @@ public AggregateGroupByResponse nearVector( var reply = grpcClient.grpc.aggregate(req); return new AggregateUnmarshaler(reply).grouped(); } - - public AggregateGroupByResponse nearVector( - Float[] vector, - Consumer nearVectorOptions, - AggregateGroupByRequest.GroupBy groupBy, - Consumer options) { - var aggregation = AggregateRequest.with(collectionName, options); - var nearVector = NearVector.with(vector, nearVectorOptions); - - var req = new AggregateMarshaler(aggregation.collectionName()) - .addAggregation(aggregation) - .addGroupBy(groupBy) - .addNearVector(nearVector) - .marshal(); - var reply = grpcClient.grpc.aggregate(req); - return new AggregateUnmarshaler(reply).grouped(); - } } From 9acbbdaa9cb3cc49dc3175cd4bfe8bf09d7356d3 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 29 Apr 2025 13:18:25 +0200 Subject: [PATCH 14/16] chore: fix javadoc --- .../client6/v1/collections/data/InsertObjectRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java b/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java index 18623c41e..a04f993ea 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java +++ b/src/main/java/io/weaviate/client6/v1/collections/data/InsertObjectRequest.java @@ -59,7 +59,7 @@ public Builder id(String id) { /** * Supply one or more (named) vectors. Calls to {@link #vectors} are not - * chainable. Use {@link Vectors#of(Consumer) to pass multiple vectors. + * chainable. Use {@link Vectors#of(Consumer)} to pass multiple vectors. */ public Builder vectors(Vectors vectors) { this.vectors = vectors; From 9a6cf10082bcd25d1faa64fdac694eacd596e253 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 30 Apr 2025 17:49:22 +0200 Subject: [PATCH 15/16] chore: make Builder properties private --- .../client6/v1/collections/Collection.java | 23 +++++++++---------- .../v1/collections/CollectionsClient.java | 2 +- .../weaviate/client6/v1/collections/HNSW.java | 16 ++++++------- .../client6/v1/collections/VectorIndex.java | 2 +- .../collections/aggregate/TopOccurrences.java | 0 .../v1/collections/object/ObjectMetadata.java | 4 ++-- 6 files changed, 23 insertions(+), 24 deletions(-) delete mode 100644 src/main/java/io/weaviate/client6/v1/collections/aggregate/TopOccurrences.java diff --git a/src/main/java/io/weaviate/client6/v1/collections/Collection.java b/src/main/java/io/weaviate/client6/v1/collections/Collection.java index ad9588b76..d0fe10903 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/Collection.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Collection.java @@ -9,42 +9,41 @@ public record Collection(String name, List properties, Vectors vectors) { - public static Collection with(String name, Consumer options) { - var config = new Configuration(options); + public static Collection with(String name, Consumer options) { + var config = new Builder(options); return new Collection(name, config.properties, config.vectors); } - // Tucked Builder for additional collection configuration. - public static class Configuration { - public List properties = new ArrayList<>(); - public Vectors vectors; + public static class Builder { + private List properties = new ArrayList<>(); + private Vectors vectors; - public Configuration properties(Property... properties) { + public Builder properties(Property... properties) { this.properties = Arrays.asList(properties); return this; } - public Configuration vectors(Vectors vectors) { + public Builder vectors(Vectors vectors) { this.vectors = vectors; return this; } - public Configuration vector(VectorIndex vector) { + public Builder vector(VectorIndex vector) { this.vectors = Vectors.of(vector); return this; } - public Configuration vector(String name, VectorIndex vector) { + public Builder vector(String name, VectorIndex vector) { this.vectors = new Vectors(name, vector); return this; } - public Configuration vectors(Consumer named) { + public Builder vectors(Consumer named) { this.vectors = Vectors.with(named); return this; } - Configuration(Consumer options) { + Builder(Consumer options) { options.accept(this); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java index 9ff438128..052af3559 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java @@ -116,7 +116,7 @@ public void create(String name) throws IOException { }); } - public void create(String name, Consumer options) throws IOException { + public void create(String name, Consumer options) throws IOException { var collection = Collection.with(name, options); ClassicHttpRequest httpPost = ClassicRequestBuilder .post(config.baseUrl() + "/schema") diff --git a/src/main/java/io/weaviate/client6/v1/collections/HNSW.java b/src/main/java/io/weaviate/client6/v1/collections/HNSW.java index cac6b45b4..938c44ffa 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/HNSW.java +++ b/src/main/java/io/weaviate/client6/v1/collections/HNSW.java @@ -17,26 +17,26 @@ public enum Distance { this(null, null); } - static HNSW with(Consumer options) { - var opt = new Options(options); + static HNSW with(Consumer options) { + var opt = new Builder(options); return new HNSW(opt.distance, opt.skip); } - public static class Options { - public Distance distance; - public Boolean skip; + public static class Builder { + private Distance distance; + private Boolean skip; - public Options distance(Distance distance) { + public Builder distance(Distance distance) { this.distance = distance; return this; } - public Options disableIndexation() { + public Builder disableIndexation() { this.skip = true; return this; } - public Options(Consumer options) { + public Builder(Consumer options) { options.accept(this); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/VectorIndex.java b/src/main/java/io/weaviate/client6/v1/collections/VectorIndex.java index 5db348263..b8aea7f0c 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/VectorIndex.java +++ b/src/main/java/io/weaviate/client6/v1/collections/VectorIndex.java @@ -29,7 +29,7 @@ public static IndexingStrategy hnsw() { return new HNSW(); } - public static IndexingStrategy hnsw(Consumer options) { + public static IndexingStrategy hnsw(Consumer options) { return HNSW.with(options); } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/aggregate/TopOccurrences.java b/src/main/java/io/weaviate/client6/v1/collections/aggregate/TopOccurrences.java deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java index e4cf36602..61ffcb9de 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java +++ b/src/main/java/io/weaviate/client6/v1/collections/object/ObjectMetadata.java @@ -10,8 +10,8 @@ public static ObjectMetadata with(Consumer options) { } public static class Builder { - public String id; - public Vectors vectors; + private String id; + private Vectors vectors; public Builder id(String id) { this.id = id; From c6b8e35af5809b8a8e377759026fc5da91d39a8f Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Tue, 6 May 2025 13:06:06 +0200 Subject: [PATCH 16/16] feat: add Contextionary vectorizer and simple NearText search --- .../io/weaviate/containers/Container.java | 5 ++- .../io/weaviate/containers/Contextionary.java | 2 +- .../java/io/weaviate/containers/Weaviate.java | 6 +++ .../integration/CollectionsITest.java | 4 +- ...VectorQueryITest.java => SearchITest.java} | 41 +++++++++++++++++-- .../codec/grpc/v1/SearchMarshaler.java | 19 +++++++++ .../v1/collections/CollectionsClient.java | 1 + .../collections/ContextionaryVectorizer.java | 37 +++++++++++++++++ .../client6/v1/collections/Vectorizer.java | 10 +++++ .../v1/collections/query/NearText.java | 35 ++++++++++++++++ .../v1/collections/query/QueryClient.java | 20 +++++++++ 11 files changed, 171 insertions(+), 9 deletions(-) rename src/it/java/io/weaviate/integration/{NearVectorQueryITest.java => SearchITest.java} (73%) create mode 100644 src/main/java/io/weaviate/client6/v1/collections/ContextionaryVectorizer.java create mode 100644 src/main/java/io/weaviate/client6/v1/collections/query/NearText.java diff --git a/src/it/java/io/weaviate/containers/Container.java b/src/it/java/io/weaviate/containers/Container.java index 9ef69cfe8..974e56a9e 100644 --- a/src/it/java/io/weaviate/containers/Container.java +++ b/src/it/java/io/weaviate/containers/Container.java @@ -54,6 +54,8 @@ public static class Group implements Startable { private Group(Weaviate weaviate, GenericContainer... containers) { this.weaviate = weaviate; this.containers = Arrays.asList(containers); + + weaviate.dependsOn(containers); setSharedNetwork(); } @@ -63,8 +65,7 @@ public WeaviateClient getClient() { @Override public void start() { - containers.forEach(GenericContainer::start); - weaviate.start(); + weaviate.start(); // testcontainers will resolve dependencies } @Override diff --git a/src/it/java/io/weaviate/containers/Contextionary.java b/src/it/java/io/weaviate/containers/Contextionary.java index 76ec5aefd..69abde7df 100644 --- a/src/it/java/io/weaviate/containers/Contextionary.java +++ b/src/it/java/io/weaviate/containers/Contextionary.java @@ -35,7 +35,7 @@ public Contextionary build() { .withEnv("EXTENSIONS_STORAGE_ORIGIN", "http://weaviate:8080") .withEnv("NEIGHBOR_OCCURRENCE_IGNORE_PERCENTILE", "5") .withEnv("ENABLE_COMPOUND_SPLITTING", "'false'"); - container.withCreateContainerCmdModifier(cmd -> cmd.withHostName("contextionary")); + container.withCreateContainerCmdModifier(cmd -> cmd.withHostName(HOST_NAME)); return container; } } diff --git a/src/it/java/io/weaviate/containers/Weaviate.java b/src/it/java/io/weaviate/containers/Weaviate.java index 6126d5150..5c7c8e648 100644 --- a/src/it/java/io/weaviate/containers/Weaviate.java +++ b/src/it/java/io/weaviate/containers/Weaviate.java @@ -69,7 +69,13 @@ public Builder withDefaultVectorizer(String module) { return this; } + public Builder withContextionary() { + addModule(Contextionary.MODULE); + return this; + } + public Builder withContextionaryUrl(String url) { + withContextionary(); contextionaryUrl = url; return this; } diff --git a/src/it/java/io/weaviate/integration/CollectionsITest.java b/src/it/java/io/weaviate/integration/CollectionsITest.java index 759e0908c..80c19c36c 100644 --- a/src/it/java/io/weaviate/integration/CollectionsITest.java +++ b/src/it/java/io/weaviate/integration/CollectionsITest.java @@ -9,7 +9,6 @@ import io.weaviate.ConcurrentTest; import io.weaviate.client6.WeaviateClient; import io.weaviate.client6.v1.collections.Collection; -import io.weaviate.client6.v1.collections.NoneVectorizer; import io.weaviate.client6.v1.collections.Property; import io.weaviate.client6.v1.collections.VectorIndex; import io.weaviate.client6.v1.collections.VectorIndex.IndexType; @@ -17,6 +16,7 @@ import io.weaviate.client6.v1.collections.Vectorizer; import io.weaviate.client6.v1.collections.Vectors; import io.weaviate.containers.Container; +import io.weaviate.containers.Contextionary; public class CollectionsITest extends ConcurrentTest { private static WeaviateClient client = Container.WEAVIATE.getClient(); @@ -36,7 +36,7 @@ public void testCreateGetDelete() throws IOException { .extracting(Collection::vectors).extracting(Vectors::getDefault) .as("default vector").satisfies(defaultVector -> { Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer) - .as("has none vectorizer").isInstanceOf(NoneVectorizer.class); + .as("has none vectorizer").isInstanceOf(Contextionary.class); Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration) .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type); }); diff --git a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java b/src/it/java/io/weaviate/integration/SearchITest.java similarity index 73% rename from src/it/java/io/weaviate/integration/NearVectorQueryITest.java rename to src/it/java/io/weaviate/integration/SearchITest.java index 863652781..2496e2801 100644 --- a/src/it/java/io/weaviate/integration/NearVectorQueryITest.java +++ b/src/it/java/io/weaviate/integration/SearchITest.java @@ -9,7 +9,9 @@ import org.assertj.core.api.Assertions; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TestRule; import io.weaviate.ConcurrentTest; import io.weaviate.client6.WeaviateClient; @@ -22,9 +24,17 @@ import io.weaviate.client6.v1.collections.query.MetadataField; import io.weaviate.client6.v1.collections.query.NearVector; import io.weaviate.containers.Container; - -public class NearVectorQueryITest extends ConcurrentTest { - private static final WeaviateClient client = Container.WEAVIATE.getClient(); +import io.weaviate.containers.Container.Group; +import io.weaviate.containers.Contextionary; +import io.weaviate.containers.Weaviate; + +public class SearchITest extends ConcurrentTest { + private static final Group compose = Container.compose( + Weaviate.custom().withContextionaryUrl(Contextionary.URL).build(), + Container.CONTEXTIONARY); + @ClassRule // Bind containers to lifetime to the test + public static final TestRule _rule = compose.asTestRule(); + private static final WeaviateClient client = compose.getClient(); private static final String COLLECTION = unique("Things"); private static final String VECTOR_INDEX = "bring_your_own"; @@ -81,7 +91,6 @@ public void testNearVector_groupBy() { Assertions.assertThat(result.objects) .as("object belongs a group") .allMatch(obj -> result.groups.get(obj.belongsToGroup).objects().contains(obj)); - } /** @@ -117,4 +126,28 @@ private static void createTestCollection() throws IOException { .properties(Property.text("category")) .vector(VECTOR_INDEX, new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none()))); } + + @Test + public void testNearText() throws IOException { + var nsSongs = ns("Songs"); + client.collections.create(nsSongs, + col -> col + .properties(Property.text("title")) + .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.contextionary()))); + + var songs = client.collections.use(nsSongs); + songs.data.insert(Map.of("title", "Yellow Submarine")); + songs.data.insert(Map.of("title", "Run Through The Jungle")); + songs.data.insert(Map.of("title", "Welcome To The Jungle")); + + var result = songs.query.nearText("forest", + opt -> opt + .distance(0.5f) + .returnProperties("title")); + + Assertions.assertThat(result.objects).hasSize(2) + .extracting(obj -> obj.properties).allSatisfy( + properties -> Assertions.assertThat(properties) + .allSatisfy((_k, v) -> Assertions.assertThat((String) v).contains("Jungle"))); + } } diff --git a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java index ca6769f01..b1d532218 100644 --- a/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java +++ b/src/main/java/io/weaviate/client6/internal/codec/grpc/v1/SearchMarshaler.java @@ -10,6 +10,7 @@ import io.weaviate.client6.internal.GRPC; import io.weaviate.client6.internal.codec.grpc.GrpcMarshaler; import io.weaviate.client6.v1.collections.query.CommonQueryOptions; +import io.weaviate.client6.v1.collections.query.NearText; import io.weaviate.client6.v1.collections.query.NearVector; public class SearchMarshaler implements GrpcMarshaler { @@ -43,10 +44,28 @@ public SearchMarshaler addNearVector(NearVector nv) { nearVector.setDistance(nv.distance()); } + // TODO: add targets, vector_for_targets req.setNearVector(nearVector); return this; } + public SearchMarshaler addNearText(NearText nt) { + setCommon(nt.common()); + + var nearText = WeaviateProtoBaseSearch.NearTextSearch.newBuilder(); + nearText.addAllQuery(nt.text()); + + if (nt.certainty() != null) { + nearText.setCertainty(nt.certainty()); + } else if (nt.distance() != null) { + nearText.setDistance(nt.distance()); + } + + // TODO: add move_to, move_away, targets + req.setNearText(nearText); + return this; + } + private void setCommon(CommonQueryOptions o) { if (o.limit() != null) { req.setLimit(o.limit()); diff --git a/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java b/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java index 052af3559..ea8ac14ab 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/CollectionsClient.java @@ -75,6 +75,7 @@ private static class VectorizerSerde @Override public Vectorizer deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + // TODO: deserialize different kinds of vectorizers return Vectorizer.none(); } diff --git a/src/main/java/io/weaviate/client6/v1/collections/ContextionaryVectorizer.java b/src/main/java/io/weaviate/client6/v1/collections/ContextionaryVectorizer.java new file mode 100644 index 000000000..1bb580ada --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/ContextionaryVectorizer.java @@ -0,0 +1,37 @@ +package io.weaviate.client6.v1.collections; + +import java.util.Map; +import java.util.function.Consumer; + +import com.google.gson.annotations.SerializedName; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public class ContextionaryVectorizer extends Vectorizer { + @SerializedName("text2vec-contextionary") + private Map configuration; + + public static ContextionaryVectorizer of() { + return new Builder().build(); + } + + public static ContextionaryVectorizer of(Consumer fn) { + var builder = new Builder(); + fn.accept(builder); + return builder.build(); + } + + public static class Builder { + private boolean vectorizeCollectionName = false; + + public Builder vectorizeCollectionName() { + this.vectorizeCollectionName = true; + return this; + } + + public ContextionaryVectorizer build() { + return new ContextionaryVectorizer(Map.of("vectorizeClassName", vectorizeCollectionName)); + } + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/Vectorizer.java b/src/main/java/io/weaviate/client6/v1/collections/Vectorizer.java index ad9c4260f..d421d926a 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/Vectorizer.java +++ b/src/main/java/io/weaviate/client6/v1/collections/Vectorizer.java @@ -1,8 +1,18 @@ package io.weaviate.client6.v1.collections; +import java.util.function.Consumer; + // This class is WIP, I haven't decided how to structure it yet. public abstract class Vectorizer { public static NoneVectorizer none() { return new NoneVectorizer(); } + + public static ContextionaryVectorizer contextionary() { + return ContextionaryVectorizer.of(); + } + + public static ContextionaryVectorizer contextionary(Consumer fn) { + return ContextionaryVectorizer.of(fn); + } } diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/NearText.java b/src/main/java/io/weaviate/client6/v1/collections/query/NearText.java new file mode 100644 index 000000000..0a585a281 --- /dev/null +++ b/src/main/java/io/weaviate/client6/v1/collections/query/NearText.java @@ -0,0 +1,35 @@ +package io.weaviate.client6.v1.collections.query; + +import java.util.List; +import java.util.function.Consumer; + +public record NearText(List text, Float distance, Float certainty, CommonQueryOptions common) { + + public static NearText with(String text, Consumer fn) { + return with(List.of(text), fn); + } + + public static NearText with(List text, Consumer fn) { + var opt = new Builder(); + fn.accept(opt); + return new NearText(text, opt.distance, opt.certainty, new CommonQueryOptions(opt)); + } + + public static class Builder extends CommonQueryOptions.Builder { + private Float distance; + private Float certainty; + + public Builder distance(float distance) { + this.distance = distance; + return this; + } + + public Builder certainty(float certainty) { + this.certainty = certainty; + return this; + } + } + + public static record GroupBy(String property, int maxGroups, int maxObjectsPerGroup) { + } +} diff --git a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java index 66a8a6540..cb767d635 100644 --- a/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java +++ b/src/main/java/io/weaviate/client6/v1/collections/query/QueryClient.java @@ -31,6 +31,13 @@ public QueryClient(String collectionName, GrpcClient grpc) { this.collectionName = collectionName; } + public QueryResult nearVector(Float[] vector) { + var query = NearVector.with(vector, opt -> { + }); + var req = new SearchMarshaler(collectionName).addNearVector(query); + return search(req.marshal()); + } + public QueryResult nearVector(Float[] vector, Consumer options) { var query = NearVector.with(vector, options); var req = new SearchMarshaler(collectionName).addNearVector(query); @@ -53,6 +60,19 @@ public GroupedQueryResult nearVector(Float[] vector, NearVector.GroupBy group return searchGrouped(req.marshal()); } + public QueryResult nearText(String text, Consumer fn) { + var query = NearText.with(text, fn); + var req = new SearchMarshaler(collectionName).addNearText(query); + return search(req.marshal()); + } + + public QueryResult nearText(String text) { + var query = NearText.with(text, opt -> { + }); + var req = new SearchMarshaler(collectionName).addNearText(query); + return search(req.marshal()); + } + private QueryResult search(SearchRequest req) { var reply = grpcClient.grpc.search(req); return deserializeUntyped(reply);