v6: Collection management and near vector search

pom.xml

@@ -264,7 +264,13 @@
                 @see: https://www.oracle.com/corporate/features/understanding-java-9-modules.html
               -->
               --add-opens=java.base/java.lang=ALL-UNNAMED
-            </argLine>
+              </argLine>
+              <properties>
+                <property>
+                  <name>listener</name>
+                  <value>io.weaviate.containers.TestListener</value>
+                </property>
+              </properties>
           </configuration>
         </plugin>
         <plugin>
@@ -334,12 +340,24 @@
                 </artifacts>
               </configuration>
             </execution>
+            <execution>
+              <id>add-test-source</id>
+              <phase>process-resources</phase>
+              <goals>
+                <goal>add-test-source</goal>
+              </goals>
+              <configuration>
+                <sources>
+                  <source>${project.basedir}/src/it/java</source>
+                </sources>
+              </configuration>
+            </execution>
           </executions>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-javadoc-plugin</artifactId>
-          <version>3.2.0</version>
+          <version>3.11.2</version>
           <executions>
             <execution>
               <id>attach-javadocs</id>

src/it/java/io/weaviate/ConcurrentTest.java

@@ -0,0 +1,64 @@
+package io.weaviate;
+
+import java.util.Random;
+import java.util.UUID;
+import java.util.stream.IntStream;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+/**
+ * ConcurrentTest is the base class for integration tests, which provides
+ * utility methods to uniqualize collections and objects created in the
+ * database.
+ *
+ * Because we want to re-use the same database container across most of the
+ * test suites and (eventually) run them in parallel,
+ * test classes should extend this class and use its methods
+ * to avoid name clashes in the shared Weaviate instance.
+ */
+public abstract class ConcurrentTest {
+  @Rule
+  public TestName currentTest = new TestName();
+
+  protected static final Random rand = new Random();
+
+  /**
+   * Add unique namespace prefix to the string.
+   *
+   * @param value Collection name, object ID, etc., which has to be unique across
+   *              all test suites.
+   * @return Value prefixed with the name of the current test suite + test method.
+   */
+  protected String ns(String value) {
+    String cls = getClass().getSimpleName();
+    String method = currentTest.getMethodName();
+    return cls + "_" + method + "_" + value;
+  }
+
+  /** Appends random characters to create unique value. */
+  protected static String unique(String value) {
+    var randString = RandomStringUtils.insecure().next(8, true, false);
+    return value + "_" + randString;
+  }
+
+  /** Generate random UUID. */
+  protected static String randomUUID() {
+    return UUID.randomUUID().toString();
+  }
+
+  /**
+   * Generate a random vector.
+   *
+   * @param length Vector length.
+   * @param origin Value range lower bound.
+   * @param bound  Value range upper bound.
+   * @return
+   */
+  protected static Float[] randomVector(int length, float origin, float bound) {
+    return IntStream.range(0, length)
+        .<Float>mapToObj(f -> rand.nextFloat(origin, bound))
+        .toArray(Float[]::new);
+  }
+}

src/it/java/io/weaviate/client6/internal/GRPCTest.java

@@ -0,0 +1,48 @@
+package io.weaviate.client6.internal;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.junit.Test;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Note: Java's {@code byte} is signed (int8) and is different from {@code byte}
+ * in Go, which is an alias for uint8.
+ *
+ * For this tests purposes the distinction is immaterial, as "want" arrays
+ * are "golden values" meant to be a readable respresentation for the test.
+ */
+public class GRPCTest {
+  @Test
+  public void test_toBytesString_1d() {
+    Float[] vector = { 1f, 2f, 3f };
+    byte[] want = { 0, 0, -128, 63, 0, 0, 0, 64, 0, 0, 64, 64 };
+    byte[] got = GRPC.toByteString(vector).toByteArray();
+    assertArrayEquals(want, got);
+  }
+
+  @Test
+  public void test_fromBytesString_1d() {
+    byte[] bytes = { 0, 0, -128, 63, 0, 0, 0, 64, 0, 0, 64, 64 };
+    Float[] want = { 1f, 2f, 3f };
+    Float[] got = GRPC.fromByteString(ByteString.copyFrom(bytes));
+    assertArrayEquals(want, got);
+  }
+
+  @Test
+  public void test_toBytesString_2d() {
+    Float[][] vector = { { 1f, 2f, 3f }, { 4f, 5f, 6f } };
+    byte[] want = { 3, 0, 0, 0, -128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, -128, 64, 0, 0, -96, 64, 0, 0, -64, 64 };
+    byte[] got = GRPC.toByteString(vector).toByteArray();
+    assertArrayEquals(want, got);
+  }
+
+  @Test
+  public void test_fromBytesString_2d() {
+    byte[] bytes = { 3, 0, 0, 0, -128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, -128, 64, 0, 0, -96, 64, 0, 0, -64, 64 };
+    Float[][] want = { { 1f, 2f, 3f }, { 4f, 5f, 6f } };
+    Float[][] got = GRPC.fromByteStringMulti(ByteString.copyFrom(bytes));
+    assertArrayEquals(want, got);
+  }
+}

src/it/java/io/weaviate/client6/v1/DataITest.java

@@ -0,0 +1,67 @@
+package io.weaviate.client6.v1;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.assertj.core.api.Assertions;
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.weaviate.ConcurrentTest;
+import io.weaviate.client6.WeaviateClient;
+import io.weaviate.client6.v1.collections.Property;
+import io.weaviate.client6.v1.collections.VectorIndex;
+import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy;
+import io.weaviate.client6.v1.collections.Vectorizer;
+import io.weaviate.containers.Container;
+
+public class DataITest extends ConcurrentTest {
+
+  private static WeaviateClient client = Container.WEAVIATE.getClient();
+  private static final String COLLECTION = unique("Things");
+  private static final String VECTOR_INDEX = "bring_your_own";
+
+  @BeforeClass
+  public static void beforeAll() throws IOException {
+    createTestCollection();
+  }
+
+  @Test
+  public void testCreateGetDelete() throws IOException {
+    var things = client.collections.use(COLLECTION);
+    var id = randomUUID();
+    Float[] vector = { 1f, 2f, 3f };
+
+    things.data.insert(Map.of("username", "john doe"), metadata -> metadata
+        .id(id)
+        .vectors(Vectors.of(VECTOR_INDEX, vector)));
+
+    var object = things.data.get(id, query -> query.withVector());
+    Assertions.assertThat(object)
+        .as("object exists after insert").get()
+        .satisfies(obj -> {
+          Assertions.assertThat(obj.metadata().id())
+              .as("object id").isEqualTo(id);
+
+          Assertions.assertThat(obj.metadata().vectors()).extracting(Vectors::getSingle)
+              .asInstanceOf(InstanceOfAssertFactories.OPTIONAL).as("has single vector").get()
+              .asInstanceOf(InstanceOfAssertFactories.array(Float[].class)).containsExactly(vector);
+
+          Assertions.assertThat(obj.properties())
+              .as("has expected properties")
+              .containsEntry("username", "john doe");
+        });
+
+    things.data.delete(id);
+    object = things.data.get(id);
+    Assertions.assertThat(object).isEmpty().as("object not exists after deletion");
+  }
+
+  private static void createTestCollection() throws IOException {
+    client.collections.create(COLLECTION,
+        col -> col
+            .properties(Property.text("username"), Property.integer("age"))
+            .vector(VECTOR_INDEX, new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none())));
+  }
+}

src/it/java/io/weaviate/client6/v1/collections/CollectionsITest.java

@@ -0,0 +1,41 @@
+package io.weaviate.client6.v1.collections;
+
+import java.io.IOException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import io.weaviate.ConcurrentTest;
+import io.weaviate.client6.WeaviateClient;
+import io.weaviate.client6.v1.collections.VectorIndex.IndexType;
+import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy;
+import io.weaviate.containers.Container;
+
+public class CollectionsITest extends ConcurrentTest {
+  private static WeaviateClient client = Container.WEAVIATE.getClient();
+
+  @Test
+  public void testCreateGetDelete() throws IOException {
+    var collectionName = ns("Things_1");
+    client.collections.create(collectionName,
+        col -> col
+            .properties(Property.text("username"), Property.integer("age"))
+            .vector(new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none())));
+
+    var thingsCollection = client.collections.getConfig(collectionName);
+
+    Assertions.assertThat(thingsCollection).get()
+        .hasFieldOrPropertyWithValue("name", collectionName)
+        .extracting(CollectionDefinition::vectors).extracting(Vectors::getDefault)
+        .as("default vector").satisfies(defaultVector -> {
+          Assertions.assertThat(defaultVector).extracting(VectorIndex::vectorizer)
+              .as("has none vectorizer").isInstanceOf(NoneVectorizer.class);
+          Assertions.assertThat(defaultVector).extracting(VectorIndex::configuration)
+              .as("has hnsw index").returns(IndexType.HNSW, IndexingStrategy::type);
+        });
+
+    client.collections.delete(collectionName);
+    var noCollection = client.collections.getConfig(collectionName);
+    Assertions.assertThat(noCollection).as("after delete").isEmpty();
+  }
+}

src/it/java/io/weaviate/client6/v1/query/NearVectorQueryITest.java

@@ -0,0 +1,88 @@
+package io.weaviate.client6.v1.query;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.assertj.core.api.Assertions;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.weaviate.ConcurrentTest;
+import io.weaviate.client6.WeaviateClient;
+import io.weaviate.client6.v1.Vectors;
+import io.weaviate.client6.v1.collections.VectorIndex;
+import io.weaviate.client6.v1.collections.VectorIndex.IndexingStrategy;
+import io.weaviate.client6.v1.collections.Vectorizer;
+import io.weaviate.containers.Container;
+
+public class NearVectorQueryITest extends ConcurrentTest {
+  private static final WeaviateClient client = Container.WEAVIATE.getClient();
+
+  private static final String COLLECTION = unique("Things");
+  private static final String VECTOR_INDEX = "bring_your_own";
+
+  /**
+   * One of the inserted vectors which will be used as target vector for search.
+   */
+  private static Float[] searchVector;
+
+  @BeforeClass
+  public static void beforeAll() throws IOException {
+    createTestCollection();
+    var created = createVectors(10);
+    searchVector = created.values().iterator().next();
+  }
+
+  @Test
+  public void testNearVector() {
+    // TODO: test that we return the results in the expected order
+    // Because re-ranking should work correctly
+    var things = client.collections.use(COLLECTION);
+    QueryResult<Map<String, Object>> result = things.query.nearVector(searchVector,
+        opt -> opt
+            .distance(2f)
+            .limit(3)
+            .returnMetadata(MetadataField.DISTANCE));
+
+    Assertions.assertThat(result.objects).hasSize(3);
+    float maxDistance = Collections.max(result.objects,
+        Comparator.comparing(obj -> obj.metadata.distance)).metadata.distance;
+    Assertions.assertThat(maxDistance).isLessThanOrEqualTo(2f);
+  }
+
+  /**
+   * Insert 10 objects with random vectors.
+   *
+   * @returns IDs of inserted objects and their corresponding vectors.
+   */
+  private static Map<String, Float[]> createVectors(int n) throws IOException {
+    var created = new HashMap<String, Float[]>();
+
+    var things = client.collections.use(COLLECTION);
+    for (int i = 0; i < n; i++) {
+      var vector = randomVector(10, -.01f, .001f);
+      var object = things.data.insert(
+          Map.of(),
+          metadata -> metadata
+              .id(randomUUID())
+              .vectors(Vectors.of(VECTOR_INDEX, vector)));
+
+      created.put(object.metadata().id(), vector);
+    }
+
+    return created;
+  }
+
+  /**
+   * Create {@link COLLECTION} with {@link VECTOR_INDEX} vector index.
+   *
+   * @throws IOException
+   */
+  private static void createTestCollection() throws IOException {
+    client.collections.create(COLLECTION, cfg -> cfg
+        .vector(VECTOR_INDEX, new VectorIndex<>(IndexingStrategy.hnsw(), Vectorizer.none())));
+  }
+}