Generate code based on 2025-01 api spec

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 [comment]: <> (When bumping [pc:VERSION_LATEST_RELEASE] create a new entry below)
 ### Unreleased version
+### 4.0.0
+- Add support for sparse indexes
+- Generate code based on 2025-01 open-api spec
+
 ### 3.1.0
 - Add support to pass base url for control and data plane operations
 

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023 Pinecone Systems, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -15,20 +15,20 @@ Maven:
 <dependency>
   <groupId>io.pinecone</groupId>
   <artifactId>pinecone-client</artifactId>
-  <version>3.0.0</version>
+  <version>4.0.0</version>
 </dependency>
 ```
 
 [comment]: <> (^ [pc:VERSION_LATEST_RELEASE])
 
 Gradle:
 ```
-implementation "io.pinecone:pinecone-client:3.0.0"
+implementation "io.pinecone:pinecone-client:4.0.0"
 ```
 
 [comment]: <> (^ [pc:VERSION_LATEST_RELEASE])
 
-Alternatively, you can use our standalone uberjar [pinecone-client-3.0.0-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/3.0.0/pinecone-client-3.0.0-all.jar), which bundles the Pinecone
+Alternatively, you can use our standalone uberjar [pinecone-client-4.0.0-all.jar](https://repo1.maven.org/maven2/io/pinecone/pinecone-client/4.0.0/pinecone-client-4.0.0-all.jar), which bundles the Pinecone
 SDK and all dependencies together. You can include this in your classpath like you do with any 3rd party JAR without
 having to obtain the *pinecone-client* dependencies separately.
 
@@ -162,7 +162,7 @@ Operations related to the building and managing of Pinecone indexes are called [
 You can use the Java SDK to create two types of indexes: [serverless indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes) (recommended for most use cases) and 
 [pod-based indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#pod-based-indexes) (recommended for high-throughput use cases).
 
-### Create a serverless index
+### Create a dense serverless index
 
 The following is an example of creating a serverless index in the `us-west-2` region of AWS. For more information on 
 serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes).
@@ -187,6 +187,30 @@ tags.put("env", "test");
 IndexModel indexModel = pinecone.createServerlessIndex(indexName, similarityMetric, dimension, cloud, region, DeletionProtection.ENABLED, tags);
 ```
 
+### Create a sparse serverless index
+
+The following is an example of creating a sparse serverless index in the `us-east-1` region of AWS. For more information on
+serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/sparse-indexes).
+
+```java
+import io.pinecone.clients.Pinecone;
+import org.openapitools.db_control.client.model.IndexModel;
+import org.openapitools.db_control.client.model.DeletionProtection;
+import java.util.HashMap;
+...
+
+Pinecone pinecone = new Pinecone.Builder("PINECONE_API_KEY").build();
+
+String indexName = "example-index";
+String cloud = "aws";
+String region = "us-east-1";
+HashMap<String, String> tags = new HashMap<>();
+tags.put("env", "test");
+String vectorType = "sparse";
+
+IndexModel indexModel = pinecone.createSparseServelessIndex(indexName, cloud, region, DeletionProtection.ENABLED, tags, vectorType);
+```
+
 ### Create a pod index
 
 The following is a minimal example of creating a pod-based index. For all the possible configuration options, see 

build.gradle

@@ -52,7 +52,7 @@ dependencies {
     implementation 'com.google.code.gson:gson:2.9.1'
     implementation 'io.gsonfire:gson-fire:1.8.5'
     implementation 'org.openapitools:jackson-databind-nullable:0.2.6'
-    implementation 'com.google.protobuf:protobuf-java:3.25.2'
+    implementation 'com.google.protobuf:protobuf-java:4.29.3'
     compileOnly "org.apache.tomcat:annotations-api:6.0.53" // necessary for Java 9+
 
     testImplementation("com.squareup.okhttp3:mockwebserver:4.12.0")

codegen/buf.gen.yaml

@@ -0,0 +1,6 @@
+version: v1
+plugins:
+  - name: java
+    out: gen/java
+  - name: grpc-java
+    out: gen/java

codegen/buf.lock

@@ -0,0 +1,6 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+  - name: buf.build/googleapis/googleapis
+    commit: e93e34f48be043dab55be31b4b47f458
+    digest: b5:cebe5dfac5f7d67c55296f37ad9d368dba8d9862777e69d5d99eb1d72dc95fa68cd6323b483ca42cf70e66060002c1bc36e1f5f754b217a5c771c108eb243dbf

codegen/buf.yaml

@@ -0,0 +1,12 @@
+# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml
+version: v2
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - FILE
+deps:
+  - buf.build/googleapis/googleapis
+modules:
+  - path: apis/_build/2025-01

codegen/build-grpc.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -eux -o pipefail
+version=$1 # e.g. 2024-07
+
+update_apis_repo() {
+	echo "Updating apis repo"
+	pushd codegen/apis
+		git fetch
+		git checkout main
+		git pull
+		just build
+	popd
+}
+
+update_buf_config() {
+    pushd codegen
+        # Update buf config to find correct proto version
+        sed -i '' "s/[0-9][0-9][0-9][0-9]-[0-1][0-9]/${version}/g" buf.yaml
+
+        # Clean before building
+        rm -rf gen
+
+        # Ensure path valid by running the buf build command
+        buf build
+    popd
+}
+
+buf_generate() {
+    pushd codegen
+        # Generate the java code
+        buf generate
+    popd
+}
+
+update_apis_repo
+update_buf_config
+buf_generate
+
+dest="src/main/java/io/pinecone/proto/"
+
+# Remove existing files in dest
+rm -rf "${dest}*.java"
+
+# Copy the new generated files to dest directory
+cp codegen/gen/java/io/pinecone/proto/*.java ${dest}
+
+# Cleanup the intermediate files that were generated
+rm -rf codegen/gen

gradle.properties

@@ -1 +1 @@
-pineconeClientVersion = 3.1.0
+pineconeClientVersion = 4.0.0

src/integration/java/io/pinecone/helpers/TestUtilities.java

@@ -1,15 +1,10 @@
 package io.pinecone.helpers;
 
-import io.pinecone.clients.Index;
 import io.pinecone.clients.Pinecone;
-import io.pinecone.proto.DescribeIndexStatsResponse;
-import io.pinecone.proto.NamespaceSummary;
 import org.openapitools.db_control.client.model.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.*;
-
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.fail;
 

src/integration/java/io/pinecone/integration/controlPlane/pod/CollectionTest.java

@@ -5,9 +5,9 @@
 import io.pinecone.exceptions.PineconeException;
 import io.pinecone.exceptions.PineconeValidationException;
 import io.pinecone.helpers.RandomStringBuilder;
-import io.pinecone.proto.FetchResponse;
 import io.pinecone.helpers.TestResourcesManager;
 
+import io.pinecone.proto.FetchResponse;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;

src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java

@@ -0,0 +1,110 @@
+package io.pinecone.integration.controlPlane.serverless;
+
+import io.pinecone.clients.Index;
+import io.pinecone.clients.Pinecone;
+import io.pinecone.exceptions.PineconeNotFoundException;
+import io.pinecone.helpers.RandomStringBuilder;
+import io.pinecone.proto.UpsertResponse;
+import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
+import org.junit.jupiter.api.*;
+import org.openapitools.db_control.client.model.DeletionProtection;
+import org.openapitools.db_control.client.model.IndexModel;
+
+import java.util.*;
+
+import static io.pinecone.helpers.TestUtilities.waitUntilIndexIsReady;
+import static org.junit.jupiter.api.Assertions.*;
+
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class SparseIndexTest {
+    static String indexName;
+    static Pinecone pinecone;
+
+    @BeforeAll
+    public static void setUp() throws InterruptedException {
+        indexName = RandomStringBuilder.build("sparse-index", 8);
+        pinecone = new Pinecone
+                .Builder(System.getenv("PINECONE_API_KEY"))
+                .withSourceTag("pinecone_test")
+                .build();
+    }
+
+    @Test
+    @Order(1)
+    public void createSparseIndex() {
+        Map<String, String> tags = new HashMap<>();
+        tags.put("env", "test");
+
+        // Create sparse Index
+        IndexModel indexModel = pinecone.createSparseServelessIndex(indexName,
+                "aws",
+                "us-east-1",
+                DeletionProtection.ENABLED,
+                tags,
+                "sparse");
+
+        assertNotNull(indexModel);
+        assertEquals(indexName, indexModel.getName());
+        assertEquals(IndexModel.MetricEnum.DOTPRODUCT, indexModel.getMetric());
+        assertEquals(indexModel.getDeletionProtection(), DeletionProtection.ENABLED);
+        assertEquals(indexModel.getTags(), tags);
+        assertEquals(indexModel.getVectorType(), "sparse");
+    }
+
+    @Test
+    @Order(2)
+    public void configureSparseIndex() throws InterruptedException {
+        String key = "flag";
+        String value = "internal";
+        Map<String, String> tags = new HashMap<>();
+        tags.put(key, value);
+
+        // Wait until index is ready
+        waitUntilIndexIsReady(pinecone, indexName, 200000);
+
+        // Disable deletion protection and add more index tags
+        pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags);
+        Thread.sleep(7000);
+
+        // Describe index to confirm deletion protection is disabled
+        IndexModel indexModel = pinecone.describeIndex(indexName);
+        assertEquals(indexModel.getDeletionProtection(), DeletionProtection.DISABLED);
+        assert indexModel.getTags() != null;
+        assertEquals(indexModel.getTags().get(key), value);
+    }
+
+    @Disabled
+    // @Order(3)
+    public void upsertAndQueryVectors() {
+        Index index = pinecone.getIndexConnection(indexName);
+        String id = "v1";
+        ArrayList<Long> indices = new ArrayList<>();
+        indices.add(1L);
+        indices.add(2L);
+
+        ArrayList<Float> values = new ArrayList<>();
+        values.add(1f);
+        values.add(2f);
+
+        UpsertResponse upsertResponse = index.upsert("v1", Collections.emptyList(), indices, values, null, "");
+        assertEquals(upsertResponse.getUpsertedCount(), 1);
+
+        // Query by vector id
+        QueryResponseWithUnsignedIndices queryResponse = index.queryByVectorId(1, id, true, false);
+        assertEquals(queryResponse.getMatchesList().size(), 1);
+        assertEquals(queryResponse.getMatches(0).getId(), id);
+        assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getIndicesWithUnsigned32IntList(), indices);
+        assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getValuesList(), values);
+    }
+
+    @Test
+    @Order(4)
+    public void deleteSparseIndex() throws InterruptedException {
+        // Delete sparse index
+        pinecone.deleteIndex(indexName);
+        Thread.sleep(5000);
+
+        // Confirm the index is deleted by calling describe index which should return resource not found
+        assertThrows(PineconeNotFoundException.class, () -> pinecone.describeIndex(indexName));
+    }
+}

src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java

@@ -48,17 +48,7 @@ public void upsertWithApiKeyMissingSyncTest() {
             index.upsert(null, values);
             fail("Expecting invalid upsert request exception");
         } catch (PineconeException expected) {
-            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided.");
-        }
-    }
-
-    @Test
-    public void upsertWhenValuesMissingSyncTest() {
-        try {
-            index.upsert("some_id", null);
-            fail("Expecting invalid upsert request exception");
-        } catch (PineconeException expected) {
-            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided.");
+            assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that id is provided.");
         }
     }
 
@@ -126,17 +116,7 @@ public void upsertWithApiKeyMissingFutureTest() {
             asyncIndex.upsert(null, values);
             fail("Expecting invalid upsert request exception");
         } catch (PineconeException expected) {
-            assertTrue(expected.getMessage().contains("ensure that both id and values are provided."));
-        }
-    }
-
-    @Test
-    public void upsertWhenValuesMissingFutureTest() {
-        try {
-            asyncIndex.upsert("some_id", null);
-            fail("Expecting invalid upsert request exception");
-        } catch (PineconeException expected) {
-            assertTrue(expected.getMessage().contains("ensure that both id and values are provided."));
+            assertTrue(expected.getMessage().contains("ensure that id is provided."));
         }
     }
 

src/integration/java/io/pinecone/integration/inference/EmbedTest.java

@@ -35,7 +35,7 @@ public void testGenerateEmbeddings() throws ApiException {
 
         assertNotNull(embeddings, "Expected embedding to be not null");
         Assertions.assertEquals(embeddingModel, embeddings.getModel());
-        Assertions.assertEquals(1024, embeddings.getData().get(0).getValues().size());
+        Assertions.assertEquals(1024, embeddings.getData().get(0).getDenseEmbedding().getValues().size());
         Assertions.assertEquals(2, embeddings.getData().size());
     }