From e6b8db843f3603d7b04d996364715b010e4c92a6 Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Fri, 31 Jan 2025 16:41:53 -0500 Subject: [PATCH 1/6] add create sparse index method --- .../serverless/SparseIndexTest.java | 99 +++++++++++++++++++ .../java/io/pinecone/clients/Pinecone.java | 75 +++++++++++++- .../io/pinecone/commons/IndexInterface.java | 6 +- 3 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java new file mode 100644 index 00000000..fbb64ad9 --- /dev/null +++ b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java @@ -0,0 +1,99 @@ +package io.pinecone.integration.controlPlane.serverless; + +import io.pinecone.clients.Index; +import io.pinecone.clients.Pinecone; +import io.pinecone.exceptions.PineconeNotFoundException; +import io.pinecone.proto.UpsertResponse; +import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices; +import org.junit.jupiter.api.*; +import org.openapitools.db_control.client.model.DeletionProtection; +import org.openapitools.db_control.client.model.IndexModel; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SparseIndexTest { + String indexName = "sparse-index"; + Pinecone pinecone = new Pinecone + .Builder(System.getenv("PINECONE_API_KEY")) + .withSourceTag("pinecone_test") + .build(); + + @Test + @Order(1) + public void createSparseIndex() { + Map tags = new HashMap<>(); + tags.put("env", "test"); + + // Create sparse Index + IndexModel indexModel = pinecone.createSparseServelessIndex(indexName, + "aws", + "us-east-1", + DeletionProtection.ENABLED, + tags, + "sparse"); + + assertNotNull(indexModel); + assertEquals(indexName, indexModel.getName()); + assertEquals(IndexModel.MetricEnum.DOTPRODUCT, indexModel.getMetric()); + assertEquals(indexModel.getDeletionProtection(), DeletionProtection.ENABLED); + assertEquals(indexModel.getTags(), tags); + assertEquals(indexModel.getVectorType(), "sparse"); + } + + @Test + @Order(2) + public void configureSparseIndex() throws InterruptedException { + String key = "flag"; + String value = "internal"; + Map tags = new HashMap<>(); + tags.put(key, value); + + // Disable deletion protection and add more index tags + pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags); + Thread.sleep(7000); + + // Describe index to confirm deletion protection is disabled + IndexModel indexModel = pinecone.describeIndex(indexName); + assertEquals(indexModel.getDeletionProtection(), DeletionProtection.DISABLED); + assert indexModel.getTags() != null; + assertEquals(indexModel.getTags().get(key), value); + } + + @Disabled + // @Order(3) + public void upsertAndQueryVectors() { + Index index = pinecone.getIndexConnection(indexName); + String id = "v1"; + ArrayList indices = new ArrayList<>(); + indices.add(1L); + indices.add(2L); + + ArrayList values = new ArrayList<>(); + values.add(1f); + values.add(2f); + + UpsertResponse upsertResponse = index.upsert("v1", Collections.emptyList(), indices, values, null, ""); + assertEquals(upsertResponse.getUpsertedCount(), 1); + + // Query by vector id + QueryResponseWithUnsignedIndices queryResponse = index.queryByVectorId(1, id, true, false); + assertEquals(queryResponse.getMatchesList().size(), 1); + assertEquals(queryResponse.getMatches(0).getId(), id); + assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getIndicesWithUnsigned32IntList(), indices); + assertEquals(queryResponse.getMatches(0).getSparseValuesWithUnsignedIndices().getValuesList(), values); + } + + @Test + @Order(4) + public void deleteSparseIndex() throws InterruptedException { + // Delete sparse index + pinecone.deleteIndex(indexName); + Thread.sleep(5000); + + // Confirm the index is deleted by calling describe index which should return resource not found + assertThrows(PineconeNotFoundException.class, () -> pinecone.describeIndex(indexName)); + } +} diff --git a/src/main/java/io/pinecone/clients/Pinecone.java b/src/main/java/io/pinecone/clients/Pinecone.java index 7e4e7117..6f10623d 100644 --- a/src/main/java/io/pinecone/clients/Pinecone.java +++ b/src/main/java/io/pinecone/clients/Pinecone.java @@ -87,8 +87,9 @@ public IndexModel createServerlessIndex(String indexName, } if (metric == null || metric.isEmpty()) { - throw new PineconeValidationException("Metric cannot be null or empty. Must be one of " + Arrays.toString(CreateIndexRequest.MetricEnum.values())); + metric = "cosine"; } + try { CreateIndexRequest.MetricEnum.fromValue(metric.toLowerCase()); } catch (IllegalArgumentException e) { @@ -142,6 +143,78 @@ public IndexModel createServerlessIndex(String indexName, return indexModel; } + /** + * Creates a new sparse serverless index. + *

+ * Example: + * 

{@code
+     *     client.createServerlessIndex("YOUR-INDEX", "cosine", 1536, "aws", "us-west-2", DeletionProtection.ENABLED);
+     * }
+ * + * @param indexName The name of the index to be created. + * @param cloud The cloud provider for the index. + * @param region The cloud region for the index. + * @param deletionProtection Enable or disable deletion protection for the index. + * @param tags A map of tags to associate with the Index. + * @param vectorType The metric type for the index. Must be one of "cosine", "euclidean", or "dotproduct". + * @return {@link IndexModel} representing the created serverless index. + * @throws PineconeException if the API encounters an error during index creation or if any of the arguments are invalid. + */ + public IndexModel createSparseServelessIndex(String indexName, + String cloud, + String region, + DeletionProtection deletionProtection, + Map tags, + String vectorType) throws PineconeException { + if (indexName == null || indexName.isEmpty()) { + throw new PineconeValidationException("Index name cannot be null or empty"); + } + + if (cloud == null || cloud.isEmpty()) { + throw new PineconeValidationException("Cloud cannot be null or empty. Must be one of " + Arrays.toString(ServerlessSpec.CloudEnum.values())); + } + + try { + ServerlessSpec.CloudEnum.fromValue(cloud.toLowerCase()); + } catch (IllegalArgumentException e) { + throw new PineconeValidationException("Cloud cannot be null or empty. Must be one of " + Arrays.toString(ServerlessSpec.CloudEnum.values())); + } + + if (region == null || region.isEmpty()) { + throw new PineconeValidationException("Region cannot be null or empty"); + } + + if(!vectorType.equalsIgnoreCase("sparse") && !vectorType.equalsIgnoreCase("dense")) { + throw new PineconeValidationException("vectorType must be sparse or dense"); + } + + // Convert user string for "cloud" arg into ServerlessSpec.CloudEnum + ServerlessSpec.CloudEnum cloudProvider = ServerlessSpec.CloudEnum.fromValue(cloud.toLowerCase()); + + ServerlessSpec serverlessSpec = new ServerlessSpec().cloud(cloudProvider).region(region); + IndexSpec createServerlessIndexRequestSpec = new IndexSpec().serverless(serverlessSpec); + + IndexModel indexModel = null; + + try { + CreateIndexRequest createIndexRequest = new CreateIndexRequest() + .name(indexName) + .metric(CreateIndexRequest.MetricEnum.DOTPRODUCT) + .spec(createServerlessIndexRequestSpec) + .deletionProtection(deletionProtection) + .vectorType(vectorType); + + if(tags != null && !tags.isEmpty()) { + createIndexRequest.tags(tags); + } + + indexModel = manageIndexesApi.createIndex(createIndexRequest); + } catch (ApiException apiException) { + handleApiException(apiException); + } + return indexModel; + } + /** * Overload for creating a new pods index with environment and podType, the minimum required parameters. *

diff --git a/src/main/java/io/pinecone/commons/IndexInterface.java b/src/main/java/io/pinecone/commons/IndexInterface.java index 50349362..582d6020 100644 --- a/src/main/java/io/pinecone/commons/IndexInterface.java +++ b/src/main/java/io/pinecone/commons/IndexInterface.java @@ -103,12 +103,10 @@ default Vector buildUpsertVector(String id, List sparseIndices, List sparseValues, Struct metadata) { - if (id == null || id.isEmpty() || values == null || values.isEmpty()) { - throw new PineconeValidationException("Invalid upsert request. Please ensure that both id and values are " + - "provided."); + if (id == null || id.isEmpty()) { + throw new PineconeValidationException("Invalid upsert request. Please ensure that id is provided."); } - Vector.Builder vectorBuilder = Vector.newBuilder() .setId(id) .addAllValues(values); From e257a8f832066c4fe899026a32ba6265ab84f04e Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Fri, 31 Jan 2025 16:50:33 -0500 Subject: [PATCH 2/6] forgot to add a file in previous commit, metric is now optional for creating serverless index() --- .../io/pinecone/PineconeIndexOperationsTest.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/test/java/io/pinecone/PineconeIndexOperationsTest.java b/src/test/java/io/pinecone/PineconeIndexOperationsTest.java index cf23a15f..b3932ec0 100644 --- a/src/test/java/io/pinecone/PineconeIndexOperationsTest.java +++ b/src/test/java/io/pinecone/PineconeIndexOperationsTest.java @@ -76,19 +76,6 @@ public void testCreateServerlessIndex() throws IOException { () -> client.createServerlessIndex(null, "cosine", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP)); assertEquals("Index name cannot be null or empty", thrownNullIndexName.getMessage()); - PineconeValidationException thrownEmptyMetric = assertThrows(PineconeValidationException.class, - () -> client.createServerlessIndex("testServerlessIndex", "", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP)); - assertEquals("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values()), thrownEmptyMetric.getMessage()); - - PineconeValidationException thrownInvalidMetric = assertThrows(PineconeValidationException.class, - () -> client.createServerlessIndex("testServerlessIndex", "blah", 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP)); - assertEquals(String.format("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values())), thrownInvalidMetric.getMessage()); - - PineconeValidationException thrownNullMetric = assertThrows(PineconeValidationException.class, - () -> client.createServerlessIndex("testServerlessIndex", null, 3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP)); - assertEquals("Metric cannot be null or empty. Must be one of " + Arrays.toString(IndexModel.MetricEnum.values()), - thrownNullMetric.getMessage()); - PineconeValidationException thrownNegativeDimension = assertThrows(PineconeValidationException.class, () -> client.createServerlessIndex("testServerlessIndex", "cosine", -3, "aws", "us-west-2", DeletionProtection.DISABLED, Collections.EMPTY_MAP)); assertEquals("Dimension must be greater than 0. See limits for more info: https://docs.pinecone.io/reference/limits", thrownNegativeDimension.getMessage()); From 1966b520a1cee7b0fa18897819572bc8ed4e071f Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Fri, 31 Jan 2025 17:08:33 -0500 Subject: [PATCH 3/6] update tests --- .../serverless/SparseIndexTest.java | 3 ++- .../dataPlane/UpsertErrorTest.java | 24 ++----------------- 2 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java index fbb64ad9..3be1c17e 100644 --- a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java +++ b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java @@ -3,6 +3,7 @@ import io.pinecone.clients.Index; import io.pinecone.clients.Pinecone; import io.pinecone.exceptions.PineconeNotFoundException; +import io.pinecone.helpers.RandomStringBuilder; import io.pinecone.proto.UpsertResponse; import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices; import org.junit.jupiter.api.*; @@ -15,7 +16,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class SparseIndexTest { - String indexName = "sparse-index"; + String indexName = RandomStringBuilder.build("sparse-index", 8); Pinecone pinecone = new Pinecone .Builder(System.getenv("PINECONE_API_KEY")) .withSourceTag("pinecone_test") diff --git a/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java b/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java index 1cba4d0a..bbe7351f 100644 --- a/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java +++ b/src/integration/java/io/pinecone/integration/dataPlane/UpsertErrorTest.java @@ -48,17 +48,7 @@ public void upsertWithApiKeyMissingSyncTest() { index.upsert(null, values); fail("Expecting invalid upsert request exception"); } catch (PineconeException expected) { - assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided."); - } - } - - @Test - public void upsertWhenValuesMissingSyncTest() { - try { - index.upsert("some_id", null); - fail("Expecting invalid upsert request exception"); - } catch (PineconeException expected) { - assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that both id and values are provided."); + assertEquals(expected.getMessage(), "Invalid upsert request. Please ensure that id is provided."); } } @@ -126,17 +116,7 @@ public void upsertWithApiKeyMissingFutureTest() { asyncIndex.upsert(null, values); fail("Expecting invalid upsert request exception"); } catch (PineconeException expected) { - assertTrue(expected.getMessage().contains("ensure that both id and values are provided.")); - } - } - - @Test - public void upsertWhenValuesMissingFutureTest() { - try { - asyncIndex.upsert("some_id", null); - fail("Expecting invalid upsert request exception"); - } catch (PineconeException expected) { - assertTrue(expected.getMessage().contains("ensure that both id and values are provided.")); + assertTrue(expected.getMessage().contains("ensure that id is provided.")); } } From 14e355912177a29a0cbd11f61f3f3c9fb374c13c Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Fri, 31 Jan 2025 17:30:58 -0500 Subject: [PATCH 4/6] poll to check for index readiness --- .../integration/controlPlane/serverless/SparseIndexTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java index 3be1c17e..d28b10f9 100644 --- a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java +++ b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java @@ -12,6 +12,7 @@ import java.util.*; +import static io.pinecone.helpers.TestUtilities.waitUntilIndexIsReady; import static org.junit.jupiter.api.Assertions.*; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -52,6 +53,9 @@ public void configureSparseIndex() throws InterruptedException { Map tags = new HashMap<>(); tags.put(key, value); + // Wait until index is ready + waitUntilIndexIsReady(pinecone, indexName, 200000); + // Disable deletion protection and add more index tags pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags); Thread.sleep(7000); From 2d658be1cbc5ec9e690076a6d133c28aebfe04fc Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Fri, 31 Jan 2025 18:20:05 -0500 Subject: [PATCH 5/6] add beforeAll block to create a random indexname --- .../serverless/SparseIndexTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java index d28b10f9..d6a70e74 100644 --- a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java +++ b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java @@ -17,11 +17,17 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class SparseIndexTest { - String indexName = RandomStringBuilder.build("sparse-index", 8); - Pinecone pinecone = new Pinecone - .Builder(System.getenv("PINECONE_API_KEY")) - .withSourceTag("pinecone_test") - .build(); + static String indexName; + static Pinecone pinecone; + + @BeforeAll + public static void setUp() throws InterruptedException { + indexName = RandomStringBuilder.build("sparse-index", 8); + pinecone = new Pinecone + .Builder(System.getenv("PINECONE_API_KEY")) + .withSourceTag("pinecone_test") + .build(); + } @Test @Order(1) @@ -54,7 +60,7 @@ public void configureSparseIndex() throws InterruptedException { tags.put(key, value); // Wait until index is ready - waitUntilIndexIsReady(pinecone, indexName, 200000); + waitUntilIndexIsReady(pinecone, indexName, 200000); // Disable deletion protection and add more index tags pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags); From 82501faeb8a8ba51281899a71f257d8be68b7588 Mon Sep 17 00:00:00 2001 From: rohanshah18 Date: Mon, 3 Feb 2025 11:25:01 -0500 Subject: [PATCH 6/6] add example to the README --- README.md | 27 ++++++++++++++++++- .../serverless/SparseIndexTest.java | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c33f801..ee179996 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Operations related to the building and managing of Pinecone indexes are called [ You can use the Java SDK to create two types of indexes: [serverless indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes) (recommended for most use cases) and [pod-based indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#pod-based-indexes) (recommended for high-throughput use cases). -### Create a serverless index +### Create a dense serverless index The following is an example of creating a serverless index in the `us-west-2` region of AWS. For more information on serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes). @@ -187,6 +187,31 @@ tags.put("env", "test"); IndexModel indexModel = pinecone.createServerlessIndex(indexName, similarityMetric, dimension, cloud, region, DeletionProtection.ENABLED, tags); ``` +### Create a sparse serverless index + +The following is an example of creating a sparse serverless index in the `us-east-1` region of AWS. For more information on +serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/sparse-indexes). + +```java +import io.pinecone.clients.Pinecone; +import org.openapitools.db_control.client.model.IndexModel; +import org.openapitools.db_control.client.model.DeletionProtection; +import java.util.HashMap; +... + +Pinecone pinecone = new Pinecone.Builder("PINECONE_API_KEY").build(); + +String indexName = "example-index"; +int dimension = 1538; +String cloud = "aws"; +String region = "us-east-1"; +HashMap tags = new HashMap<>(); +tags.put("env", "test"); +String vectorType = "sparse"; + +IndexModel indexModel = pinecone.createSparseServelessIndex(indexName, cloud, region, DeletionProtection.ENABLED, tags, vectorType); +``` + ### Create a pod index The following is a minimal example of creating a pod-based index. For all the possible configuration options, see diff --git a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java index d6a70e74..ee749979 100644 --- a/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java +++ b/src/integration/java/io/pinecone/integration/controlPlane/serverless/SparseIndexTest.java @@ -60,7 +60,7 @@ public void configureSparseIndex() throws InterruptedException { tags.put(key, value); // Wait until index is ready - waitUntilIndexIsReady(pinecone, indexName, 200000); + waitUntilIndexIsReady(pinecone, indexName, 200000); // Disable deletion protection and add more index tags pinecone.configureServerlessIndex(indexName, DeletionProtection.DISABLED, tags);