Fail fast in current region when Document requests hit 410 Lease Not Found.

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicyTest.java

@@ -9,6 +9,7 @@
 import com.azure.cosmos.implementation.GoneException;
 import com.azure.cosmos.implementation.HttpConstants;
 import com.azure.cosmos.implementation.InvalidPartitionException;
+import com.azure.cosmos.implementation.LeaseNotFoundException;
 import com.azure.cosmos.implementation.OperationType;
 import com.azure.cosmos.implementation.PartitionIsMigratingException;
 import com.azure.cosmos.implementation.PartitionKeyRangeIsSplittingException;
@@ -76,6 +77,35 @@ public void shouldRetryReadWithGoneException() {
         assertThat(shouldRetryResult.backOffTime.getSeconds()).isEqualTo(4);
     }
 
+    @Test(groups = { "unit" }, timeOut = TIMEOUT)
+    public void shouldNotRetryReadWithLeaseNotFoundException() {
+        RxDocumentServiceRequest request = RxDocumentServiceRequest.create(
+            mockDiagnosticsClientContext(),
+            OperationType.Read,
+            ResourceType.Document);
+        GoneAndRetryWithRetryPolicy goneAndRetryWithRetryPolicy = new GoneAndRetryWithRetryPolicy(request, 30);
+        Mono<ShouldRetryResult> singleShouldRetry = goneAndRetryWithRetryPolicy
+            .shouldRetry(new LeaseNotFoundException("0", null));
+        ShouldRetryResult shouldRetryResult = singleShouldRetry.block();
+        assertThat(shouldRetryResult.shouldRetry).isFalse();
+        assertThat(shouldRetryResult.policyArg.getValue0()).isTrue();
+        assertThat(shouldRetryResult.policyArg.getValue3()).isEqualTo(1);
+        assertThat(shouldRetryResult.backOffTime).isNull();
+
+        request = RxDocumentServiceRequest.create(
+            mockDiagnosticsClientContext(),
+            OperationType.Create,
+            ResourceType.Document);
+        goneAndRetryWithRetryPolicy = new GoneAndRetryWithRetryPolicy(request, 30);
+        singleShouldRetry = goneAndRetryWithRetryPolicy
+            .shouldRetry(new LeaseNotFoundException("0", null));
+        shouldRetryResult = singleShouldRetry.block();
+        assertThat(shouldRetryResult.shouldRetry).isFalse();
+        assertThat(shouldRetryResult.policyArg.getValue0()).isTrue();
+        assertThat(shouldRetryResult.policyArg.getValue3()).isEqualTo(1);
+        assertThat(shouldRetryResult.backOffTime).isNull();
+    }
+
     /**
      * Retry with GoneException for write which is not yet sent to the wire,
      * retried 4 times and verified the returned

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/directconnectivity/StoreReaderTest.java

@@ -7,11 +7,13 @@
 import com.azure.cosmos.CosmosException;
 import com.azure.cosmos.implementation.DiagnosticsClientContext;
 import com.azure.cosmos.implementation.DocumentServiceRequestContext;
+import com.azure.cosmos.implementation.Exceptions;
 import com.azure.cosmos.implementation.FailureValidator;
 import com.azure.cosmos.implementation.GoneException;
 import com.azure.cosmos.implementation.HttpConstants;
 import com.azure.cosmos.implementation.ISessionContainer;
 import com.azure.cosmos.implementation.ISessionToken;
+import com.azure.cosmos.implementation.LeaseNotFoundException;
 import com.azure.cosmos.implementation.NotFoundException;
 import com.azure.cosmos.implementation.OperationType;
 import com.azure.cosmos.implementation.PartitionIsMigratingException;
@@ -149,6 +151,7 @@ public Object[][] storeResponseArgProvider() {
             { new PartitionKeyRangeIsSplittingException() , null, },
             { new PartitionIsMigratingException(), null, },
             { new GoneException(), null, },
+            { new LeaseNotFoundException(null, 0L, null, null), null },
             { null, Mockito.mock(StoreResponse.class), }
         };
     }
@@ -915,9 +918,15 @@ public void storeResponseRecordedOnException(Exception ex, StoreResponse storeRe
                 try {
                     StoreReader.verifyCanContinueOnException((CosmosException) ex);
 
-                    // for continuable exception, SDK will retry on all other replicas, so the failed endpoints should match replica counts.
-                    List<Uri> expectedFailedEndpoints = Arrays.asList(primaryUri, secondaryUri1, secondaryUri2, secondaryUri3);
-                    assertThat(dsr.requestContext.getFailedEndpoints()).hasSize(expectedFailedEndpoints.size()).containsAll(expectedFailedEndpoints);
+                    if (Exceptions.isAvoidQuorumSelectionException((CosmosException) ex)) {
+                        // while the exception is continuable, it is avoid quorum selection exception, so such results are collected
+                        // while these results are not valid, they are still collected in the failed endpoints and also contribute towards decrementing replicaCountToRead to avoid quorum reselection.
+                        assertThat(dsr.requestContext.getFailedEndpoints().size()).isEqualTo(3);
+                    } else {
+                        // for continuable exception, SDK will retry on all other replicas, so the failed endpoints should match replica counts.
+                        List<Uri> expectedFailedEndpoints = Arrays.asList(primaryUri, secondaryUri1, secondaryUri2, secondaryUri3);
+                        assertThat(dsr.requestContext.getFailedEndpoints()).hasSize(expectedFailedEndpoints.size()).containsAll(expectedFailedEndpoints);
+                    }
 
                 } catch (Exception exception) {
                     if (exception instanceof CosmosException) {