From d011be09158d3dea85151f55bae1fc560af50915 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Wed, 27 Jul 2022 20:41:07 +0800 Subject: [PATCH 1/4] add unit test for batch put Signed-off-by: iosmanthus --- src/test/java/org/tikv/raw/RawKVClientTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/org/tikv/raw/RawKVClientTest.java b/src/test/java/org/tikv/raw/RawKVClientTest.java index ee20df3ee55..39464aca3a2 100644 --- a/src/test/java/org/tikv/raw/RawKVClientTest.java +++ b/src/test/java/org/tikv/raw/RawKVClientTest.java @@ -1084,4 +1084,20 @@ public int compare(ByteString startKey, ByteString endKey) { return FastByteComparisons.compareTo(startKey.toByteArray(), endKey.toByteArray()); } } + + @Test + public void testBatchPut() throws Exception { + TiConfiguration conf = session.getConf(); + conf.setRawKVBatchWriteTimeoutInMS(100000); + conf.setTimeout(100000); + try(TiSession newSession = TiSession.create(conf)){ + try(RawKVClient client=newSession.createRawClient()) { + HashMap kvs = new HashMap<>(); + for (int i = 0; i < 2048; i++) { + kvs.put(ByteString.copyFromUtf8("key@" + i), rawValue("value@" + i)); + } + client.batchPut(kvs); + } + }; + } } From af6ab502332db6f2a40aaadbcc648afcbbb58a34 Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Thu, 28 Jul 2022 16:51:20 +0800 Subject: [PATCH 2/4] fix rawBatchPut Signed-off-by: iosmanthus --- src/main/java/org/tikv/raw/RawKVClient.java | 18 +++++++-------- .../java/org/tikv/raw/RawKVClientTest.java | 22 ++++++++----------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/tikv/raw/RawKVClient.java b/src/main/java/org/tikv/raw/RawKVClient.java index 45fd0699ee9..368d2191847 100644 --- a/src/main/java/org/tikv/raw/RawKVClient.java +++ b/src/main/java/org/tikv/raw/RawKVClient.java @@ -749,17 +749,17 @@ private void doSendBatchPut( while (!taskQueue.isEmpty()) { List task = taskQueue.poll(); for (Batch batch : task) { - completionService.submit( - () -> doSendBatchPutInBatchesWithRetry(batch.getBackOffer(), batch, ttl)); + futureList.add(completionService.submit( + () -> doSendBatchPutInBatchesWithRetry(batch.getBackOffer(), batch, ttl))); + } - try { - getTasks(completionService, taskQueue, task, deadline - System.currentTimeMillis()); - } catch (Exception e) { - for (Future> future : futureList) { - future.cancel(true); - } - throw e; + try { + getTasks(completionService, taskQueue, task, deadline - System.currentTimeMillis()); + } catch (Exception e) { + for (Future> future : futureList) { + future.cancel(true); } + throw e; } } } diff --git a/src/test/java/org/tikv/raw/RawKVClientTest.java b/src/test/java/org/tikv/raw/RawKVClientTest.java index 39464aca3a2..9e926f3fd83 100644 --- a/src/test/java/org/tikv/raw/RawKVClientTest.java +++ b/src/test/java/org/tikv/raw/RawKVClientTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.tikv.raw.RawKVClientBase.MAX_RAW_BATCH_LIMIT; import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; @@ -1086,18 +1087,13 @@ public int compare(ByteString startKey, ByteString endKey) { } @Test - public void testBatchPut() throws Exception { - TiConfiguration conf = session.getConf(); - conf.setRawKVBatchWriteTimeoutInMS(100000); - conf.setTimeout(100000); - try(TiSession newSession = TiSession.create(conf)){ - try(RawKVClient client=newSession.createRawClient()) { - HashMap kvs = new HashMap<>(); - for (int i = 0; i < 2048; i++) { - kvs.put(ByteString.copyFromUtf8("key@" + i), rawValue("value@" + i)); - } - client.batchPut(kvs); - } - }; + public void testBatchPutForIssue634() { + ByteString prefix = ByteString.copyFromUtf8("testBatchPutForIssue634"); + client.deletePrefix(prefix); + HashMap kvs = new HashMap<>(); + for (int i = 0; i < MAX_RAW_BATCH_LIMIT * 4; i++) { + kvs.put(prefix.concat(ByteString.copyFromUtf8("key@" + i)), rawValue("value@" + i)); + } + client.batchPut(kvs); } } From cdd70e9f447007d9336f33c07ba0027989478b8b Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Fri, 29 Jul 2022 11:31:04 +0800 Subject: [PATCH 3/4] ./dev/javafmt Signed-off-by: iosmanthus --- src/main/java/org/tikv/raw/RawKVClient.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/tikv/raw/RawKVClient.java b/src/main/java/org/tikv/raw/RawKVClient.java index 368d2191847..3fa292c9d80 100644 --- a/src/main/java/org/tikv/raw/RawKVClient.java +++ b/src/main/java/org/tikv/raw/RawKVClient.java @@ -749,8 +749,9 @@ private void doSendBatchPut( while (!taskQueue.isEmpty()) { List task = taskQueue.poll(); for (Batch batch : task) { - futureList.add(completionService.submit( - () -> doSendBatchPutInBatchesWithRetry(batch.getBackOffer(), batch, ttl))); + futureList.add( + completionService.submit( + () -> doSendBatchPutInBatchesWithRetry(batch.getBackOffer(), batch, ttl))); } try { From e2406e0d926530495fe829422703233bc3f939ec Mon Sep 17 00:00:00 2001 From: iosmanthus Date: Fri, 29 Jul 2022 14:39:50 +0800 Subject: [PATCH 4/4] fix unbounded scan range Signed-off-by: iosmanthus --- src/test/java/org/tikv/raw/RawKVClientTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/tikv/raw/RawKVClientTest.java b/src/test/java/org/tikv/raw/RawKVClientTest.java index 9e926f3fd83..08608ae11aa 100644 --- a/src/test/java/org/tikv/raw/RawKVClientTest.java +++ b/src/test/java/org/tikv/raw/RawKVClientTest.java @@ -361,7 +361,7 @@ public void scan0test() { } int i = 0; - Iterator iter = client.scan0(prefix, ByteString.EMPTY, cnt); + Iterator iter = client.scanPrefix0(prefix, cnt, false); while (iter.hasNext()) { i++; KvPair pair = iter.next(); @@ -370,7 +370,7 @@ public void scan0test() { assertEquals(cnt, i); i = 0; - iter = client.scan0(prefix, ByteString.EMPTY, true); + iter = client.scanPrefix0(prefix, true); while (iter.hasNext()) { i++; KvPair pair = iter.next(); @@ -398,7 +398,7 @@ public void ingestTest() { }); client.ingest(kvs); - assertEquals(client.scan(prefix, ByteString.EMPTY).size(), cnt); + assertEquals(client.scanPrefix(prefix).size(), cnt); } @Test