From 5f8023672639f957f89461b29ff1b485403aea4c Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 14:58:34 +0800 Subject: [PATCH 1/8] select tiflash randomly Signed-off-by: shiyuhang <1136742008@qq.com> --- .../java/org/tikv/common/region/RegionManager.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/tikv/common/region/RegionManager.java b/src/main/java/org/tikv/common/region/RegionManager.java index 44c81375107..9612913c753 100644 --- a/src/main/java/org/tikv/common/region/RegionManager.java +++ b/src/main/java/org/tikv/common/region/RegionManager.java @@ -23,6 +23,7 @@ import io.prometheus.client.Histogram; import java.util.ArrayList; import java.util.List; +import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -41,6 +42,7 @@ import org.tikv.common.util.Pair; import org.tikv.kvproto.Metapb; import org.tikv.kvproto.Metapb.Peer; +import org.tikv.kvproto.Metapb.Store; import org.tikv.kvproto.Metapb.StoreState; import org.tikv.kvproto.Pdpb; @@ -191,17 +193,21 @@ public Pair getRegionStorePairByKey( Peer peer = region.getCurrentReplica(); store = getStoreById(peer.getStoreId(), backOffer); } else { - outerLoop: + List tiflashStores = new ArrayList<>(); for (Peer peer : region.getLearnerList()) { TiStore s = getStoreById(peer.getStoreId(), backOffer); for (Metapb.StoreLabel label : s.getStore().getLabelsList()) { if (label.getKey().equals(storeType.getLabelKey()) && label.getValue().equals(storeType.getLabelValue())) { - store = s; - break outerLoop; + tiflashStores.add(s); } } } + // select a tiflash randomly + Random random = new Random(); + int randomIndex = random.nextInt(tiflashStores.size()); + store = tiflashStores.get(randomIndex); + if (store == null) { // clear the region cache, so we may get the learner peer next time cache.invalidateRegion(region); From 3f59725c1f7485e67e9d6fb62ae528329ffd9ee4 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 15:14:44 +0800 Subject: [PATCH 2/8] fix npe Signed-off-by: shiyuhang <1136742008@qq.com> --- src/main/java/org/tikv/common/region/RegionManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/tikv/common/region/RegionManager.java b/src/main/java/org/tikv/common/region/RegionManager.java index 9612913c753..b5bf64313dc 100644 --- a/src/main/java/org/tikv/common/region/RegionManager.java +++ b/src/main/java/org/tikv/common/region/RegionManager.java @@ -204,9 +204,9 @@ public Pair getRegionStorePairByKey( } } // select a tiflash randomly - Random random = new Random(); - int randomIndex = random.nextInt(tiflashStores.size()); - store = tiflashStores.get(randomIndex); + if(tiflashStores.size() > 0) { + store = tiflashStores.get(new Random().nextInt(tiflashStores.size())); + } if (store == null) { // clear the region cache, so we may get the learner peer next time From cfaa60cfc98465927a168ac83db5deff8fddfbd8 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 15:19:59 +0800 Subject: [PATCH 3/8] delete useless import Signed-off-by: shiyuhang <1136742008@qq.com> --- src/main/java/org/tikv/common/region/RegionManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/tikv/common/region/RegionManager.java b/src/main/java/org/tikv/common/region/RegionManager.java index b5bf64313dc..2cd60d354d1 100644 --- a/src/main/java/org/tikv/common/region/RegionManager.java +++ b/src/main/java/org/tikv/common/region/RegionManager.java @@ -42,7 +42,6 @@ import org.tikv.common.util.Pair; import org.tikv.kvproto.Metapb; import org.tikv.kvproto.Metapb.Peer; -import org.tikv.kvproto.Metapb.Store; import org.tikv.kvproto.Metapb.StoreState; import org.tikv.kvproto.Pdpb; From 18433432e2238fabf8445f4f8f3413dadc90e7b1 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 15:42:15 +0800 Subject: [PATCH 4/8] use RR strategy Signed-off-by: shiyuhang <1136742008@qq.com> --- src/main/java/org/tikv/common/ConfigUtils.java | 1 + .../java/org/tikv/common/region/RegionManager.java | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/tikv/common/ConfigUtils.java b/src/main/java/org/tikv/common/ConfigUtils.java index 4697db8140a..9a73f9be1fc 100644 --- a/src/main/java/org/tikv/common/ConfigUtils.java +++ b/src/main/java/org/tikv/common/ConfigUtils.java @@ -121,6 +121,7 @@ public class ConfigUtils { public static final String TIFLASH_ENABLE = "tiflash.enable"; public static final String TIKV_WARM_UP_ENABLE = "tikv.warm_up.enable"; + public static final String TIFLASH_BALANCE_STRATEGY = "tiflash.balance_strategy"; public static final String TIKV_API_VERSION = "tikv.api_version"; diff --git a/src/main/java/org/tikv/common/region/RegionManager.java b/src/main/java/org/tikv/common/region/RegionManager.java index 2cd60d354d1..0ecd416a3f2 100644 --- a/src/main/java/org/tikv/common/region/RegionManager.java +++ b/src/main/java/org/tikv/common/region/RegionManager.java @@ -23,7 +23,6 @@ import io.prometheus.client.Histogram; import java.util.ArrayList; import java.util.List; -import java.util.Random; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -69,6 +68,7 @@ public class RegionManager { private final TiConfiguration conf; private final ScheduledExecutorService executor; private final StoreHealthyChecker storeChecker; + private int TiFlashStoreIndex = 0; public RegionManager( TiConfiguration conf, ReadOnlyPDClient pdClient, ChannelFactory channelFactory) { @@ -202,9 +202,11 @@ public Pair getRegionStorePairByKey( } } } - // select a tiflash randomly - if(tiflashStores.size() > 0) { - store = tiflashStores.get(new Random().nextInt(tiflashStores.size())); + // select a tiflash with RR strategy + if (tiflashStores.size() > 0) { + store = + tiflashStores.get(TiFlashStoreIndex > tiflashStores.size() - 1 ? 0 : TiFlashStoreIndex); + TiFlashStoreIndex++; } if (store == null) { From 67e3affb4e3461bc23599a6111a972fb567bf297 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 15:43:18 +0800 Subject: [PATCH 5/8] typo Signed-off-by: shiyuhang <1136742008@qq.com> --- src/main/java/org/tikv/common/ConfigUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/tikv/common/ConfigUtils.java b/src/main/java/org/tikv/common/ConfigUtils.java index 9a73f9be1fc..4697db8140a 100644 --- a/src/main/java/org/tikv/common/ConfigUtils.java +++ b/src/main/java/org/tikv/common/ConfigUtils.java @@ -121,7 +121,6 @@ public class ConfigUtils { public static final String TIFLASH_ENABLE = "tiflash.enable"; public static final String TIKV_WARM_UP_ENABLE = "tikv.warm_up.enable"; - public static final String TIFLASH_BALANCE_STRATEGY = "tiflash.balance_strategy"; public static final String TIKV_API_VERSION = "tikv.api_version"; From fd253db0747f03022e0177a9d9ba892ed14dd2f5 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 16:58:08 +0800 Subject: [PATCH 6/8] add test Signed-off-by: shiyuhang <1136742008@qq.com> --- src/test/java/org/tikv/common/GrpcUtils.java | 5 ++ .../org/tikv/common/RegionManagerTest.java | 51 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/tikv/common/GrpcUtils.java b/src/test/java/org/tikv/common/GrpcUtils.java index e6793f01f1d..e7a268f6c31 100644 --- a/src/test/java/org/tikv/common/GrpcUtils.java +++ b/src/test/java/org/tikv/common/GrpcUtils.java @@ -24,6 +24,7 @@ import org.tikv.common.codec.Codec.BytesCodec; import org.tikv.common.codec.CodecDataOutput; import org.tikv.kvproto.Metapb.Peer; +import org.tikv.kvproto.Metapb.PeerRole; import org.tikv.kvproto.Metapb.Region; import org.tikv.kvproto.Metapb.RegionEpoch; import org.tikv.kvproto.Metapb.Store; @@ -61,6 +62,10 @@ public static Peer makePeer(long id, long storeId) { return Peer.newBuilder().setStoreId(storeId).setId(id).build(); } + public static Peer makeLearnerPeer(long id, long storeId) { + return Peer.newBuilder().setRole(PeerRole.Learner).setStoreId(storeId).setId(id).build(); + } + public static ByteString encodeKey(byte[] key) { CodecDataOutput cdo = new CodecDataOutput(); BytesCodec.writeBytes(cdo, key); diff --git a/src/test/java/org/tikv/common/RegionManagerTest.java b/src/test/java/org/tikv/common/RegionManagerTest.java index 6052640f9bc..eddd22a6c65 100644 --- a/src/test/java/org/tikv/common/RegionManagerTest.java +++ b/src/test/java/org/tikv/common/RegionManagerTest.java @@ -31,6 +31,7 @@ import org.tikv.common.region.RegionManager; import org.tikv.common.region.TiRegion; import org.tikv.common.region.TiStore; +import org.tikv.common.region.TiStoreType; import org.tikv.common.util.KeyRangeUtils; import org.tikv.common.util.Pair; import org.tikv.kvproto.Metapb; @@ -135,7 +136,7 @@ public void getStoreByKey() { Pair pair = mgr.getRegionStorePairByKey(searchKey); assertEquals(pair.first.getId(), regionId); - assertEquals(pair.first.getId(), storeId); + assertEquals(pair.second.getId(), 10); } @Test @@ -179,4 +180,52 @@ public void getStoreById() { } catch (Exception ignored) { } } + + @Test + public void getRegionStorePairByKeyWithTiFlash() { + + ByteString startKey = ByteString.copyFrom(new byte[] {1}); + ByteString endKey = ByteString.copyFrom(new byte[] {10}); + ByteString searchKey = ByteString.copyFrom(new byte[] {5}); + String testAddress = "testAddress"; + long firstStoreId = 233; + long secondStoreId = 234; + int confVer = 1026; + int ver = 1027; + long regionId = 233; + leader.addGetRegionListener( + request -> + GrpcUtils.makeGetRegionResponse( + leader.getClusterId(), + GrpcUtils.makeRegion( + regionId, + GrpcUtils.encodeKey(startKey.toByteArray()), + GrpcUtils.encodeKey(endKey.toByteArray()), + GrpcUtils.makeRegionEpoch(confVer, ver), + GrpcUtils.makeLearnerPeer(1, firstStoreId), + GrpcUtils.makeLearnerPeer(2, secondStoreId)))); + + AtomicInteger i = new AtomicInteger(0); + long[] ids = new long[] {firstStoreId, secondStoreId}; + leader.addGetStoreListener( + (request -> + GrpcUtils.makeGetStoreResponse( + leader.getClusterId(), + GrpcUtils.makeStore( + ids[i.getAndIncrement()], + testAddress, + StoreState.Up, + GrpcUtils.makeStoreLabel("engine", "tiflash"), + GrpcUtils.makeStoreLabel("k1", "v1"), + GrpcUtils.makeStoreLabel("k2", "v2"))))); + + Pair pair = mgr.getRegionStorePairByKey(searchKey, TiStoreType.TiFlash); + assertEquals(pair.first.getId(), regionId); + assertEquals(pair.second.getId(), firstStoreId); + + Pair secondPair = + mgr.getRegionStorePairByKey(searchKey, TiStoreType.TiFlash); + assertEquals(secondPair.first.getId(), regionId); + assertEquals(secondPair.second.getId(), secondStoreId); + } } From 058f6ff40d728db87fd69d961f1a262bcc66d3f4 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 17:16:01 +0800 Subject: [PATCH 7/8] fix reviewer Signed-off-by: shiyuhang <1136742008@qq.com> --- src/main/java/org/tikv/common/region/RegionManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/tikv/common/region/RegionManager.java b/src/main/java/org/tikv/common/region/RegionManager.java index 0ecd416a3f2..45cfc5160d9 100644 --- a/src/main/java/org/tikv/common/region/RegionManager.java +++ b/src/main/java/org/tikv/common/region/RegionManager.java @@ -26,6 +26,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.tikv.common.ReadOnlyPDClient; @@ -68,7 +69,7 @@ public class RegionManager { private final TiConfiguration conf; private final ScheduledExecutorService executor; private final StoreHealthyChecker storeChecker; - private int TiFlashStoreIndex = 0; + private AtomicInteger tiflashStoreIndex = new AtomicInteger(0); public RegionManager( TiConfiguration conf, ReadOnlyPDClient pdClient, ChannelFactory channelFactory) { @@ -204,9 +205,7 @@ public Pair getRegionStorePairByKey( } // select a tiflash with RR strategy if (tiflashStores.size() > 0) { - store = - tiflashStores.get(TiFlashStoreIndex > tiflashStores.size() - 1 ? 0 : TiFlashStoreIndex); - TiFlashStoreIndex++; + store = tiflashStores.get(tiflashStoreIndex.getAndIncrement() % tiflashStores.size()); } if (store == null) { From 841c29888744de25bd0809cf716e580a3c00c4d5 Mon Sep 17 00:00:00 2001 From: shiyuhang <1136742008@qq.com> Date: Mon, 31 Oct 2022 17:24:52 +0800 Subject: [PATCH 8/8] fix license Signed-off-by: shiyuhang <1136742008@qq.com> --- .github/workflows/license-checker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/license-checker.yml b/.github/workflows/license-checker.yml index 4e1cf90a4fe..cd5c12f84c4 100644 --- a/.github/workflows/license-checker.yml +++ b/.github/workflows/license-checker.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Check License Header - uses: apache/skywalking-eyes@main + uses: apache/skywalking-eyes@v0.3.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: