Skip to content

Fix gRPC forward bug when store is unreachable #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
marsishandsome merged 3 commits into from
Nov 8, 2021
Merged

Fix gRPC forward bug when store is unreachable #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cc80097
fix
Little-Wallace Nov 5, 2021
cafaf67
modify default backoff settings
birdstorm Nov 5, 2021
1e5a177
format
birdstorm Nov 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/org/tikv/common/ConfigUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ public class ConfigUtils {
public static final int DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = 10000;
public static final int DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = 600000;

public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 100;
public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 20;

public static final String NORMAL_COMMAND_PRIORITY = "NORMAL";
public static final String LOW_COMMAND_PRIORITY = "LOW";
Expand Down
28 changes: 19 additions & 9 deletions src/main/java/org/tikv/common/region/AbstractRegionStoreClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,7 @@ public boolean onNotLeader(TiRegion newRegion) {
@Override
public boolean onStoreUnreachable() {
if (!targetStore.isValid()) {
logger.warn(
String.format("store [%d] has been invalid", region.getId(), targetStore.getId()));
logger.warn(String.format("store [%d] has been invalid", targetStore.getId()));
targetStore = regionManager.getStoreById(targetStore.getId());
updateClientStub();
return true;
Expand Down Expand Up @@ -159,7 +158,7 @@ public boolean onStoreUnreachable() {
}
logger.warn(
String.format(
"retry time exceed for region[%d], invalid this region[%d]",
"retry time exceed for region[%d], invalid store[%d]",
region.getId(), targetStore.getId()));
regionManager.onRequestFail(region);
return false;
Expand Down Expand Up @@ -194,11 +193,6 @@ public void tryUpdateRegionStore() {
// create a new store object, which is can-forward.
regionManager.updateStore(originStore, targetStore);
} else {
// If we try to forward request to leader by follower failed, it means that the store of old
// leader may be
// unavailable but the new leader has not been report to PD. So we can ban this store for a
// short time to
// avoid too many request try forward rather than try other peer.
originStore.forwardFail();
}
}
Expand Down Expand Up @@ -260,7 +254,18 @@ private void updateClientStub() {
}

private boolean retryOtherStoreByProxyForward() {
if (!conf.getEnableGrpcForward() || retryForwardTimes > region.getFollowerList().size()) {
if (!conf.getEnableGrpcForward()) {
return false;
}
if (retryForwardTimes >= region.getFollowerList().size()) {
// If we try to forward request to leader by follower failed, it means that the store of old
// leader may be
// unavailable but the new leader has not been report to PD. So we can ban this store for a
// short time to
// avoid too many request try forward rather than try other peer.
if (originStore != null) {
originStore.forwardFail();
}
return false;
}
TiStore proxyStore = switchProxyStore();
Expand All @@ -269,6 +274,11 @@ private boolean retryOtherStoreByProxyForward() {
String.format(
"no forward store can be selected for store [%s] and region[%d]",
targetStore.getStore().getAddress(), region.getId()));
if (originStore != null) {
originStore.forwardFail();
} else {
targetStore.forwardFail();
}
return false;
}
if (originStore == null) {
Expand Down