From 389bc82a13f04fab87dd3ba65ef4310b71c77877 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 14 Jul 2023 13:46:33 +0800 Subject: [PATCH 1/3] build: Make sure tikv-client works on stable rust Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 16 ---------------- Makefile | 11 +---------- rust-toolchain.toml | 3 +++ src/lib.rs | 1 - 4 files changed, 4 insertions(+), 27 deletions(-) create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78d15155..99318793 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,12 +12,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - - run: rustup component add rustfmt clippy - name: Install Protoc uses: arduino/setup-protoc@v1 with: @@ -35,11 +29,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - name: Install Protoc uses: arduino/setup-protoc@v1 with: @@ -57,11 +46,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - override: true - name: Install Protoc uses: arduino/setup-protoc@v1 with: diff --git a/Makefile b/Makefile index 521bbf2d..a96d584a 100644 --- a/Makefile +++ b/Makefile @@ -26,18 +26,9 @@ integration-test: test: unit-test integration-test -doc: +doc: cargo doc --workspace --exclude tikv-client-proto --document-private-items --no-deps -# Deprecated -# docker-pd: -# docker run -d -v $(shell pwd)/config:/config --net=host --name pd --rm pingcap/pd:latest --name "pd" --data-dir "pd" --client-urls "http://127.0.0.1:2379" --advertise-client-urls "http://127.0.0.1:2379" --config /config/pd.toml - -# docker-kv: -# docker run -d -v $(shell pwd)/config:/config --net=host --name kv --rm --ulimit nofile=90000:90000 pingcap/tikv:latest --pd-endpoints "127.0.0.1:2379" --addr "127.0.0.1:2378" --data-dir "kv" --config /config/tikv.toml - -# docker: docker-pd docker-kv - tiup: tiup playground nightly --mode tikv-slim --kv 3 --without-monitor --kv.config $(shell pwd)/config/tikv.toml --pd.config $(shell pwd)/config/pd.toml & diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..73cb934d --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "stable" +components = ["rustfmt", "clippy"] diff --git a/src/lib.rs b/src/lib.rs index caf94623..e273bad7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,7 +93,6 @@ // To support both prost & rust-protobuf. #![cfg_attr(feature = "prost-codec", allow(clippy::useless_conversion))] #![allow(clippy::field_reassign_with_default)] -#![allow(clippy::arc_with_non_send_sync)] pub mod backoff; #[doc(hidden)] From 5d8660b21e7ee38f8e7f9c7eb7a64d8964f41423 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 14 Jul 2023 13:52:31 +0800 Subject: [PATCH 2/3] fix Signed-off-by: Xuanwo --- examples/raw.rs | 29 ++++++++++++++++------------- examples/transaction.rs | 8 ++++---- rustfmt.toml | 17 +++++++++-------- src/kv/codec.rs | 32 ++++++++++++++++++++------------ src/mock.rs | 4 +++- src/pd/client.rs | 34 ++++++++++++++++++++-------------- src/raw/client.rs | 35 ++++++++++++++++++++--------------- src/region_cache.rs | 33 +++++++++++++++++---------------- src/request/plan.rs | 18 ++++++++++++------ src/request/plan_builder.rs | 13 +++++++++---- src/transaction/buffer.rs | 19 +++++++++++-------- src/transaction/requests.rs | 27 ++++++++++++++++----------- src/util/iter.rs | 27 +++++++++++---------------- tests/integration_tests.rs | 9 ++++----- 14 files changed, 172 insertions(+), 133 deletions(-) diff --git a/examples/raw.rs b/examples/raw.rs index 32ccc97a..e5b24e1f 100644 --- a/examples/raw.rs +++ b/examples/raw.rs @@ -100,10 +100,10 @@ async fn main() -> Result<()> { .expect("Could not scan"); let keys: Vec<_> = pairs.into_iter().map(|p| p.key().clone()).collect(); - assert_eq!(&keys, &[ - Key::from("k1".to_owned()), - Key::from("k2".to_owned()), - ]); + assert_eq!( + &keys, + &[Key::from("k1".to_owned()), Key::from("k2".to_owned()),] + ); println!("Scanning from {start:?} to {end:?} gives: {keys:?}"); let k1 = "k1"; @@ -122,15 +122,18 @@ async fn main() -> Result<()> { .into_iter() .map(|p| String::from_utf8(p.1).unwrap()) .collect(); - assert_eq!(&vals, &[ - "v1".to_owned(), - "v2".to_owned(), - "v2".to_owned(), - "v3".to_owned(), - "v1".to_owned(), - "v2".to_owned(), - "v3".to_owned() - ]); + assert_eq!( + &vals, + &[ + "v1".to_owned(), + "v2".to_owned(), + "v2".to_owned(), + "v3".to_owned(), + "v1".to_owned(), + "v2".to_owned(), + "v3".to_owned() + ] + ); println!("Scanning batch scan from {batch_scan_keys:?} gives: {vals:?}"); // Cleanly exit. diff --git a/examples/transaction.rs b/examples/transaction.rs index d5dbdca2..119fc88c 100644 --- a/examples/transaction.rs +++ b/examples/transaction.rs @@ -110,10 +110,10 @@ async fn main() { let key1_exists = key_exists(&txn, key1.clone()).await; let key2: Key = b"key_not_exist".to_vec().into(); let key2_exists = key_exists(&txn, key2.clone()).await; - println!("check exists {:?}", vec![ - (key1, key1_exists), - (key2, key2_exists) - ]); + println!( + "check exists {:?}", + vec![(key1, key1_exists), (key2, key2_exists)] + ); // scan let key1: Key = b"key1".to_vec().into(); diff --git a/rustfmt.toml b/rustfmt.toml index 757638be..f2bfe396 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,10 +1,11 @@ edition = "2021" -version = "Two" reorder_imports = true -imports_granularity = "Item" -group_imports = "StdExternalCrate" -where_single_line = true -trailing_comma = "Vertical" -overflow_delimited_expr = true -format_code_in_doc_comments = true -normalize_comments = true + +# version = "Two" +# imports_granularity = "Item" +# group_imports = "StdExternalCrate" +# where_single_line = true +# trailing_comma = "Vertical" +# overflow_delimited_expr = true +# format_code_in_doc_comments = true +# normalize_comments = true diff --git a/src/kv/codec.rs b/src/kv/codec.rs index 3a92dfaf..29355dde 100644 --- a/src/kv/codec.rs +++ b/src/kv/codec.rs @@ -154,18 +154,26 @@ pub mod test { #[test] fn test_enc_dec_bytes() { let pairs = vec![ - (vec![], vec![0, 0, 0, 0, 0, 0, 0, 0, 247], vec![ - 255, 255, 255, 255, 255, 255, 255, 255, 8, - ]), - (vec![0], vec![0, 0, 0, 0, 0, 0, 0, 0, 248], vec![ - 255, 255, 255, 255, 255, 255, 255, 255, 7, - ]), - (vec![1, 2, 3], vec![1, 2, 3, 0, 0, 0, 0, 0, 250], vec![ - 254, 253, 252, 255, 255, 255, 255, 255, 5, - ]), - (vec![1, 2, 3, 0], vec![1, 2, 3, 0, 0, 0, 0, 0, 251], vec![ - 254, 253, 252, 255, 255, 255, 255, 255, 4, - ]), + ( + vec![], + vec![0, 0, 0, 0, 0, 0, 0, 0, 247], + vec![255, 255, 255, 255, 255, 255, 255, 255, 8], + ), + ( + vec![0], + vec![0, 0, 0, 0, 0, 0, 0, 0, 248], + vec![255, 255, 255, 255, 255, 255, 255, 255, 7], + ), + ( + vec![1, 2, 3], + vec![1, 2, 3, 0, 0, 0, 0, 0, 250], + vec![254, 253, 252, 255, 255, 255, 255, 255, 5], + ), + ( + vec![1, 2, 3, 0], + vec![1, 2, 3, 0, 0, 0, 0, 0, 251], + vec![254, 253, 252, 255, 255, 255, 255, 255, 4], + ), ( vec![1, 2, 3, 4, 5, 6, 7], vec![1, 2, 3, 4, 5, 6, 7, 0, 254], diff --git a/src/mock.rs b/src/mock.rs index 75b762ed..eada6a8e 100644 --- a/src/mock.rs +++ b/src/mock.rs @@ -57,7 +57,9 @@ pub struct MockKvClient { impl MockKvClient { pub fn with_dispatch_hook(dispatch: F) -> MockKvClient - where F: Fn(&dyn Any) -> Result> + Send + Sync + 'static { + where + F: Fn(&dyn Any) -> Result> + Send + Sync + 'static, + { MockKvClient { addr: String::new(), dispatch: Some(Arc::new(dispatch)), diff --git a/src/pd/client.rs b/src/pd/client.rs index d4b6e558..31f88968 100644 --- a/src/pd/client.rs +++ b/src/pd/client.rs @@ -372,16 +372,19 @@ pub mod test { let mut stream = executor::block_on_stream(stream); let result: Vec = stream.next().unwrap().unwrap().1; - assert_eq!(result, vec![ - vec![1].into(), - vec![2].into(), - vec![3].into(), - vec![5, 2].into() - ]); - assert_eq!(stream.next().unwrap().unwrap().1, vec![ - vec![12].into(), - vec![11, 4].into() - ]); + assert_eq!( + result, + vec![ + vec![1].into(), + vec![2].into(), + vec![3].into(), + vec![5, 2].into() + ] + ); + assert_eq!( + stream.next().unwrap().unwrap().1, + vec![vec![12].into(), vec![11, 4].into()] + ); assert!(stream.next().is_none()); } @@ -425,10 +428,13 @@ pub mod test { let ranges4 = stream.next().unwrap().unwrap(); assert_eq!(ranges1.0.id(), 1); - assert_eq!(ranges1.1, vec![ - make_key_range(k1.clone(), k2.clone()), - make_key_range(k1.clone(), k_split.clone()), - ]); + assert_eq!( + ranges1.1, + vec![ + make_key_range(k1.clone(), k2.clone()), + make_key_range(k1.clone(), k_split.clone()), + ] + ); assert_eq!(ranges2.0.id(), 2); assert_eq!(ranges2.1, vec![make_key_range(k_split.clone(), k3.clone())]); assert_eq!(ranges3.0.id(), 1); diff --git a/src/raw/client.rs b/src/raw/client.rs index e5123865..1ae4e7bc 100644 --- a/src/raw/client.rs +++ b/src/raw/client.rs @@ -693,21 +693,26 @@ mod tests { .into_iter() .map(|(data, ranges)| (String::from_utf8(data).unwrap(), ranges)) .collect(); - assert_eq!(resps, vec![ - ("1:[Key(05)..Key(0A)]".to_string(), vec![ - Key::from(vec![5])..Key::from(vec![10]) - ]), - ( - "2:[Key(0A)..Key(0F), Key(14)..Key(FAFA)]".to_string(), - vec![ - Key::from(vec![10])..Key::from(vec![15]), - Key::from(vec![20])..Key::from(vec![250, 250]) - ] - ), - ("3:[Key(FAFA)..Key()]".to_string(), vec![ - Key::from(vec![250, 250])..Key::from(vec![]) - ]) - ]); + assert_eq!( + resps, + vec![ + ( + "1:[Key(05)..Key(0A)]".to_string(), + vec![Key::from(vec![5])..Key::from(vec![10])] + ), + ( + "2:[Key(0A)..Key(0F), Key(14)..Key(FAFA)]".to_string(), + vec![ + Key::from(vec![10])..Key::from(vec![15]), + Key::from(vec![20])..Key::from(vec![250, 250]) + ] + ), + ( + "3:[Key(FAFA)..Key()]".to_string(), + vec![Key::from(vec![250, 250])..Key::from(vec![])] + ) + ] + ); Ok(()) } } diff --git a/src/region_cache.rs b/src/region_cache.rs index 33be7608..6e8fe0ab 100644 --- a/src/region_cache.rs +++ b/src/region_cache.rs @@ -318,11 +318,9 @@ mod test { async fn cache_is_used() -> Result<()> { let retry_client = Arc::new(MockRetryClient::default()); let cache = RegionCache::new(retry_client.clone()); - retry_client - .regions - .lock() - .await - .insert(1, RegionWithLeader { + retry_client.regions.lock().await.insert( + 1, + RegionWithLeader { region: metapb::Region { id: 1, start_key: vec![], @@ -337,12 +335,11 @@ mod test { store_id: 1, ..Default::default() }), - }); - retry_client - .regions - .lock() - .await - .insert(2, RegionWithLeader { + }, + ); + retry_client.regions.lock().await.insert( + 2, + RegionWithLeader { region: metapb::Region { id: 2, start_key: vec![101], @@ -357,7 +354,8 @@ mod test { store_id: 2, ..Default::default() }), - }); + }, + ); assert_eq!(retry_client.get_region_count.load(SeqCst), 0); @@ -378,10 +376,13 @@ mod test { // update leader should work cache - .update_leader(cache.get_region_by_id(2).await?.ver_id(), metapb::Peer { - store_id: 102, - ..Default::default() - }) + .update_leader( + cache.get_region_by_id(2).await?.ver_id(), + metapb::Peer { + store_id: 102, + ..Default::default() + }, + ) .await?; assert_eq!( cache.get_region_by_id(2).await?.leader.unwrap().store_id, diff --git a/src/request/plan.rs b/src/request/plan.rs index 6448538e..905e7fad 100644 --- a/src/request/plan.rs +++ b/src/request/plan.rs @@ -87,7 +87,8 @@ pub struct RetryableMultiRegion { } impl RetryableMultiRegion -where P::Result: HasKeyErrors + HasRegionError +where + P::Result: HasKeyErrors + HasRegionError, { // A plan may involve multiple shards #[async_recursion] @@ -286,7 +287,8 @@ impl Clone for RetryableMultiRegion { #[async_trait] impl Plan for RetryableMultiRegion -where P::Result: HasKeyErrors + HasRegionError +where + P::Result: HasKeyErrors + HasRegionError, { type Result = Vec>; @@ -417,7 +419,8 @@ impl Clone for ResolveLock { #[async_trait] impl Plan for ResolveLock -where P::Result: HasLocks +where + P::Result: HasLocks, { type Result = P::Result; @@ -517,7 +520,8 @@ impl Clone for CleanupLocks { #[async_trait] impl Plan for CleanupLocks -where P::Result: HasLocks + HasNextBatch + HasKeyErrors + HasRegionError +where + P::Result: HasLocks + HasNextBatch + HasKeyErrors + HasRegionError, { type Result = CleanupLocksResult; @@ -621,7 +625,8 @@ impl Clone for ExtractError

{ #[async_trait] impl Plan for ExtractError

-where P::Result: HasKeyErrors + HasRegionErrors +where + P::Result: HasKeyErrors + HasRegionErrors, { type Result = P::Result; @@ -663,7 +668,8 @@ impl Clone for PreserveShard

{ #[async_trait] impl

Plan for PreserveShard

-where P: Plan + Shardable +where + P: Plan + Shardable, { type Result = ResponseWithShard; diff --git a/src/request/plan_builder.rs b/src/request/plan_builder.rs index ddced0d4..5ce2350c 100644 --- a/src/request/plan_builder.rs +++ b/src/request/plan_builder.rs @@ -69,7 +69,9 @@ impl PlanBuilder { impl PlanBuilder { /// If there is a lock error, then resolve the lock and retry the request. pub fn resolve_lock(self, backoff: Backoff) -> PlanBuilder, Ph> - where P::Result: HasLocks { + where + P::Result: HasLocks, + { PlanBuilder { pd_client: self.pd_client.clone(), plan: ResolveLock { @@ -143,7 +145,8 @@ impl PlanBuilder { } impl PlanBuilder -where P::Result: HasKeyErrors + HasRegionError +where + P::Result: HasKeyErrors + HasRegionError, { /// Split the request into shards sending a request to the region of each shard. pub fn retry_multi_region( @@ -191,7 +194,8 @@ impl PlanBuilder, NoTarget> { } impl PlanBuilder -where P::Result: HasKeyErrors +where + P::Result: HasKeyErrors, { pub fn preserve_shard(self) -> PlanBuilder, NoTarget> { PlanBuilder { @@ -206,7 +210,8 @@ where P::Result: HasKeyErrors } impl PlanBuilder -where P::Result: HasKeyErrors + HasRegionErrors +where + P::Result: HasKeyErrors + HasRegionErrors, { pub fn extract_error(self) -> PlanBuilder, Targetted> { PlanBuilder { diff --git a/src/transaction/buffer.rs b/src/transaction/buffer.rs index dbf9079d..b2f19717 100644 --- a/src/transaction/buffer.rs +++ b/src/transaction/buffer.rs @@ -490,15 +490,18 @@ mod tests { ready(Ok(vec![])) }), ); - assert_eq!(r1.unwrap().collect::>(), vec![ - KvPair(k1.clone(), v1.clone()), - KvPair(k2.clone(), v2.clone()) - ]); + assert_eq!( + r1.unwrap().collect::>(), + vec![ + KvPair(k1.clone(), v1.clone()), + KvPair(k2.clone(), v2.clone()) + ] + ); assert_eq!(r2.unwrap().unwrap(), v2); - assert_eq!(r3.unwrap().collect::>(), vec![ - KvPair(k1, v1), - KvPair(k2, v2) - ]); + assert_eq!( + r3.unwrap().collect::>(), + vec![KvPair(k1, v1), KvPair(k2, v2)] + ); } // Check that multiple writes to the same key combine in the correct way. diff --git a/src/transaction/requests.rs b/src/transaction/requests.rs index 201f5ce4..9cc4993f 100644 --- a/src/transaction/requests.rs +++ b/src/transaction/requests.rs @@ -508,7 +508,11 @@ impl Merge { // FIXME: implement other iterator methods like size_hint, etc. impl< - T: IntoIterator, - U: Iterator>, - F: FnMut(I) -> T, - Ti: Iterator, - I, - E, -> Iterator for FlatMapOk + T: IntoIterator, + U: Iterator>, + F: FnMut(I) -> T, + Ti: Iterator, + I, + E, + > Iterator for FlatMapOk { type Item = std::result::Result; @@ -92,14 +92,9 @@ mod test { .into_iter() .flat_map_ok(|i| vec![i, i, i].into_iter()) .collect(); - assert_eq!(result, vec![ - Ok(0), - Ok(0), - Ok(0), - Err(()), - Ok(2), - Ok(2), - Ok(2) - ]); + assert_eq!( + result, + vec![Ok(0), Ok(0), Ok(0), Err(()), Ok(2), Ok(2), Ok(2)] + ); } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4bf485c4..8244b13c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -747,11 +747,10 @@ async fn txn_lock_keys_error_handle() -> Result<()> { let mut t3 = client.begin_pessimistic().await?; t1.lock_keys(vec![k[0].clone(), k[1].clone()]).await?; - assert!( - t2.lock_keys(vec![k[0].clone(), k[2].clone()]) - .await - .is_err() - ); + assert!(t2 + .lock_keys(vec![k[0].clone(), k[2].clone()]) + .await + .is_err()); t3.lock_keys(vec![k[2].clone(), k[3].clone()]).await?; t1.rollback().await?; From a2e8c154fd4f713f4635ed8938cb1e056da4cbf3 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 16 Jul 2023 23:14:22 +0800 Subject: [PATCH 3/3] Add comments Signed-off-by: Xuanwo --- rustfmt.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rustfmt.toml b/rustfmt.toml index f2bfe396..b88a546f 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,6 +1,9 @@ edition = "2021" reorder_imports = true +# Those options are disabled for not supported in stable rust. +# We should enable them back once stabilized. +# # version = "Two" # imports_granularity = "Item" # group_imports = "StdExternalCrate"