From 981eead76ad42ef13676f8598c07003b093e33fc Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Thu, 4 May 2023 16:23:42 +0800 Subject: [PATCH 1/7] port code to GdriveCore & add path_2_id cache --- core/src/services/gdrive/backend.rs | 167 ++-------------------- core/src/services/gdrive/core.rs | 209 ++++++++++++++++++++++++++++ core/src/services/gdrive/mod.rs | 1 + core/src/services/gdrive/writer.rs | 13 +- 4 files changed, 232 insertions(+), 158 deletions(-) create mode 100644 core/src/services/gdrive/core.rs diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index babda4279bc5..6d1fca844f4d 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -18,7 +18,8 @@ use async_trait::async_trait; use http::{header, Request, Response, StatusCode}; use serde::Deserialize; -use std::fmt::Debug; +use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; +use tokio::sync::Mutex; use crate::{ ops::{OpDelete, OpRead, OpWrite}, @@ -30,33 +31,26 @@ use crate::{ Capability, Error, ErrorKind, }; -use super::{error::parse_error, writer::GdriveWriter}; +use super::{core::GdriveCore, error::parse_error, writer::GdriveWriter}; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct GdriveBackend { - root: String, - access_token: String, - client: HttpClient, + core: Arc, } impl GdriveBackend { pub(crate) fn new(root: String, access_token: String, http_client: HttpClient) -> Self { GdriveBackend { - root, - access_token, - client: http_client, + core: Arc::new(GdriveCore { + root, + access_token, + client: http_client, + path_2_id: Arc::new(Mutex::new(HashMap::new())), + }), } } } -impl Debug for GdriveBackend { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut de = f.debug_struct("GoolgeDriveBackend"); - de.field("root", &self.root); - de.finish() - } -} - #[async_trait] impl Accessor for GdriveBackend { type Reader = IncomingAsyncBody; @@ -69,7 +63,7 @@ impl Accessor for GdriveBackend { fn info(&self) -> AccessorInfo { let mut ma = AccessorInfo::default(); ma.set_scheme(crate::Scheme::Gdrive) - .set_root(&self.root) + .set_root(&self.core.root) .set_capability(Capability { read: true, write: true, @@ -81,7 +75,7 @@ impl Accessor for GdriveBackend { } async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> { - let resp = self.gdrive_get(path).await?; + let resp = self.core.gdrive_get(path).await?; let status = resp.status(); @@ -104,12 +98,12 @@ impl Accessor for GdriveBackend { Ok(( RpWrite::default(), - GdriveWriter::new(self.clone(), args, String::from(path)), + GdriveWriter::new(self.core.clone(), args, String::from(path)), )) } async fn delete(&self, path: &str, _: OpDelete) -> Result { - let resp = self.gdrive_delete(path).await?; + let resp = self.core.gdrive_delete(path).await?; let status = resp.status(); @@ -119,134 +113,3 @@ impl Accessor for GdriveBackend { } } } - -impl GdriveBackend { - async fn get_abs_root_id(&self) -> String { - let mut req = Request::get("https://www.googleapis.com/drive/v3/files/root"); - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - let req = req - .body(AsyncBody::Empty) - .map_err(new_request_build_error) - .unwrap(); - - let resp = self.client.send(req).await.unwrap(); - - let body_value: GdriveFile = - serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); - let root_id = String::from(body_value.id.as_str()); - root_id - } - - async fn get_file_id_by_path(&self, file_path: &str) -> String { - let path = build_rooted_abs_path(&self.root, file_path); - let auth_header_content = format!("Bearer {}", self.access_token); - - let mut parent_id = self.get_abs_root_id().await; - let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); - - for (i, item) in file_path_items.iter().enumerate() { - let mut query = format!( - "name = '{}' and parents = '{}' and trashed = false", - item, parent_id - ); - if i != file_path_items.len() - 1 { - query += "and mimeType = 'application/vnd.google-apps.folder'"; - } - let query: String = query.chars().filter(|c| !c.is_whitespace()).collect(); - - let mut req = Request::get(format!( - "https://www.googleapis.com/drive/v3/files?q={}", - query - )); - req = req.header(header::AUTHORIZATION, &auth_header_content); - let req = req - .body(AsyncBody::default()) - .map_err(new_request_build_error) - .unwrap(); - - let resp = self.client.send(req).await.unwrap(); - - let body_value: GdriveFileList = - serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); - parent_id = String::from(body_value.files[0].id.as_str()); - } - - parent_id - } - - async fn gdrive_get(&self, path: &str) -> Result> { - let url: String = format!( - "https://www.googleapis.com/drive/v3/files/{}?alt=media", - self.get_file_id_by_path(path).await - ); - - let auth_header_content = format!("Bearer {}", self.access_token); - let mut req = Request::get(&url); - req = req.header(header::AUTHORIZATION, auth_header_content); - - let req = req - .body(AsyncBody::Empty) - .map_err(new_request_build_error)?; - - self.client.send(req).await - } - - pub async fn gdrive_update( - &self, - path: &str, - size: Option, - content_type: Option<&str>, - body: AsyncBody, - ) -> Result> { - let url = format!( - "https://www.googleapis.com/upload/drive/v3/files/{}", - self.get_file_id_by_path(path).await - ); - - let mut req = Request::patch(&url); - - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - - if let Some(size) = size { - req = req.header(header::CONTENT_LENGTH, size) - } - - if let Some(mime) = content_type { - req = req.header(header::CONTENT_TYPE, mime) - } - - let req = req.body(body).map_err(new_request_build_error)?; - - self.client.send(req).await - } - - async fn gdrive_delete(&self, path: &str) -> Result> { - let url = format!( - "https://www.googleapis.com/drive/v3/files/{}", - self.get_file_id_by_path(path).await - ); - - let mut req = Request::delete(&url); - - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - - let req = req - .body(AsyncBody::Empty) - .map_err(new_request_build_error)?; - - self.client.send(req).await - } -} - -#[derive(Deserialize)] -struct GdriveFile { - id: String, -} - -#[derive(Deserialize)] -struct GdriveFileList { - files: Vec, -} diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs new file mode 100644 index 000000000000..9320b172da7b --- /dev/null +++ b/core/src/services/gdrive/core.rs @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; +use std::sync::Arc; + +use crate::raw::HttpClient; + +use async_trait::async_trait; +use http::{header, Request, Response, StatusCode}; +use serde::Deserialize; +use tokio::sync::Mutex; + +use crate::{ + ops::{OpDelete, OpRead, OpWrite}, + raw::{ + build_rooted_abs_path, new_request_build_error, parse_into_metadata, Accessor, + AccessorInfo, AsyncBody, IncomingAsyncBody, RpDelete, RpRead, RpWrite, + }, + types::Result, + Capability, Error, ErrorKind, +}; + +pub struct GdriveCore { + pub root: String, + pub access_token: String, + pub client: HttpClient, + pub path_2_id: Arc>>, +} + +impl Debug for GdriveCore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("GdriveCore"); + de.field("root", &self.root); + de.finish() + } +} + +impl GdriveCore { + async fn get_abs_root_id(&self) -> String { + let root = "root"; + + { + let cache_guard = self.path_2_id.lock().await; + if cache_guard.contains_key(root) { + return cache_guard.get(root).unwrap().to_string(); + } + } + + let mut req = Request::get("https://www.googleapis.com/drive/v3/files/root"); + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error) + .unwrap(); + + let resp = self.client.send(req).await.unwrap(); + + let body_value: GdriveFile = + serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); + let root_id = String::from(body_value.id.as_str()); + + let mut cache_guard = self.path_2_id.lock().await; + cache_guard.insert(root.to_owned(), root_id.clone()); + + root_id + } + + async fn get_file_id_by_path(&self, file_path: &str) -> String { + let path = build_rooted_abs_path(&self.root, file_path); + + { + let mut cache_guard = self.path_2_id.lock().await; + if cache_guard.contains_key(&path) { + return cache_guard.get(&path).unwrap().to_string(); + } + } + + let auth_header_content = format!("Bearer {}", self.access_token); + + let mut parent_id = self.get_abs_root_id().await; + let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); + + for (i, item) in file_path_items.iter().enumerate() { + let mut query = format!( + "name = '{}' and parents = '{}' and trashed = false", + item, parent_id + ); + if i != file_path_items.len() - 1 { + query += "and mimeType = 'application/vnd.google-apps.folder'"; + } + let query: String = query.chars().filter(|c| !c.is_whitespace()).collect(); + + let mut req = Request::get(format!( + "https://www.googleapis.com/drive/v3/files?q={}", + query + )); + req = req.header(header::AUTHORIZATION, &auth_header_content); + let req = req + .body(AsyncBody::default()) + .map_err(new_request_build_error) + .unwrap(); + + let resp = self.client.send(req).await.unwrap(); + + let body_value: GdriveFileList = + serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); + parent_id = String::from(body_value.files[0].id.as_str()); + println!("id: {}", parent_id); + } + + let mut cache_guard = self.path_2_id.lock().await; + cache_guard.insert(path, parent_id.clone()); + + parent_id + } + + pub async fn gdrive_get(&self, path: &str) -> Result> { + let url: String = format!( + "https://www.googleapis.com/drive/v3/files/{}?alt=media", + self.get_file_id_by_path(path).await + ); + + let auth_header_content = format!("Bearer {}", self.access_token); + let mut req = Request::get(&url); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } + + pub async fn gdrive_update( + &self, + path: &str, + size: Option, + content_type: Option<&str>, + body: AsyncBody, + ) -> Result> { + let url = format!( + "https://www.googleapis.com/upload/drive/v3/files/{}", + self.get_file_id_by_path(path).await + ); + + let mut req = Request::patch(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + if let Some(size) = size { + req = req.header(header::CONTENT_LENGTH, size) + } + + if let Some(mime) = content_type { + req = req.header(header::CONTENT_TYPE, mime) + } + + let req = req.body(body).map_err(new_request_build_error)?; + + self.client.send(req).await + } + + pub async fn gdrive_delete(&self, path: &str) -> Result> { + let url = format!( + "https://www.googleapis.com/drive/v3/files/{}", + self.get_file_id_by_path(path).await + ); + + let mut req = Request::delete(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } +} + +#[derive(Deserialize)] +struct GdriveFile { + id: String, +} + +#[derive(Deserialize)] +struct GdriveFileList { + files: Vec, +} diff --git a/core/src/services/gdrive/mod.rs b/core/src/services/gdrive/mod.rs index 7c88f4fe7458..5f77ecc50b21 100644 --- a/core/src/services/gdrive/mod.rs +++ b/core/src/services/gdrive/mod.rs @@ -17,6 +17,7 @@ mod backend; mod builder; +mod core; mod error; pub use builder::GdriveBuilder as Gdrive; diff --git a/core/src/services/gdrive/writer.rs b/core/src/services/gdrive/writer.rs index de9067aad0e2..dcff07d29b90 100644 --- a/core/src/services/gdrive/writer.rs +++ b/core/src/services/gdrive/writer.rs @@ -15,26 +15,27 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use async_trait::async_trait; use bytes::Bytes; use http::StatusCode; -use super::backend::GdriveBackend; +use super::core::GdriveCore; use super::error::parse_error; use crate::ops::OpWrite; use crate::raw::*; use crate::*; pub struct GdriveWriter { - backend: GdriveBackend, - + core: Arc, op: OpWrite, path: String, } impl GdriveWriter { - pub fn new(backend: GdriveBackend, op: OpWrite, path: String) -> Self { - GdriveWriter { backend, op, path } + pub fn new(core: Arc, op: OpWrite, path: String) -> Self { + GdriveWriter { core, op, path } } } @@ -42,7 +43,7 @@ impl GdriveWriter { impl oio::Write for GdriveWriter { async fn write(&mut self, bs: Bytes) -> Result<()> { let resp = self - .backend + .core .gdrive_update( &self.path, Some(bs.len()), From 3b99d724784d02d300753c3407f5dcb552b7f2c6 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Thu, 4 May 2023 16:29:34 +0800 Subject: [PATCH 2/7] make clippy happy --- core/src/services/gdrive/backend.rs | 10 +++++----- core/src/services/gdrive/core.rs | 13 +++---------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index 6d1fca844f4d..fc248e7d509c 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -16,16 +16,16 @@ // under the License. use async_trait::async_trait; -use http::{header, Request, Response, StatusCode}; -use serde::Deserialize; -use std::{collections::HashMap, fmt::Debug, hash::Hash, sync::Arc}; +use http::StatusCode; + +use std::{collections::HashMap, fmt::Debug, sync::Arc}; use tokio::sync::Mutex; use crate::{ ops::{OpDelete, OpRead, OpWrite}, raw::{ - build_rooted_abs_path, new_request_build_error, parse_into_metadata, Accessor, - AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpDelete, RpRead, RpWrite, + parse_into_metadata, Accessor, AccessorInfo, HttpClient, IncomingAsyncBody, RpDelete, + RpRead, RpWrite, }, types::Result, Capability, Error, ErrorKind, diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 9320b172da7b..af19e2ff3996 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -22,19 +22,13 @@ use std::sync::Arc; use crate::raw::HttpClient; -use async_trait::async_trait; -use http::{header, Request, Response, StatusCode}; +use http::{header, Request, Response}; use serde::Deserialize; use tokio::sync::Mutex; use crate::{ - ops::{OpDelete, OpRead, OpWrite}, - raw::{ - build_rooted_abs_path, new_request_build_error, parse_into_metadata, Accessor, - AccessorInfo, AsyncBody, IncomingAsyncBody, RpDelete, RpRead, RpWrite, - }, + raw::{build_rooted_abs_path, new_request_build_error, AsyncBody, IncomingAsyncBody}, types::Result, - Capability, Error, ErrorKind, }; pub struct GdriveCore { @@ -87,7 +81,7 @@ impl GdriveCore { let path = build_rooted_abs_path(&self.root, file_path); { - let mut cache_guard = self.path_2_id.lock().await; + let cache_guard = self.path_2_id.lock().await; if cache_guard.contains_key(&path) { return cache_guard.get(&path).unwrap().to_string(); } @@ -123,7 +117,6 @@ impl GdriveCore { let body_value: GdriveFileList = serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); parent_id = String::from(body_value.files[0].id.as_str()); - println!("id: {}", parent_id); } let mut cache_guard = self.path_2_id.lock().await; From e6fd8dd1b0ea81cbd2c6fbc6ddf763cb50383a67 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Fri, 5 May 2023 18:21:14 +0800 Subject: [PATCH 3/7] deal with error --- core/src/services/gdrive/backend.rs | 2 +- core/src/services/gdrive/core.rs | 90 ++++++++++++++++++----------- 2 files changed, 57 insertions(+), 35 deletions(-) diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index fc248e7d509c..55a7bb5cf382 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -45,7 +45,7 @@ impl GdriveBackend { root, access_token, client: http_client, - path_2_id: Arc::new(Mutex::new(HashMap::new())), + path_cache: Arc::default(), }), } } diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index af19e2ff3996..5afb7966dcb0 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -21,6 +21,9 @@ use std::fmt::Formatter; use std::sync::Arc; use crate::raw::HttpClient; +use crate::Error; +use crate::ErrorKind; +use crate::Scheme; use http::{header, Request, Response}; use serde::Deserialize; @@ -35,7 +38,7 @@ pub struct GdriveCore { pub root: String, pub access_token: String, pub client: HttpClient, - pub path_2_id: Arc>>, + pub path_cache: Arc>>, } impl Debug for GdriveCore { @@ -47,14 +50,11 @@ impl Debug for GdriveCore { } impl GdriveCore { - async fn get_abs_root_id(&self) -> String { + async fn get_abs_root_id(&self) -> Result { let root = "root"; - { - let cache_guard = self.path_2_id.lock().await; - if cache_guard.contains_key(root) { - return cache_guard.get(root).unwrap().to_string(); - } + if let Some(root_id) = self.path_cache.lock().await.get(root) { + return Ok(root_id.to_string()); } let mut req = Request::get("https://www.googleapis.com/drive/v3/files/root"); @@ -62,34 +62,40 @@ impl GdriveCore { req = req.header(header::AUTHORIZATION, auth_header_content); let req = req .body(AsyncBody::Empty) - .map_err(new_request_build_error) - .unwrap(); + .map_err(new_request_build_error)?; + + let resp = self.client.send(req).await?; + let resp_body = &resp.into_body().bytes().await.map_err(|e| { + Error::new(ErrorKind::Unexpected, "read respone body error") + .with_context("service", Scheme::Gdrive) + .set_source(e) + })?; - let resp = self.client.send(req).await.unwrap(); + let gdrive_file: GdriveFile = serde_json::from_slice(resp_body).map_err(|e| { + Error::new(ErrorKind::Unexpected, "deserialize json error") + .with_context("service", Scheme::Gdrive) + .with_context("origin json vaule", String::from_utf8_lossy(resp_body)) + .set_source(e) + })?; - let body_value: GdriveFile = - serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); - let root_id = String::from(body_value.id.as_str()); + let root_id = gdrive_file.id; - let mut cache_guard = self.path_2_id.lock().await; + let mut cache_guard = self.path_cache.lock().await; cache_guard.insert(root.to_owned(), root_id.clone()); - root_id + Ok(root_id) } - async fn get_file_id_by_path(&self, file_path: &str) -> String { + async fn get_file_id_by_path(&self, file_path: &str) -> Result { let path = build_rooted_abs_path(&self.root, file_path); - { - let cache_guard = self.path_2_id.lock().await; - if cache_guard.contains_key(&path) { - return cache_guard.get(&path).unwrap().to_string(); - } + if let Some(file_id) = self.path_cache.lock().await.get(&path) { + return Ok(file_id.to_string()); } let auth_header_content = format!("Bearer {}", self.access_token); - let mut parent_id = self.get_abs_root_id().await; + let mut parent_id = self.get_abs_root_id().await?; let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); for (i, item) in file_path_items.iter().enumerate() { @@ -109,26 +115,40 @@ impl GdriveCore { req = req.header(header::AUTHORIZATION, &auth_header_content); let req = req .body(AsyncBody::default()) - .map_err(new_request_build_error) - .unwrap(); - - let resp = self.client.send(req).await.unwrap(); + .map_err(new_request_build_error)?; + + let resp = self.client.send(req).await?; + let resp_body = &resp.into_body().bytes().await.map_err(|e| { + Error::new(ErrorKind::Unexpected, "read respone body error") + .with_context("service", Scheme::Gdrive) + .set_source(e) + })?; + + let gdrive_file_list: GdriveFileList = + serde_json::from_slice(resp_body).map_err(|e| { + Error::new(ErrorKind::Unexpected, "deserialize json error") + .with_context("service", Scheme::Gdrive) + .with_context("origin json vaule", String::from_utf8_lossy(resp_body)) + .set_source(e) + })?; + + if gdrive_file_list.files.len() != 1 { + return Err(Error::new(ErrorKind::Unexpected, &format!("Please ensure that the file corresponding to the path exists and is unique. The response body is {}", String::from_utf8_lossy(resp_body)))); + } - let body_value: GdriveFileList = - serde_json::from_slice(&resp.into_body().bytes().await.unwrap()).unwrap(); - parent_id = String::from(body_value.files[0].id.as_str()); + parent_id = gdrive_file_list.files[0].id.clone(); } - let mut cache_guard = self.path_2_id.lock().await; + let mut cache_guard = self.path_cache.lock().await; cache_guard.insert(path, parent_id.clone()); - parent_id + Ok(parent_id) } pub async fn gdrive_get(&self, path: &str) -> Result> { let url: String = format!( "https://www.googleapis.com/drive/v3/files/{}?alt=media", - self.get_file_id_by_path(path).await + self.get_file_id_by_path(path).await? ); let auth_header_content = format!("Bearer {}", self.access_token); @@ -151,7 +171,7 @@ impl GdriveCore { ) -> Result> { let url = format!( "https://www.googleapis.com/upload/drive/v3/files/{}", - self.get_file_id_by_path(path).await + self.get_file_id_by_path(path).await? ); let mut req = Request::patch(&url); @@ -175,7 +195,7 @@ impl GdriveCore { pub async fn gdrive_delete(&self, path: &str) -> Result> { let url = format!( "https://www.googleapis.com/drive/v3/files/{}", - self.get_file_id_by_path(path).await + self.get_file_id_by_path(path).await? ); let mut req = Request::delete(&url); @@ -191,11 +211,13 @@ impl GdriveCore { } } +// refer to https://developers.google.com/drive/api/reference/rest/v3/files#File #[derive(Deserialize)] struct GdriveFile { id: String, } +// refer to https://developers.google.com/drive/api/reference/rest/v3/files/list #[derive(Deserialize)] struct GdriveFileList { files: Vec, From 57afd4dea6fe689d0cbb7be244a6f66124dbc435 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Fri, 5 May 2023 18:33:54 +0800 Subject: [PATCH 4/7] typos fix --- core/src/services/gdrive/core.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 5afb7966dcb0..39fb672e721c 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -66,7 +66,7 @@ impl GdriveCore { let resp = self.client.send(req).await?; let resp_body = &resp.into_body().bytes().await.map_err(|e| { - Error::new(ErrorKind::Unexpected, "read respone body error") + Error::new(ErrorKind::Unexpected, "read response body error") .with_context("service", Scheme::Gdrive) .set_source(e) })?; @@ -74,7 +74,7 @@ impl GdriveCore { let gdrive_file: GdriveFile = serde_json::from_slice(resp_body).map_err(|e| { Error::new(ErrorKind::Unexpected, "deserialize json error") .with_context("service", Scheme::Gdrive) - .with_context("origin json vaule", String::from_utf8_lossy(resp_body)) + .with_context("origin json value", String::from_utf8_lossy(resp_body)) .set_source(e) })?; @@ -119,7 +119,7 @@ impl GdriveCore { let resp = self.client.send(req).await?; let resp_body = &resp.into_body().bytes().await.map_err(|e| { - Error::new(ErrorKind::Unexpected, "read respone body error") + Error::new(ErrorKind::Unexpected, "read response body error") .with_context("service", Scheme::Gdrive) .set_source(e) })?; @@ -128,7 +128,7 @@ impl GdriveCore { serde_json::from_slice(resp_body).map_err(|e| { Error::new(ErrorKind::Unexpected, "deserialize json error") .with_context("service", Scheme::Gdrive) - .with_context("origin json vaule", String::from_utf8_lossy(resp_body)) + .with_context("origin json value", String::from_utf8_lossy(resp_body)) .set_source(e) })?; From c67e80e704648859ac7adcbbedd19a9c56cf3267 Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Fri, 5 May 2023 20:02:23 +0800 Subject: [PATCH 5/7] extract auth to core.sign --- core/src/services/gdrive/backend.rs | 3 +-- core/src/services/gdrive/core.rs | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/src/services/gdrive/backend.rs b/core/src/services/gdrive/backend.rs index 55a7bb5cf382..03d8c83eab77 100644 --- a/core/src/services/gdrive/backend.rs +++ b/core/src/services/gdrive/backend.rs @@ -18,8 +18,7 @@ use async_trait::async_trait; use http::StatusCode; -use std::{collections::HashMap, fmt::Debug, sync::Arc}; -use tokio::sync::Mutex; +use std::{fmt::Debug, sync::Arc}; use crate::{ ops::{OpDelete, OpRead, OpWrite}, diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 39fb672e721c..636928e13dd0 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -25,6 +25,7 @@ use crate::Error; use crate::ErrorKind; use crate::Scheme; +use http::request::Builder; use http::{header, Request, Response}; use serde::Deserialize; use tokio::sync::Mutex; @@ -57,10 +58,10 @@ impl GdriveCore { return Ok(root_id.to_string()); } - let mut req = Request::get("https://www.googleapis.com/drive/v3/files/root"); - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - let req = req + let req = self + .sign(Request::get( + "https://www.googleapis.com/drive/v3/files/root", + )) .body(AsyncBody::Empty) .map_err(new_request_build_error)?; @@ -93,8 +94,6 @@ impl GdriveCore { return Ok(file_id.to_string()); } - let auth_header_content = format!("Bearer {}", self.access_token); - let mut parent_id = self.get_abs_root_id().await?; let file_path_items: Vec<&str> = path.split('/').filter(|&x| !x.is_empty()).collect(); @@ -108,12 +107,11 @@ impl GdriveCore { } let query: String = query.chars().filter(|c| !c.is_whitespace()).collect(); - let mut req = Request::get(format!( - "https://www.googleapis.com/drive/v3/files?q={}", - query - )); - req = req.header(header::AUTHORIZATION, &auth_header_content); - let req = req + let req = self + .sign(Request::get(format!( + "https://www.googleapis.com/drive/v3/files?q={}", + query + ))) .body(AsyncBody::default()) .map_err(new_request_build_error)?; @@ -209,6 +207,12 @@ impl GdriveCore { self.client.send(req).await } + + fn sign(&self, mut req: Builder) -> Builder { + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + req + } } // refer to https://developers.google.com/drive/api/reference/rest/v3/files#File From a7a51309be84ac9a7641a5311f7c8f88cccbe68e Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Sun, 7 May 2023 14:40:39 +0800 Subject: [PATCH 6/7] code review --- core/src/services/gdrive/core.rs | 39 ++++++++------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 636928e13dd0..435ab99583fa 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -20,10 +20,10 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use crate::raw::percent_encode_path; use crate::raw::HttpClient; use crate::Error; use crate::ErrorKind; -use crate::Scheme; use http::request::Builder; use http::{header, Request, Response}; @@ -66,15 +66,10 @@ impl GdriveCore { .map_err(new_request_build_error)?; let resp = self.client.send(req).await?; - let resp_body = &resp.into_body().bytes().await.map_err(|e| { - Error::new(ErrorKind::Unexpected, "read response body error") - .with_context("service", Scheme::Gdrive) - .set_source(e) - })?; + let resp_body = &resp.into_body().bytes().await?; let gdrive_file: GdriveFile = serde_json::from_slice(resp_body).map_err(|e| { Error::new(ErrorKind::Unexpected, "deserialize json error") - .with_context("service", Scheme::Gdrive) .with_context("origin json value", String::from_utf8_lossy(resp_body)) .set_source(e) })?; @@ -105,27 +100,21 @@ impl GdriveCore { if i != file_path_items.len() - 1 { query += "and mimeType = 'application/vnd.google-apps.folder'"; } - let query: String = query.chars().filter(|c| !c.is_whitespace()).collect(); let req = self .sign(Request::get(format!( "https://www.googleapis.com/drive/v3/files?q={}", - query + percent_encode_path(&query) ))) .body(AsyncBody::default()) .map_err(new_request_build_error)?; let resp = self.client.send(req).await?; - let resp_body = &resp.into_body().bytes().await.map_err(|e| { - Error::new(ErrorKind::Unexpected, "read response body error") - .with_context("service", Scheme::Gdrive) - .set_source(e) - })?; + let resp_body = &resp.into_body().bytes().await?; let gdrive_file_list: GdriveFileList = serde_json::from_slice(resp_body).map_err(|e| { Error::new(ErrorKind::Unexpected, "deserialize json error") - .with_context("service", Scheme::Gdrive) .with_context("origin json value", String::from_utf8_lossy(resp_body)) .set_source(e) })?; @@ -149,11 +138,8 @@ impl GdriveCore { self.get_file_id_by_path(path).await? ); - let auth_header_content = format!("Bearer {}", self.access_token); - let mut req = Request::get(&url); - req = req.header(header::AUTHORIZATION, auth_header_content); - - let req = req + let req = self + .sign(Request::get(&url)) .body(AsyncBody::Empty) .map_err(new_request_build_error)?; @@ -174,9 +160,6 @@ impl GdriveCore { let mut req = Request::patch(&url); - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - if let Some(size) = size { req = req.header(header::CONTENT_LENGTH, size) } @@ -185,7 +168,7 @@ impl GdriveCore { req = req.header(header::CONTENT_TYPE, mime) } - let req = req.body(body).map_err(new_request_build_error)?; + let req = self.sign(req).body(body).map_err(new_request_build_error)?; self.client.send(req).await } @@ -196,12 +179,8 @@ impl GdriveCore { self.get_file_id_by_path(path).await? ); - let mut req = Request::delete(&url); - - let auth_header_content = format!("Bearer {}", self.access_token); - req = req.header(header::AUTHORIZATION, auth_header_content); - - let req = req + let req = self + .sign(Request::delete(&url)) .body(AsyncBody::Empty) .map_err(new_request_build_error)?; From 83996b16c5927e8f4c7ea6f59f1eda2401d74bed Mon Sep 17 00:00:00 2001 From: Young-Flash <871946895@qq.com> Date: Mon, 8 May 2023 16:41:36 +0800 Subject: [PATCH 7/7] parse resp according to StatusCode --- core/src/services/gdrive/core.rs | 52 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/core/src/services/gdrive/core.rs b/core/src/services/gdrive/core.rs index 435ab99583fa..6f66bf8bddec 100644 --- a/core/src/services/gdrive/core.rs +++ b/core/src/services/gdrive/core.rs @@ -20,12 +20,14 @@ use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; +use crate::raw::new_json_deserialize_error; use crate::raw::percent_encode_path; use crate::raw::HttpClient; use crate::Error; use crate::ErrorKind; use http::request::Builder; +use http::StatusCode; use http::{header, Request, Response}; use serde::Deserialize; use tokio::sync::Mutex; @@ -35,6 +37,8 @@ use crate::{ types::Result, }; +use super::error::parse_error; + pub struct GdriveCore { pub root: String, pub access_token: String, @@ -66,20 +70,24 @@ impl GdriveCore { .map_err(new_request_build_error)?; let resp = self.client.send(req).await?; - let resp_body = &resp.into_body().bytes().await?; + let status = resp.status(); - let gdrive_file: GdriveFile = serde_json::from_slice(resp_body).map_err(|e| { - Error::new(ErrorKind::Unexpected, "deserialize json error") - .with_context("origin json value", String::from_utf8_lossy(resp_body)) - .set_source(e) - })?; + match status { + StatusCode::OK => { + let resp_body = &resp.into_body().bytes().await?; - let root_id = gdrive_file.id; + let gdrive_file: GdriveFile = + serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?; - let mut cache_guard = self.path_cache.lock().await; - cache_guard.insert(root.to_owned(), root_id.clone()); + let root_id = gdrive_file.id; - Ok(root_id) + let mut cache_guard = self.path_cache.lock().await; + cache_guard.insert(root.to_owned(), root_id.clone()); + + Ok(root_id) + } + _ => Err(parse_error(resp).await?), + } } async fn get_file_id_by_path(&self, file_path: &str) -> Result { @@ -110,20 +118,22 @@ impl GdriveCore { .map_err(new_request_build_error)?; let resp = self.client.send(req).await?; - let resp_body = &resp.into_body().bytes().await?; + let status = resp.status(); - let gdrive_file_list: GdriveFileList = - serde_json::from_slice(resp_body).map_err(|e| { - Error::new(ErrorKind::Unexpected, "deserialize json error") - .with_context("origin json value", String::from_utf8_lossy(resp_body)) - .set_source(e) - })?; + if status == StatusCode::OK { + let resp_body = &resp.into_body().bytes().await?; - if gdrive_file_list.files.len() != 1 { - return Err(Error::new(ErrorKind::Unexpected, &format!("Please ensure that the file corresponding to the path exists and is unique. The response body is {}", String::from_utf8_lossy(resp_body)))); - } + let gdrive_file_list: GdriveFileList = + serde_json::from_slice(resp_body).map_err(new_json_deserialize_error)?; - parent_id = gdrive_file_list.files[0].id.clone(); + if gdrive_file_list.files.len() != 1 { + return Err(Error::new(ErrorKind::Unexpected, &format!("Please ensure that the file corresponding to the path exists and is unique. The response body is {}", String::from_utf8_lossy(resp_body)))); + } + + parent_id = gdrive_file_list.files[0].id.clone(); + } else { + return Err(parse_error(resp).await?); + } } let mut cache_guard = self.path_cache.lock().await;