From 44784b45f0fa637f058db9c3278a0f9043c80f09 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 8 Apr 2023 14:59:13 -0700 Subject: [PATCH 01/35] basic onedrive --- core/src/services/mod.rs | 2 + src/services/onedrive/backend.rs | 90 ++++++++++++++++++++++++++++ src/services/onedrive/builder.rs | 31 ++++++++++ src/services/onedrive/graph_model.rs | 89 +++++++++++++++++++++++++++ src/services/onedrive/mod.rs | 3 + 5 files changed, 215 insertions(+) create mode 100644 src/services/onedrive/backend.rs create mode 100644 src/services/onedrive/builder.rs create mode 100644 src/services/onedrive/graph_model.rs create mode 100644 src/services/onedrive/mod.rs diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index c02642a82a89..20545985ea17 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -101,4 +101,6 @@ mod webdav; pub use webdav::Webdav; mod webhdfs; +mod onedrive; + pub use webhdfs::Webhdfs; diff --git a/src/services/onedrive/backend.rs b/src/services/onedrive/backend.rs new file mode 100644 index 000000000000..40ac59729c7d --- /dev/null +++ b/src/services/onedrive/backend.rs @@ -0,0 +1,90 @@ +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; +use http::{Method, StatusCode}; + +use reqwest::{header::{self, HeaderValue}, Client, Request, Response, Url}; +use url::Url; + +use super::{Accessor, AccessorCapability, AccessorHint, AccessorMetadata, ObjectMode}; +use crate::{error::{new_request_build_error, parse_error, Error, Result}, input::{self, AsyncBody, IncomingAsyncBody, Reader}, rp::{ + BytesRange, ObjectMetadata, OpCreate, OpDelete, OpRead, OpStat, OpWrite, RpCreate, + RpDelete, RpRead, RpStat, RpWrite, +}, Scheme}; +use crate::ops::OpRead; +use crate::raw::{Accessor, AccessorCapability, AccessorHint, AccessorMetadata, AsyncBody, IncomingAsyncBody, RpRead}; + +/// OneDrive backend implementation. +pub struct OneDriveBackend { + client: Arc, + endpoint: String, + token: String, +} + + +impl Debug for OneDriveBackend { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + todo!() + } +} + +#[async_trait] +impl Accessor for OneDriveBackend { + type Reader = IncomingAsyncBody; + type BlockingReader = (); + type Pager = (); + type BlockingPager = (); + + fn metadata(&self) -> AccessorMetadata { + let mut ma = AccessorMetadata::default(); + ma.set_scheme(Scheme::OneDrive) + .set_capabilities(AccessorCapability::Read | AccessorCapability::Write | AccessorCapability::List) + .set_hints(AccessorHint::ReadStreamable); + ma + } + + // read + async fn read(&self, path: &str, range: Option) -> Result> { + let resp = self.graph_api_request("GET", path, AsyncBody::empty()).await?; + let reader = resp.into_body(); + let metadata = ObjectMetadata::default(); + Ok(OpRead::new(reader, metadata)) + } +} + +impl OneDriveBackend { + pub fn new(client: Arc, endpoint: String, token: String) -> Self { + OneDriveBackend { + client, + endpoint, + token, + } + } + + async fn graph_api_request( + &self, + method: &str, + path: &str, + body: AsyncBody, + ) -> Result { + let url = Url::parse(&format!("{}/me/drive/root:/{}", self.endpoint, path))?; + let mut req = Request::new(method.parse()?, url); + let token = HeaderValue::from_str(&format!("Bearer {}", self.token))?; + req.headers_mut().insert(header::AUTHORIZATION, token); + req.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + req.headers_mut().insert(header::ACCEPT, HeaderValue::from_static("application/json")); + req.headers_mut().insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); + + let req = req.body(body).map_err(new_request_build_error)?; + + let resp = self.client.execute(req).await?; + + if !resp.status().is_success() { + Err(Error::OneDriveError(resp.status().as_u16())) + } else { + Ok(resp) + } + } +} diff --git a/src/services/onedrive/builder.rs b/src/services/onedrive/builder.rs new file mode 100644 index 000000000000..85b27c8a9459 --- /dev/null +++ b/src/services/onedrive/builder.rs @@ -0,0 +1,31 @@ +use std::fmt::{Debug, Formatter}; + +#[derive(Default)] +pub struct OneDriveBuilder { + access_token: Option, +} + +impl Debug for OneDriveBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("Builder"); + de.field("endpoint", &self.access_token); + de.finish() + } +} + +impl OneDriveBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn access_token(mut self, access_token: &str) -> Self { + self.access_token = Some(access_token.to_string()); + self + } + + pub fn build(self) -> OneDrive { + OneDrive { + access_token: self.access_token, + } + } +} diff --git a/src/services/onedrive/graph_model.rs b/src/services/onedrive/graph_model.rs new file mode 100644 index 000000000000..659219c65813 --- /dev/null +++ b/src/services/onedrive/graph_model.rs @@ -0,0 +1,89 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize)] +struct DriveItem { + createdDateTime: String, + cTag: String, + eTag: String, + id: String, + lastModifiedDateTime: String, + name: String, + size: u64, + webUrl: String, + reactions: Option, + createdBy: Option, + lastModifiedBy: Option, + parentReference: ParentReference, + fileSystemInfo: FileSystemInfo, + folder: Option, + specialFolder: Option, +} + +#[derive(Debug, Deserialize)] +struct Reactions { + commentCount: u64, +} + +#[derive(Debug, Deserialize)] +struct CreatedBy { + user: Option, + application: Option, +} + +#[derive(Debug, Deserialize)] +struct LastModifiedBy { + user: Option, + application: Option, +} + +#[derive(Debug, Deserialize)] +struct User { + displayName: String, + id: String, +} + +#[derive(Debug, Deserialize)] +struct Application { + displayName: String, + id: String, +} + +#[derive(Debug, Deserialize)] +struct ParentReference { + driveId: String, + driveType: String, + id: String, + path: String, +} + +#[derive(Debug, Deserialize)] +struct FileSystemInfo { + createdDateTime: String, + lastModifiedDateTime: String, +} + +#[derive(Debug, Deserialize)] +struct Folder { + childCount: u64, + view: View, +} + +#[derive(Debug, Deserialize)] +struct View { + viewType: String, + sortBy: String, + sortOrder: String, +} + +#[derive(Debug, Deserialize)] +struct SpecialFolder { + name: String, +} + +#[derive(Debug, Deserialize)] +struct DriveItemList { + value: Vec, + count: u64, + context: String, +} + diff --git a/src/services/onedrive/mod.rs b/src/services/onedrive/mod.rs new file mode 100644 index 000000000000..8e92f0b8419b --- /dev/null +++ b/src/services/onedrive/mod.rs @@ -0,0 +1,3 @@ +mod backend; +mod builder; +mod graph_model; From 9d934f0ff9d091ed74f672d3211288c233fdb3c9 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 8 Apr 2023 15:51:32 -0700 Subject: [PATCH 02/35] fix data model --- core/src/services/onedrive/backend.rs | 1 + .../src}/services/onedrive/builder.rs | 10 +- core/src/services/onedrive/graph_model.rs | 177 ++++++++++++++++++ {src => core/src}/services/onedrive/mod.rs | 0 src/services/onedrive/backend.rs | 90 --------- src/services/onedrive/graph_model.rs | 89 --------- 6 files changed, 183 insertions(+), 184 deletions(-) create mode 100644 core/src/services/onedrive/backend.rs rename {src => core/src}/services/onedrive/builder.rs (81%) create mode 100644 core/src/services/onedrive/graph_model.rs rename {src => core/src}/services/onedrive/mod.rs (100%) delete mode 100644 src/services/onedrive/backend.rs delete mode 100644 src/services/onedrive/graph_model.rs diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/core/src/services/onedrive/backend.rs @@ -0,0 +1 @@ + diff --git a/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs similarity index 81% rename from src/services/onedrive/builder.rs rename to core/src/services/onedrive/builder.rs index 85b27c8a9459..f1ae50ba3147 100644 --- a/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -23,9 +23,9 @@ impl OneDriveBuilder { self } - pub fn build(self) -> OneDrive { - OneDrive { - access_token: self.access_token, - } - } + // pub fn build(self) -> OneDrive { + // OneDrive { + // access_token: self.access_token, + // } + // } } diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs new file mode 100644 index 000000000000..3bff28b05159 --- /dev/null +++ b/core/src/services/onedrive/graph_model.rs @@ -0,0 +1,177 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Result; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +struct GraphApiOneDriveResponse { + #[serde(rename = "@odata.context")] + odata_context: String, + + #[serde(rename = "@odata.count")] + odata_count: usize, + + value: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct OneDriveItem { + #[serde(rename = "createdDateTime")] + created_date_time: String, + eTag: String, + id: String, + lastModifiedDateTime: String, + name: String, + size: usize, + webUrl: String, + parentReference: ParentReference, + fileSystemInfo: FileSystemInfo, + #[serde(flatten)] + item_type: ItemType, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ParentReference { + driveId: String, + driveType: String, + id: String, + path: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FileSystemInfo { + createdDateTime: String, + lastModifiedDateTime: String, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +enum ItemType { + Folder { + folder: HashMap, + #[serde(rename = "specialFolder")] + special_folder: HashMap, + }, + File { + file: HashMap, + }, +} + +fn parse_one_drive_json(data: &str) -> Result { + let response: GraphApiOneDriveResponse = serde_json::from_str(data)?; + Ok(response) +} + +#[test] +fn test_parse_one_drive_json() { + let data = r#"{ + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children", + "@odata.count": 1, + "value": [ + { + "createdDateTime": "2020-01-01T00:00:00Z", + "cTag": "cTag", + "eTag": "eTag", + "id": "id", + "lastModifiedDateTime": "2020-01-01T00:00:00Z", + "name": "name", + "size": 0, + "webUrl": "webUrl", + "reactions": { + "like": 0 + }, + "parentReference": { + "driveId": "driveId", + "driveType": "driveType", + "id": "id", + "path": "/drive/root:" + }, + "fileSystemInfo": { + "createdDateTime": "2020-01-01T00:00:00Z", + "lastModifiedDateTime": "2020-01-01T00:00:00Z" + }, + "folder": { + "childCount": 0 + }, + "specialFolder": { + "name": "name" + } + }, + { + "@microsoft.graph.downloadUrl": "https://public.ch.files.1drv.com/y4mPh7u0QjYTl5j9aZDj77EoplXNhXFzSbakI4iYoUXMaGUOSmpx1d20AnCoU9G32nj6W2qsKNfecsgfmF6O8ZE89yUYj7qnhsIvfikcJjJ0_skDA12gl2cCScQ3opoza_RcG2Lb_Pa2jyqiqgruh0TJRcC1y7mtEw89wqXx2bgjOvmo0ozTAwopTtpti9yo43Zb7nBI1efm3IwWhFKcHUUKx7WlD_8VPXPB4Xffokz61NiXoxMeq0hbwrblcznywz2AcE71SprDyCi8E7kDRjwmiTNoyfZc_FuUMZDO29WUbA", + "createdDateTime": "2018-12-30T05:32:55.46Z", + "cTag": "aYzozMjIxN0ZDMTE1NEFFQzNEITEwMi4yNTc", + "eTag": "aMzIyMTdGQzExNTRBRUMzRCExMDIuMw", + "id": "32217FC1154AEC3D!102", + "lastModifiedDateTime": "2018-12-30T05:33:23.557Z", + "name": "Getting started with OneDrive.pdf", + "size": 1025867, + "webUrl": "https://1drv.ms/b/s!AD3sShXBfyEyZg", + "reactions": { + "commentCount": 0 + }, + "createdBy": { + "user": { + "displayName": "Great Cat", + "id": "32217fc1154aec3d" + } + }, + "lastModifiedBy": { + "user": { + "displayName": "Great Cat", + "id": "32217fc1154aec3d" + } + }, + "parentReference": { + "driveId": "32217fc1154aec3d", + "driveType": "personal", + "id": "32217FC1154AEC3D!101", + "path": "/drive/root:" + }, + "file": { + "mimeType": "application/pdf", + "hashes": { + "quickXorHash": "NIfFZIvQVZH260260iUuQN5GscM=", + "sha1Hash": "E8890F3D1CE6E3FCCE46D08B188275D6CAE3292C" + } + }, + "fileSystemInfo": { + "createdDateTime": "2018-12-30T05:32:55.46Z", + "lastModifiedDateTime": "2018-12-30T05:32:55.46Z" + } + } + ] + }"#; + let response = parse_one_drive_json(data).unwrap(); + assert_eq!( + response.odata_context, + "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children" + ); + assert_eq!(response.odata_count, 1); + assert_eq!(response.value.len(), 2); + let item = &response.value[0]; + assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); + assert_eq!(item.eTag, "eTag"); + assert_eq!(item.id, "id"); + assert_eq!(item.lastModifiedDateTime, "2020-01-01T00:00:00Z"); + assert_eq!(item.name, "name"); + assert_eq!(item.size, 0); + assert_eq!(item.webUrl, "webUrl"); + assert_eq!( + item.item_type, + ItemType::Folder { + folder: { + let mut map = HashMap::new(); + map.insert( + "childCount".to_string(), + serde_json::Value::Number(0.into()), + ); + map + }, + special_folder: { + let mut map = HashMap::new(); + map.insert("name".to_string(), "name".to_string()); + map + }, + } + ); +} diff --git a/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs similarity index 100% rename from src/services/onedrive/mod.rs rename to core/src/services/onedrive/mod.rs diff --git a/src/services/onedrive/backend.rs b/src/services/onedrive/backend.rs deleted file mode 100644 index 40ac59729c7d..000000000000 --- a/src/services/onedrive/backend.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; -use http::{Method, StatusCode}; - -use reqwest::{header::{self, HeaderValue}, Client, Request, Response, Url}; -use url::Url; - -use super::{Accessor, AccessorCapability, AccessorHint, AccessorMetadata, ObjectMode}; -use crate::{error::{new_request_build_error, parse_error, Error, Result}, input::{self, AsyncBody, IncomingAsyncBody, Reader}, rp::{ - BytesRange, ObjectMetadata, OpCreate, OpDelete, OpRead, OpStat, OpWrite, RpCreate, - RpDelete, RpRead, RpStat, RpWrite, -}, Scheme}; -use crate::ops::OpRead; -use crate::raw::{Accessor, AccessorCapability, AccessorHint, AccessorMetadata, AsyncBody, IncomingAsyncBody, RpRead}; - -/// OneDrive backend implementation. -pub struct OneDriveBackend { - client: Arc, - endpoint: String, - token: String, -} - - -impl Debug for OneDriveBackend { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - todo!() - } -} - -#[async_trait] -impl Accessor for OneDriveBackend { - type Reader = IncomingAsyncBody; - type BlockingReader = (); - type Pager = (); - type BlockingPager = (); - - fn metadata(&self) -> AccessorMetadata { - let mut ma = AccessorMetadata::default(); - ma.set_scheme(Scheme::OneDrive) - .set_capabilities(AccessorCapability::Read | AccessorCapability::Write | AccessorCapability::List) - .set_hints(AccessorHint::ReadStreamable); - ma - } - - // read - async fn read(&self, path: &str, range: Option) -> Result> { - let resp = self.graph_api_request("GET", path, AsyncBody::empty()).await?; - let reader = resp.into_body(); - let metadata = ObjectMetadata::default(); - Ok(OpRead::new(reader, metadata)) - } -} - -impl OneDriveBackend { - pub fn new(client: Arc, endpoint: String, token: String) -> Self { - OneDriveBackend { - client, - endpoint, - token, - } - } - - async fn graph_api_request( - &self, - method: &str, - path: &str, - body: AsyncBody, - ) -> Result { - let url = Url::parse(&format!("{}/me/drive/root:/{}", self.endpoint, path))?; - let mut req = Request::new(method.parse()?, url); - let token = HeaderValue::from_str(&format!("Bearer {}", self.token))?; - req.headers_mut().insert(header::AUTHORIZATION, token); - req.headers_mut().insert( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - ); - req.headers_mut().insert(header::ACCEPT, HeaderValue::from_static("application/json")); - req.headers_mut().insert(header::ACCEPT_ENCODING, HeaderValue::from_static("gzip")); - - let req = req.body(body).map_err(new_request_build_error)?; - - let resp = self.client.execute(req).await?; - - if !resp.status().is_success() { - Err(Error::OneDriveError(resp.status().as_u16())) - } else { - Ok(resp) - } - } -} diff --git a/src/services/onedrive/graph_model.rs b/src/services/onedrive/graph_model.rs deleted file mode 100644 index 659219c65813..000000000000 --- a/src/services/onedrive/graph_model.rs +++ /dev/null @@ -1,89 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Deserialize)] -struct DriveItem { - createdDateTime: String, - cTag: String, - eTag: String, - id: String, - lastModifiedDateTime: String, - name: String, - size: u64, - webUrl: String, - reactions: Option, - createdBy: Option, - lastModifiedBy: Option, - parentReference: ParentReference, - fileSystemInfo: FileSystemInfo, - folder: Option, - specialFolder: Option, -} - -#[derive(Debug, Deserialize)] -struct Reactions { - commentCount: u64, -} - -#[derive(Debug, Deserialize)] -struct CreatedBy { - user: Option, - application: Option, -} - -#[derive(Debug, Deserialize)] -struct LastModifiedBy { - user: Option, - application: Option, -} - -#[derive(Debug, Deserialize)] -struct User { - displayName: String, - id: String, -} - -#[derive(Debug, Deserialize)] -struct Application { - displayName: String, - id: String, -} - -#[derive(Debug, Deserialize)] -struct ParentReference { - driveId: String, - driveType: String, - id: String, - path: String, -} - -#[derive(Debug, Deserialize)] -struct FileSystemInfo { - createdDateTime: String, - lastModifiedDateTime: String, -} - -#[derive(Debug, Deserialize)] -struct Folder { - childCount: u64, - view: View, -} - -#[derive(Debug, Deserialize)] -struct View { - viewType: String, - sortBy: String, - sortOrder: String, -} - -#[derive(Debug, Deserialize)] -struct SpecialFolder { - name: String, -} - -#[derive(Debug, Deserialize)] -struct DriveItemList { - value: Vec, - count: u64, - context: String, -} - From ab80bbf866a436c1e2e6a859d94886a838ef64bf Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 8 Apr 2023 16:09:44 -0700 Subject: [PATCH 03/35] implement basic --- core/src/services/onedrive/backend.rs | 44 +++++++++++++++++++++++++++ core/src/types/scheme.rs | 3 ++ 2 files changed, 47 insertions(+) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 8b137891791f..79538e20b2c0 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1 +1,45 @@ +use async_trait::async_trait; +use std::fmt::Debug; +use crate::raw::{Accessor, AccessorCapability, AccessorHint, AccessorInfo, IncomingAsyncBody}; + +#[derive(Clone)] +pub struct OneDriveBackend { + root: String, + access_token: String, +} + +impl Debug for OneDriveBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("OneDriveBackend"); + de.field("root", &self.root); + de.field("access_token", &self.access_token); + de.finish() + } +} + +#[async_trait] +impl Accessor for OneDriveBackend { + type Reader = IncomingAsyncBody; + type BlockingReader = (); + type Writer = (); + type BlockingWriter = (); + type Pager = (); + type BlockingPager = (); + + fn info(&self) -> AccessorInfo { + let mut ma = AccessorInfo::default(); + ma.set_scheme(crate::Scheme::Onedrive) + .set_root(&self.root) + .set_capabilities( + AccessorCapability::Read + | AccessorCapability::Write + | AccessorCapability::Copy + | AccessorCapability::Rename + | AccessorCapability::List, + ) + .set_hints(AccessorHint::ReadStreamable); + + ma + } +} diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 3fb9d2fc5db3..312f5d559933 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -67,6 +67,8 @@ pub enum Scheme { Moka, /// [obs][crate::services::Obs]: Huawei Cloud OBS services. Obs, + /// [onedrive][crate::services::Onedrive]: Microsoft OneDrive services. + Onedrive, /// [oss][crate::services::Oss]: Aliyun Object Storage Services Oss, /// [redis][crate::services::Redis]: Redis services @@ -178,6 +180,7 @@ impl From for &'static str { #[cfg(feature = "services-moka")] Scheme::Moka => "moka", Scheme::Obs => "obs", + Scheme::Onedrive => "onedrive", #[cfg(feature = "services-redis")] Scheme::Redis => "redis", #[cfg(feature = "services-rocksdb")] From e0c293506c1e9983e7a4b164e5b673224f46834f Mon Sep 17 00:00:00 2001 From: imWildCat Date: Mon, 24 Apr 2023 16:00:29 -0600 Subject: [PATCH 04/35] set hints --- core/src/services/onedrive/backend.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 79538e20b2c0..04efb175f55f 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -31,14 +31,13 @@ impl Accessor for OneDriveBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(crate::Scheme::Onedrive) .set_root(&self.root) - .set_capabilities( + .set_capability( AccessorCapability::Read | AccessorCapability::Write | AccessorCapability::Copy | AccessorCapability::Rename | AccessorCapability::List, - ) - .set_hints(AccessorHint::ReadStreamable); + ); ma } From 69b552791c731590e8bb722273079ac15a162b68 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Mon, 24 Apr 2023 16:01:49 -0600 Subject: [PATCH 05/35] fix fix fix --- core/src/services/onedrive/backend.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 04efb175f55f..7010b53783f1 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use std::fmt::Debug; -use crate::raw::{Accessor, AccessorCapability, AccessorHint, AccessorInfo, IncomingAsyncBody}; +use crate::raw::{Accessor, AccessorInfo, IncomingAsyncBody}; #[derive(Clone)] pub struct OneDriveBackend { @@ -31,13 +31,15 @@ impl Accessor for OneDriveBackend { let mut ma = AccessorInfo::default(); ma.set_scheme(crate::Scheme::Onedrive) .set_root(&self.root) - .set_capability( - AccessorCapability::Read - | AccessorCapability::Write - | AccessorCapability::Copy - | AccessorCapability::Rename - | AccessorCapability::List, - ); + .set_capability(Capability { + read: true, + read_can_next: true, + write: true, + list: true, + copy: true, + rename: true, + ..Default::default() + }); ma } From 08772d02a4979e999f320ba5570116c141ba3210 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Mon, 24 Apr 2023 16:06:12 -0600 Subject: [PATCH 06/35] fix all --- core/src/services/onedrive/backend.rs | 2 +- core/src/types/scheme.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 7010b53783f1..aa9220043041 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use std::fmt::Debug; -use crate::raw::{Accessor, AccessorInfo, IncomingAsyncBody}; +use crate::{raw::{Accessor, AccessorInfo, IncomingAsyncBody}, Capability}; #[derive(Clone)] pub struct OneDriveBackend { diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 0a22c610f882..da75582f3bbc 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -160,7 +160,6 @@ impl From for &'static str { Scheme::Moka => "moka", Scheme::Obs => "obs", Scheme::Onedrive => "onedrive", - #[cfg(feature = "services-redis")] Scheme::Redis => "redis", Scheme::Rocksdb => "rocksdb", Scheme::S3 => "s3", From 4f12b6750c5b81cb964139428b64018af7f655fc Mon Sep 17 00:00:00 2001 From: imWildCat Date: Mon, 24 Apr 2023 22:43:54 -0600 Subject: [PATCH 07/35] fix --- core/src/services/onedrive/graph_model.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs index 3bff28b05159..2abc218155ac 100644 --- a/core/src/services/onedrive/graph_model.rs +++ b/core/src/services/onedrive/graph_model.rs @@ -40,7 +40,8 @@ struct ParentReference { #[derive(Debug, Serialize, Deserialize)] struct FileSystemInfo { createdDateTime: String, - lastModifiedDateTime: String, + #[serde[rename = "lastModifiedDateTime"]] + last_modified_date_time: String, } #[derive(Debug, Serialize, Deserialize, PartialEq)] From 8d22016f283dabc2890862f03473835e7bfd0dcc Mon Sep 17 00:00:00 2001 From: imWildCat Date: Mon, 24 Apr 2023 22:44:27 -0600 Subject: [PATCH 08/35] Squashed commit --- core/src/services/mod.rs | 4 + core/src/services/onedrive/backend.rs | 46 ++++++ core/src/services/onedrive/builder.rs | 31 ++++ core/src/services/onedrive/graph_model.rs | 178 ++++++++++++++++++++++ core/src/services/onedrive/mod.rs | 3 + core/src/types/scheme.rs | 3 + 6 files changed, 265 insertions(+) create mode 100644 core/src/services/onedrive/backend.rs create mode 100644 core/src/services/onedrive/builder.rs create mode 100644 core/src/services/onedrive/graph_model.rs create mode 100644 core/src/services/onedrive/mod.rs diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index 4c1b73971648..f9575e9e2456 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -131,5 +131,9 @@ pub use webdav::Webdav; #[cfg(feature = "services-webhdfs")] mod webhdfs; + + +mod onedrive; + #[cfg(feature = "services-webhdfs")] pub use webhdfs::Webhdfs; diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs new file mode 100644 index 000000000000..aa9220043041 --- /dev/null +++ b/core/src/services/onedrive/backend.rs @@ -0,0 +1,46 @@ +use async_trait::async_trait; +use std::fmt::Debug; + +use crate::{raw::{Accessor, AccessorInfo, IncomingAsyncBody}, Capability}; + +#[derive(Clone)] +pub struct OneDriveBackend { + root: String, + access_token: String, +} + +impl Debug for OneDriveBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("OneDriveBackend"); + de.field("root", &self.root); + de.field("access_token", &self.access_token); + de.finish() + } +} + +#[async_trait] +impl Accessor for OneDriveBackend { + type Reader = IncomingAsyncBody; + type BlockingReader = (); + type Writer = (); + type BlockingWriter = (); + type Pager = (); + type BlockingPager = (); + + fn info(&self) -> AccessorInfo { + let mut ma = AccessorInfo::default(); + ma.set_scheme(crate::Scheme::Onedrive) + .set_root(&self.root) + .set_capability(Capability { + read: true, + read_can_next: true, + write: true, + list: true, + copy: true, + rename: true, + ..Default::default() + }); + + ma + } +} diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs new file mode 100644 index 000000000000..f1ae50ba3147 --- /dev/null +++ b/core/src/services/onedrive/builder.rs @@ -0,0 +1,31 @@ +use std::fmt::{Debug, Formatter}; + +#[derive(Default)] +pub struct OneDriveBuilder { + access_token: Option, +} + +impl Debug for OneDriveBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut de = f.debug_struct("Builder"); + de.field("endpoint", &self.access_token); + de.finish() + } +} + +impl OneDriveBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn access_token(mut self, access_token: &str) -> Self { + self.access_token = Some(access_token.to_string()); + self + } + + // pub fn build(self) -> OneDrive { + // OneDrive { + // access_token: self.access_token, + // } + // } +} diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs new file mode 100644 index 000000000000..2abc218155ac --- /dev/null +++ b/core/src/services/onedrive/graph_model.rs @@ -0,0 +1,178 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Result; +use std::collections::HashMap; + +#[derive(Debug, Serialize, Deserialize)] +struct GraphApiOneDriveResponse { + #[serde(rename = "@odata.context")] + odata_context: String, + + #[serde(rename = "@odata.count")] + odata_count: usize, + + value: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct OneDriveItem { + #[serde(rename = "createdDateTime")] + created_date_time: String, + eTag: String, + id: String, + lastModifiedDateTime: String, + name: String, + size: usize, + webUrl: String, + parentReference: ParentReference, + fileSystemInfo: FileSystemInfo, + #[serde(flatten)] + item_type: ItemType, +} + +#[derive(Debug, Serialize, Deserialize)] +struct ParentReference { + driveId: String, + driveType: String, + id: String, + path: String, +} + +#[derive(Debug, Serialize, Deserialize)] +struct FileSystemInfo { + createdDateTime: String, + #[serde[rename = "lastModifiedDateTime"]] + last_modified_date_time: String, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +#[serde(untagged)] +enum ItemType { + Folder { + folder: HashMap, + #[serde(rename = "specialFolder")] + special_folder: HashMap, + }, + File { + file: HashMap, + }, +} + +fn parse_one_drive_json(data: &str) -> Result { + let response: GraphApiOneDriveResponse = serde_json::from_str(data)?; + Ok(response) +} + +#[test] +fn test_parse_one_drive_json() { + let data = r#"{ + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children", + "@odata.count": 1, + "value": [ + { + "createdDateTime": "2020-01-01T00:00:00Z", + "cTag": "cTag", + "eTag": "eTag", + "id": "id", + "lastModifiedDateTime": "2020-01-01T00:00:00Z", + "name": "name", + "size": 0, + "webUrl": "webUrl", + "reactions": { + "like": 0 + }, + "parentReference": { + "driveId": "driveId", + "driveType": "driveType", + "id": "id", + "path": "/drive/root:" + }, + "fileSystemInfo": { + "createdDateTime": "2020-01-01T00:00:00Z", + "lastModifiedDateTime": "2020-01-01T00:00:00Z" + }, + "folder": { + "childCount": 0 + }, + "specialFolder": { + "name": "name" + } + }, + { + "@microsoft.graph.downloadUrl": "https://public.ch.files.1drv.com/y4mPh7u0QjYTl5j9aZDj77EoplXNhXFzSbakI4iYoUXMaGUOSmpx1d20AnCoU9G32nj6W2qsKNfecsgfmF6O8ZE89yUYj7qnhsIvfikcJjJ0_skDA12gl2cCScQ3opoza_RcG2Lb_Pa2jyqiqgruh0TJRcC1y7mtEw89wqXx2bgjOvmo0ozTAwopTtpti9yo43Zb7nBI1efm3IwWhFKcHUUKx7WlD_8VPXPB4Xffokz61NiXoxMeq0hbwrblcznywz2AcE71SprDyCi8E7kDRjwmiTNoyfZc_FuUMZDO29WUbA", + "createdDateTime": "2018-12-30T05:32:55.46Z", + "cTag": "aYzozMjIxN0ZDMTE1NEFFQzNEITEwMi4yNTc", + "eTag": "aMzIyMTdGQzExNTRBRUMzRCExMDIuMw", + "id": "32217FC1154AEC3D!102", + "lastModifiedDateTime": "2018-12-30T05:33:23.557Z", + "name": "Getting started with OneDrive.pdf", + "size": 1025867, + "webUrl": "https://1drv.ms/b/s!AD3sShXBfyEyZg", + "reactions": { + "commentCount": 0 + }, + "createdBy": { + "user": { + "displayName": "Great Cat", + "id": "32217fc1154aec3d" + } + }, + "lastModifiedBy": { + "user": { + "displayName": "Great Cat", + "id": "32217fc1154aec3d" + } + }, + "parentReference": { + "driveId": "32217fc1154aec3d", + "driveType": "personal", + "id": "32217FC1154AEC3D!101", + "path": "/drive/root:" + }, + "file": { + "mimeType": "application/pdf", + "hashes": { + "quickXorHash": "NIfFZIvQVZH260260iUuQN5GscM=", + "sha1Hash": "E8890F3D1CE6E3FCCE46D08B188275D6CAE3292C" + } + }, + "fileSystemInfo": { + "createdDateTime": "2018-12-30T05:32:55.46Z", + "lastModifiedDateTime": "2018-12-30T05:32:55.46Z" + } + } + ] + }"#; + let response = parse_one_drive_json(data).unwrap(); + assert_eq!( + response.odata_context, + "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children" + ); + assert_eq!(response.odata_count, 1); + assert_eq!(response.value.len(), 2); + let item = &response.value[0]; + assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); + assert_eq!(item.eTag, "eTag"); + assert_eq!(item.id, "id"); + assert_eq!(item.lastModifiedDateTime, "2020-01-01T00:00:00Z"); + assert_eq!(item.name, "name"); + assert_eq!(item.size, 0); + assert_eq!(item.webUrl, "webUrl"); + assert_eq!( + item.item_type, + ItemType::Folder { + folder: { + let mut map = HashMap::new(); + map.insert( + "childCount".to_string(), + serde_json::Value::Number(0.into()), + ); + map + }, + special_folder: { + let mut map = HashMap::new(); + map.insert("name".to_string(), "name".to_string()); + map + }, + } + ); +} diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs new file mode 100644 index 000000000000..8e92f0b8419b --- /dev/null +++ b/core/src/services/onedrive/mod.rs @@ -0,0 +1,3 @@ +mod backend; +mod builder; +mod graph_model; diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index aff72868289e..da75582f3bbc 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -61,6 +61,8 @@ pub enum Scheme { Moka, /// [obs][crate::services::Obs]: Huawei Cloud OBS services. Obs, + /// [onedrive][crate::services::Onedrive]: Microsoft OneDrive services. + Onedrive, /// [oss][crate::services::Oss]: Aliyun Object Storage Services Oss, /// [redis][crate::services::Redis]: Redis services @@ -157,6 +159,7 @@ impl From for &'static str { Scheme::Memory => "memory", Scheme::Moka => "moka", Scheme::Obs => "obs", + Scheme::Onedrive => "onedrive", Scheme::Redis => "redis", Scheme::Rocksdb => "rocksdb", Scheme::S3 => "s3", From 452c3bf15e6c4dcbbcebe91aedefe332c23dd558 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 16:34:23 -0600 Subject: [PATCH 09/35] update res --- core/src/services/onedrive/graph_model.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs index 2abc218155ac..f22df1b45e4f 100644 --- a/core/src/services/onedrive/graph_model.rs +++ b/core/src/services/onedrive/graph_model.rs @@ -17,14 +17,20 @@ struct GraphApiOneDriveResponse { struct OneDriveItem { #[serde(rename = "createdDateTime")] created_date_time: String, - eTag: String, + #[serde(rename = "eTag")] + e_tag: String, id: String, - lastModifiedDateTime: String, + #[serde(rename = "lastModifiedDateTime")] + last_modified_date_time: String, name: String, + size: usize, - webUrl: String, - parentReference: ParentReference, - fileSystemInfo: FileSystemInfo, + #[serde(rename = "webUrl")] + web_url: String, + #[serde(rename = "parentReference")] + parent_reference: ParentReference, + #[serde(rename = "fileSystemInfo")] + file_system_info: FileSystemInfo, #[serde(flatten)] item_type: ItemType, } @@ -151,12 +157,12 @@ fn test_parse_one_drive_json() { assert_eq!(response.value.len(), 2); let item = &response.value[0]; assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.eTag, "eTag"); + assert_eq!(item.e_tag, "eTag"); assert_eq!(item.id, "id"); - assert_eq!(item.lastModifiedDateTime, "2020-01-01T00:00:00Z"); + assert_eq!(item.last_modified_date_time, "2020-01-01T00:00:00Z"); assert_eq!(item.name, "name"); assert_eq!(item.size, 0); - assert_eq!(item.webUrl, "webUrl"); + assert_eq!(item.web_url, "webUrl"); assert_eq!( item.item_type, ItemType::Folder { From e5d40c7efd1d8f8adda347d967be64a5d80ad670 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 18:05:19 -0600 Subject: [PATCH 10/35] improve builder --- core/src/services/onedrive/builder.rs | 39 ++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index f1ae50ba3147..ed8bee18fddc 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -1,8 +1,16 @@ use std::fmt::{Debug, Formatter}; +use log::debug; + +use super::backend::OneDriveBackend; +use crate::raw::normalize_root; +use crate::Scheme; +use crate::*; + #[derive(Default)] pub struct OneDriveBuilder { access_token: Option, + root: Option, } impl Debug for OneDriveBuilder { @@ -14,18 +22,35 @@ impl Debug for OneDriveBuilder { } impl OneDriveBuilder { - pub fn new() -> Self { + fn new() -> Self { Self::default() } - pub fn access_token(mut self, access_token: &str) -> Self { + fn access_token(mut self, access_token: &str) -> Self { self.access_token = Some(access_token.to_string()); self } +} - // pub fn build(self) -> OneDrive { - // OneDrive { - // access_token: self.access_token, - // } - // } +impl Builder for OneDriveBuilder { + fn build(&mut self) -> Result { + let root = normalize_root(&self.root.take().unwrap_or_default()); + debug!("backend use root {}", root); + + match self.access_token { + Some(access_token) => Ok(OneDriveBackend { + access_token: access_token, + root: root, + }), + None => Err(Error::NoAccessToken), + } + } + + const SCHEME: Scheme = Scheme::Onedrive; + + type Accessor = OneDriveBackend; + + fn from_map(map: std::collections::HashMap) -> Self { + todo!() + } } From efb27a3bfb76fd632a382ac42ff8e6da34752021 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 19:07:31 -0600 Subject: [PATCH 11/35] update --- core/src/services/onedrive/backend.rs | 11 ++++++++++- core/src/services/onedrive/builder.rs | 9 +++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index aa9220043041..8a268a5a4843 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,7 +1,10 @@ use async_trait::async_trait; use std::fmt::Debug; -use crate::{raw::{Accessor, AccessorInfo, IncomingAsyncBody}, Capability}; +use crate::{ + raw::{Accessor, AccessorInfo, IncomingAsyncBody}, + Capability, +}; #[derive(Clone)] pub struct OneDriveBackend { @@ -9,6 +12,12 @@ pub struct OneDriveBackend { access_token: String, } +impl OneDriveBackend { + pub(crate) fn new(root: String, access_token: String) -> Self { + Self { root, access_token } + } +} + impl Debug for OneDriveBackend { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut de = f.debug_struct("OneDriveBackend"); diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index ed8bee18fddc..82ff8e790f6d 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -37,12 +37,9 @@ impl Builder for OneDriveBuilder { let root = normalize_root(&self.root.take().unwrap_or_default()); debug!("backend use root {}", root); - match self.access_token { - Some(access_token) => Ok(OneDriveBackend { - access_token: access_token, - root: root, - }), - None => Err(Error::NoAccessToken), + match self.access_token.clone() { + Some(access_token) => Ok(OneDriveBackend::new(root, access_token)), + None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")), } } From 48016c68ee7f88746dd1b0e428038b5945d1a98c Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 20:14:28 -0600 Subject: [PATCH 12/35] finish builder --- core/src/services/onedrive/builder.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 82ff8e790f6d..bdb7cb26fc5a 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use log::debug; @@ -26,10 +27,15 @@ impl OneDriveBuilder { Self::default() } - fn access_token(mut self, access_token: &str) -> Self { + fn access_token(&mut self, access_token: &str) -> &mut Self { self.access_token = Some(access_token.to_string()); self } + + fn root(&mut self, root: &str) -> &mut Self { + self.root = Some(root.to_string()); + self + } } impl Builder for OneDriveBuilder { @@ -47,7 +53,12 @@ impl Builder for OneDriveBuilder { type Accessor = OneDriveBackend; - fn from_map(map: std::collections::HashMap) -> Self { - todo!() + fn from_map(map: HashMap) -> Self { + let mut builder = Self::default(); + + map.get("root").map(|v| builder.root(v)); + map.get("access_token").map(|v| builder.access_token(v)); + + builder } } From 27ff61a0b954655773ff158e441fa13bfa38cc1a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 21:22:11 -0600 Subject: [PATCH 13/35] fix read --- core/src/services/onedrive/backend.rs | 62 +++++++++++++++++++++-- core/src/services/onedrive/builder.rs | 23 ++++++--- core/src/services/onedrive/error.rs | 49 ++++++++++++++++++ core/src/services/onedrive/graph_model.rs | 9 ++-- core/src/services/onedrive/mod.rs | 1 + core/src/services/webdav/backend.rs | 2 +- 6 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 core/src/services/onedrive/error.rs diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 8a268a5a4843..d00140c06f6c 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,20 +1,34 @@ use async_trait::async_trait; +use http::{header, Request, Response, StatusCode}; +use hyper::server::conn::Http; use std::fmt::Debug; use crate::{ - raw::{Accessor, AccessorInfo, IncomingAsyncBody}, + ops::OpRead, + raw::{ + build_rooted_abs_path, new_request_build_error, parse_into_metadata, percent_encode_path, + Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, + }, + types::Result, Capability, }; +use super::error::parse_error; + #[derive(Clone)] pub struct OneDriveBackend { root: String, access_token: String, + client: HttpClient, } impl OneDriveBackend { - pub(crate) fn new(root: String, access_token: String) -> Self { - Self { root, access_token } + pub(crate) fn new(root: String, access_token: String, http_client: HttpClient) -> Self { + Self { + root, + access_token, + client: http_client, + } } } @@ -52,4 +66,46 @@ impl Accessor for OneDriveBackend { ma } + + async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { + let resp = self.onedrive_get(path).await?; + + let status = resp.status(); + + match status { + StatusCode::OK | StatusCode::PARTIAL_CONTENT => { + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } + _ => Err(parse_error(resp).await?), + } + } +} + +impl OneDriveBackend { + const ONEDRIVE_ENDPOINT_PREFIX: &'static str = + "https://graph.microsoft.com/v1.0/me/drive/root:"; + const ONEDRIVE_ENDPOINT_SUFFIX: &'static str = ":/content"; + + async fn onedrive_get(&self, path: &str) -> Result> { + let path = build_rooted_abs_path(&self.root, path); + + let url: String = format!( + "{}{}{}", + OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + percent_encode_path(&path), + OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + ); + + let mut req = Request::get(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } } diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index bdb7cb26fc5a..d5bbf39ee4d3 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -4,7 +4,7 @@ use std::fmt::{Debug, Formatter}; use log::debug; use super::backend::OneDriveBackend; -use crate::raw::normalize_root; +use crate::raw::{normalize_root, HttpClient}; use crate::Scheme; use crate::*; @@ -12,6 +12,7 @@ use crate::*; pub struct OneDriveBuilder { access_token: Option, root: Option, + http_client: Option, } impl Debug for OneDriveBuilder { @@ -23,10 +24,6 @@ impl Debug for OneDriveBuilder { } impl OneDriveBuilder { - fn new() -> Self { - Self::default() - } - fn access_token(&mut self, access_token: &str) -> &mut Self { self.access_token = Some(access_token.to_string()); self @@ -36,6 +33,11 @@ impl OneDriveBuilder { self.root = Some(root.to_string()); self } + + fn http_client(&mut self, http_client: HttpClient) -> &mut Self { + self.http_client = Some(http_client); + self + } } impl Builder for OneDriveBuilder { @@ -43,8 +45,17 @@ impl Builder for OneDriveBuilder { let root = normalize_root(&self.root.take().unwrap_or_default()); debug!("backend use root {}", root); + let client = if let Some(client) = self.http_client.take() { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::Onedrive) + })? + }; + match self.access_token.clone() { - Some(access_token) => Ok(OneDriveBackend::new(root, access_token)), + Some(access_token) => Ok(OneDriveBackend::new(root, access_token, client)), None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")), } } diff --git a/core/src/services/onedrive/error.rs b/core/src/services/onedrive/error.rs new file mode 100644 index 000000000000..6fe6b57d42aa --- /dev/null +++ b/core/src/services/onedrive/error.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use http::Response; +use http::StatusCode; + +use crate::raw::*; +use crate::Error; +use crate::ErrorKind; +use crate::Result; + +/// Parse error response into Error. +pub async fn parse_error(resp: Response) -> Result { + let (parts, body) = resp.into_parts(); + let bs = body.bytes().await?; + + let (kind, retryable) = match parts.status { + StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), + StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), + StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true), + _ => (ErrorKind::Unexpected, false), + }; + + let mut err = Error::new(kind, &String::from_utf8_lossy(&bs)) + .with_context("response", format!("{parts:?}")); + + if retryable { + err = err.set_temporary(); + } + + Ok(err) +} diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs index f22df1b45e4f..29c3e5b5a908 100644 --- a/core/src/services/onedrive/graph_model.rs +++ b/core/src/services/onedrive/graph_model.rs @@ -37,15 +37,18 @@ struct OneDriveItem { #[derive(Debug, Serialize, Deserialize)] struct ParentReference { - driveId: String, - driveType: String, + #[serde(rename = "driveId")] + drive_id: String, + #[serde(rename = "driveType")] + drive_type: String, id: String, path: String, } #[derive(Debug, Serialize, Deserialize)] struct FileSystemInfo { - createdDateTime: String, + #[serde(rename = "createdDateTime")] + created_date_time: String, #[serde[rename = "lastModifiedDateTime"]] last_modified_date_time: String, } diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 8e92f0b8419b..dce3de817b85 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -1,3 +1,4 @@ mod backend; mod builder; +mod error; mod graph_model; diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 4a073d6e9fea..2e9bd7e7afbd 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -433,7 +433,7 @@ impl WebdavBackend { ) -> Result> { let p = build_rooted_abs_path(&self.root, path); - let url = format!("{}{}", self.endpoint, percent_encode_path(&p)); + let url: String = format!("{}{}", self.endpoint, percent_encode_path(&p)); let mut req = Request::get(&url); From 9fae9dc84901cc29944948ecacf08d6bc703b02e Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 21:40:30 -0600 Subject: [PATCH 14/35] fix expose --- core/src/services/mod.rs | 2 +- core/src/services/onedrive/backend.rs | 1 - core/src/services/onedrive/builder.rs | 6 +++--- core/src/services/onedrive/mod.rs | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index f9575e9e2456..c3222bff0b57 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -132,8 +132,8 @@ pub use webdav::Webdav; #[cfg(feature = "services-webhdfs")] mod webhdfs; - mod onedrive; +pub use onedrive::OneDrive; #[cfg(feature = "services-webhdfs")] pub use webhdfs::Webhdfs; diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index d00140c06f6c..4eb00d72ab02 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,6 +1,5 @@ use async_trait::async_trait; use http::{header, Request, Response, StatusCode}; -use hyper::server::conn::Http; use std::fmt::Debug; use crate::{ diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index d5bbf39ee4d3..7598067da7d9 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -24,17 +24,17 @@ impl Debug for OneDriveBuilder { } impl OneDriveBuilder { - fn access_token(&mut self, access_token: &str) -> &mut Self { + pub fn access_token(&mut self, access_token: &str) -> &mut Self { self.access_token = Some(access_token.to_string()); self } - fn root(&mut self, root: &str) -> &mut Self { + pub fn root(&mut self, root: &str) -> &mut Self { self.root = Some(root.to_string()); self } - fn http_client(&mut self, http_client: HttpClient) -> &mut Self { + pub fn http_client(&mut self, http_client: HttpClient) -> &mut Self { self.http_client = Some(http_client); self } diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index dce3de817b85..b5777af7ab68 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -2,3 +2,5 @@ mod backend; mod builder; mod error; mod graph_model; + +pub use builder::OneDriveBuilder as OneDrive; From b4cb343c10fa250e638e42c15ab80086a1d62e4e Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 21:56:47 -0600 Subject: [PATCH 15/35] read impl --- core/src/services/onedrive/backend.rs | 49 +++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 4eb00d72ab02..83f77c770704 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -9,7 +9,7 @@ use crate::{ Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, }, types::Result, - Capability, + Capability, Error, ErrorKind, }; use super::error::parse_error; @@ -71,12 +71,36 @@ impl Accessor for OneDriveBackend { let status = resp.status(); - match status { - StatusCode::OK | StatusCode::PARTIAL_CONTENT => { - let meta = parse_into_metadata(path, resp.headers())?; - Ok((RpRead::with_metadata(meta), resp.into_body())) + if status.is_redirection() { + let location = resp + .headers() + .get(header::LOCATION) + .ok_or_else(|| { + Error::new( + ErrorKind::ContentIncomplete, + "redirect location not found in response", + ) + })? + .to_str() + .map_err(|e| { + Error::new( + ErrorKind::ContentIncomplete, + "redirect location is not valid utf-8", + ) + })?; + + let resp = self.onedrive_get_redirection(location).await?; + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } else { + match status { + StatusCode::OK | StatusCode::PARTIAL_CONTENT => { + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } + + _ => Err(parse_error(resp).await?), } - _ => Err(parse_error(resp).await?), } } } @@ -107,4 +131,17 @@ impl OneDriveBackend { self.client.send(req).await } + + async fn onedrive_get_redirection(&self, url: &str) -> Result> { + let mut req = Request::get(url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } } From 8b0ee93d1fc1013f2611d179ec4ee8d6316b2cc7 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 21:57:47 -0600 Subject: [PATCH 16/35] update --- core/src/services/onedrive/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 83f77c770704..1958d7d1d28c 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -66,7 +66,7 @@ impl Accessor for OneDriveBackend { ma } - async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { + async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> { let resp = self.onedrive_get(path).await?; let status = resp.status(); From ffd193c9754574266ce4a0f1d945798e898c1153 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:33:54 -0600 Subject: [PATCH 17/35] finish write --- core/src/services/onedrive/backend.rs | 56 +++++++++++++++++++-- core/src/services/onedrive/mod.rs | 1 + core/src/services/onedrive/writer.rs | 72 +++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 core/src/services/onedrive/writer.rs diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 1958d7d1d28c..71bfd5131d65 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -3,16 +3,16 @@ use http::{header, Request, Response, StatusCode}; use std::fmt::Debug; use crate::{ - ops::OpRead, + ops::{OpRead, OpWrite}, raw::{ build_rooted_abs_path, new_request_build_error, parse_into_metadata, percent_encode_path, - Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, + Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, RpWrite, }, types::Result, Capability, Error, ErrorKind, }; -use super::error::parse_error; +use super::{error::parse_error, writer::OneDriveWriter}; #[derive(Clone)] pub struct OneDriveBackend { @@ -44,7 +44,7 @@ impl Debug for OneDriveBackend { impl Accessor for OneDriveBackend { type Reader = IncomingAsyncBody; type BlockingReader = (); - type Writer = (); + type Writer = OneDriveWriter; type BlockingWriter = (); type Pager = (); type BlockingPager = (); @@ -103,6 +103,22 @@ impl Accessor for OneDriveBackend { } } } + + async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + if args.content_length().is_none() { + return Err(Error::new( + ErrorKind::Unsupported, + "write without content length is not supported", + )); + } + + let path = build_rooted_abs_path(&self.root, path); + + Ok(( + RpWrite::default(), + OneDriveWriter::new(self.clone(), args, path), + )) + } } impl OneDriveBackend { @@ -144,4 +160,36 @@ impl OneDriveBackend { self.client.send(req).await } + + pub async fn onedrive_put( + &self, + path: &str, + size: Option, + content_type: Option<&str>, + body: AsyncBody, + ) -> Result> { + let url = format!( + "{}{}{}", + OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + percent_encode_path(&path), + OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + ); + + let mut req = Request::put(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + if let Some(size) = size { + req = req.header(header::CONTENT_LENGTH, size) + } + + if let Some(mime) = content_type { + req = req.header(header::CONTENT_TYPE, mime) + } + + let req = req.body(body).map_err(new_request_build_error)?; + + self.client.send(req).await + } } diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index b5777af7ab68..47ef1328a71a 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -4,3 +4,4 @@ mod error; mod graph_model; pub use builder::OneDriveBuilder as OneDrive; +mod writer; diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs new file mode 100644 index 000000000000..abc33755e1f5 --- /dev/null +++ b/core/src/services/onedrive/writer.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_trait::async_trait; +use bytes::Bytes; +use http::StatusCode; + +use super::backend::OneDriveBackend; +use super::error::parse_error; +use crate::ops::OpWrite; +use crate::raw::*; +use crate::*; + +pub struct OneDriveWriter { + backend: OneDriveBackend, + + op: OpWrite, + path: String, +} + +impl OneDriveWriter { + pub fn new(backend: OneDriveBackend, op: OpWrite, path: String) -> Self { + OneDriveWriter { backend, op, path } + } +} + +#[async_trait] +impl oio::Write for OneDriveWriter { + async fn write(&mut self, bs: Bytes) -> Result<()> { + let resp = self + .backend + .onedrive_put( + &self.path, + Some(bs.len()), + self.op.content_type(), + AsyncBody::Bytes(bs), + ) + .await?; + + let status = resp.status(); + + match status { + StatusCode::CREATED | StatusCode::OK | StatusCode::NO_CONTENT => { + resp.into_body().consume().await?; + Ok(()) + } + _ => Err(parse_error(resp).await?), + } + } + + async fn abort(&mut self) -> Result<()> { + Ok(()) + } + + async fn close(&mut self) -> Result<()> { + Ok(()) + } +} From a9c1dbde248105f1599c359b63e62d890be2ec4b Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:34:48 -0600 Subject: [PATCH 18/35] Squashed commit of the following: --- .github/workflows/bindings_nodejs.yml | 4 + .github/workflows/service_test_hdfs.yml | 90 ++++++++--- CHANGELOG.md | 27 ++++ Cargo.lock | 12 +- Cargo.toml | 2 +- bindings/nodejs/index.d.ts | 18 ++- bindings/nodejs/npm/darwin-arm64/package.json | 2 +- bindings/nodejs/npm/darwin-x64/package.json | 2 +- .../nodejs/npm/linux-x64-gnu/package.json | 2 +- .../nodejs/npm/win32-x64-msvc/package.json | 2 +- bindings/nodejs/package.json | 2 +- core/src/layers/logging.rs | 2 +- core/src/services/azblob/error.rs | 4 +- core/src/services/gcs/backend.rs | 32 +++- core/src/services/gcs/core.rs | 35 +++- core/src/services/gcs/error.rs | 4 +- core/src/services/gcs/pager.rs | 29 +++- core/src/services/gcs/writer.rs | 1 + core/src/services/hdfs/writer.rs | 43 +++-- core/src/services/http/backend.rs | 4 + core/src/services/http/error.rs | 4 +- core/src/services/mod.rs | 2 +- core/src/services/obs/backend.rs | 6 + core/src/services/obs/error.rs | 4 +- core/src/services/onedrive/backend.rs | 153 +++++++++++++++++- core/src/services/onedrive/builder.rs | 62 +++++-- core/src/services/onedrive/error.rs | 49 ++++++ core/src/services/onedrive/graph_model.rs | 31 ++-- core/src/services/onedrive/mod.rs | 4 + core/src/services/onedrive/writer.rs | 72 +++++++++ core/src/services/oss/backend.rs | 7 + core/src/services/oss/error.rs | 4 +- core/src/services/s3/pager.rs | 2 + core/src/services/wasabi/backend.rs | 6 + core/src/services/webdav/backend.rs | 2 +- core/tests/behavior/read_only.rs | 60 +++++++ core/tests/behavior/write.rs | 37 +++++ 37 files changed, 718 insertions(+), 104 deletions(-) create mode 100644 core/src/services/onedrive/error.rs create mode 100644 core/src/services/onedrive/writer.rs diff --git a/.github/workflows/bindings_nodejs.yml b/.github/workflows/bindings_nodejs.yml index c5afbf1e2237..95cabf1f7ab1 100644 --- a/.github/workflows/bindings_nodejs.yml +++ b/.github/workflows/bindings_nodejs.yml @@ -70,6 +70,10 @@ jobs: - name: Build run: yarn build:debug + + - name: Check diff + run: git diff --exit-code + - name: Test bindings run: yarn test diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml index 2789bf3f74a7..4f7d7083e55a 100644 --- a/.github/workflows/service_test_hdfs.yml +++ b/.github/workflows/service_test_hdfs.yml @@ -37,39 +37,93 @@ concurrency: cancel-in-progress: true jobs: - hdfs: + hdfs-default: runs-on: ubuntu-latest - strategy: - matrix: - hdfs-version: ["2.10.1", "3.2.3", "3.3.2"] steps: - uses: actions/checkout@v3 - - name: Checkout python env - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - name: Checkout java env + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Setup java env uses: actions/setup-java@v3 with: distribution: temurin java-version: "11" - - name: Setup-hdfs env - uses: beyondstorage/setup-hdfs@master - with: - hdfs-version: ${{ matrix.hdfs-version }} + - name: Setup hadoop env + shell: bash + run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + - name: Test + shell: bash + working-directory: core + run: | + export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':') + cargo test services_hdfs --features services-hdfs -- --show-output + env: + RUST_BACKTRACE: full + RUST_LOG: debug + HADOOP_HOME: "/home/runner/hadoop-3.3.5" + LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} + OPENDAL_HDFS_TEST: on + OPENDAL_HDFS_ROOT: /tmp/opendal/ + OPENDAL_HDFS_NAME_NODE: default + + hdfs-cluster: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Configure Hdfs + # namenode will use ports: 9870, 9000, 8020 + # datanode will use ports: 9864 + run: | + docker run -d \ + --name namenode \ + --network host \ + -e CLUSTER_NAME=test \ + -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ + -e CORE_CONF_hadoop_http_staticuser_user=root \ + -e HDFS_CONF_dfs_permissions_enabled=false \ + bde2020/hadoop-namenode:2.0.0-hadoop3.1.3-java8 + + docker run -d \ + --name datanode \ + --network host \ + -e CLUSTER_NAME=test \ + -e WEBHDFS_CONF_dfs_webhdfs_enabled=true \ + -e CORE_CONF_hadoop_http_staticuser_user=root \ + -e HDFS_CONF_dfs_permissions_enabled=false \ + bde2020/hadoop-datanode:2.0.0-hadoop3.1.3-java8 + + curl --retry 30 --retry-delay 1 --retry-connrefused http://localhost:9870 - name: Setup Rust toolchain uses: ./.github/actions/setup + + - name: Setup java env + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: "11" + - name: Setup hadoop env + shell: bash + run: | + curl -LsSf https://archive.apache.org/dist/hadoop/common/hadoop-3.1.3/hadoop-3.1.3.tar.gz | tar zxf - -C /home/runner + - name: Test shell: bash working-directory: core - continue-on-error: true - run: cargo test hdfs --features services-hdfs -- --show-output + run: | + export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':') + + cargo test services_hdfs --features services-hdfs -- --show-output env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_HDFS_TEST: on - OPENDAL_HDFS_ROOT: / - OPENDAL_HDFS_NAME_NODE: hdfs://${{ env.HDFS_NAMENODE_ADDR }} + HADOOP_HOME: "/home/runner/hadoop-3.1.3" LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} + OPENDAL_HDFS_TEST: on + OPENDAL_HDFS_ROOT: /tmp/opendal/ + OPENDAL_HDFS_NAME_NODE: hdfs://localhost:8020 diff --git a/CHANGELOG.md b/CHANGELOG.md index 433e9c884f93..f823424ed688 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [v0.33.1] - 2023-04-25 + +### Added + +- feat: Add behavior test for read_with_if_match & stat_with_if_match (#2088) +- feat(tests): Add fuzz test for writer without content length (#2100) +- feat: add if_none_match support for obs (#2103) +- feat(services/oss): Add server side encryption support for oss (#2092) +- feat(core): update errorKind `PreconditionFailed` to `ConditionNotMatch` (#2104) +- feat(services/s3): Add `start-after` support for list (#2096) +- feat: gcs support cache control (#2116) + +### Fixed + +- fix(services/gcs): set `content length=0` for gcs initiate_resumable_upload (#2110) +- fix(bindings/nodejs): Fix index.d.ts not updated (#2117) + +### Docs + +- chore: improve LoggingLayer docs and pub use log::Level (#2089) +- docs(refactor): Add more detailed description of operator, accessor, and builder (#2094) + +### CI + +- chore(bindings/nodejs): update `package.json` repository info (#2078) +- ci: Bring hdfs test back (#2114) + ## [v0.33.0] - 2023-04-23 ### Added diff --git a/Cargo.lock b/Cargo.lock index 0922e6c72873..5ad025660522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2293,7 +2293,7 @@ dependencies = [ [[package]] name = "oay" -version = "0.33.0" +version = "0.33.1" dependencies = [ "anyhow", "clap 4.1.11", @@ -2330,7 +2330,7 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.33.0" +version = "0.33.1" dependencies = [ "async-trait", "bytes", @@ -2342,7 +2342,7 @@ dependencies = [ [[package]] name = "oli" -version = "0.33.0" +version = "0.33.1" dependencies = [ "anyhow", "assert_cmd", @@ -2373,7 +2373,7 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.33.0" +version = "0.33.1" dependencies = [ "anyhow", "async-compat", @@ -2450,7 +2450,7 @@ dependencies = [ [[package]] name = "opendal-nodejs" -version = "0.33.0" +version = "0.33.1" dependencies = [ "futures", "napi", @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "opendal-python" -version = "0.33.0" +version = "0.33.1" dependencies = [ "chrono", "futures", diff --git a/Cargo.toml b/Cargo.toml index 75cb0822af34..3a5f67fe49dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/incubator-opendal" rust-version = "1.64" -version = "0.33.0" +version = "0.33.1" [workspace.dependencies] opendal = { version = "0.33", path = "core" } diff --git a/bindings/nodejs/index.d.ts b/bindings/nodejs/index.d.ts index 1bd46945f107..e0b2adecc0e2 100644 --- a/bindings/nodejs/index.d.ts +++ b/bindings/nodejs/index.d.ts @@ -336,14 +336,14 @@ export class Operator { * ### Example * * ```javascript - * const req = op.presignRead(path, parseInt(expires)); + * const req = await op.presignRead(path, parseInt(expires)); * * console.log("method: ", req.method); * console.log("url: ", req.url); * console.log("headers: ", req.headers); * ``` */ - presignRead(path: string, expires: number): PresignedRequest + presignRead(path: string, expires: number): Promise /** * Get a presigned request for write. * @@ -352,14 +352,14 @@ export class Operator { * ### Example * * ```javascript - * const req = op.presignWrite(path, parseInt(expires)); + * const req = await op.presignWrite(path, parseInt(expires)); * * console.log("method: ", req.method); * console.log("url: ", req.url); * console.log("headers: ", req.headers); * ``` */ - presignWrite(path: string, expires: number): PresignedRequest + presignWrite(path: string, expires: number): Promise /** * Get a presigned request for stat. * @@ -368,14 +368,14 @@ export class Operator { * ### Example * * ```javascript - * const req = op.presignStat(path, parseInt(expires)); + * const req = await op.presignStat(path, parseInt(expires)); * * console.log("method: ", req.method); * console.log("url: ", req.url); * console.log("headers: ", req.headers); * ``` */ - presignStat(path: string, expires: number): PresignedRequest + presignStat(path: string, expires: number): Promise } export class Entry { /** Return the path of this entry. */ @@ -396,7 +396,11 @@ export class Metadata { get contentType(): string | null /** ETag of this object. */ get etag(): string | null - /** Last Modified of this object.(UTC) */ + /** + * Last Modified of this object. + * + * We will output this time in RFC3339 format like `1996-12-19T16:39:57+08:00`. + */ get lastModified(): string | null } export class Lister { diff --git a/bindings/nodejs/npm/darwin-arm64/package.json b/bindings/nodejs/npm/darwin-arm64/package.json index d5ca5c748f81..23dfd4eb5b6a 100644 --- a/bindings/nodejs/npm/darwin-arm64/package.json +++ b/bindings/nodejs/npm/darwin-arm64/package.json @@ -1,7 +1,7 @@ { "name": "@opendal/lib-darwin-arm64", "repository": "git@github.com/apache/incubator-opendal.git", - "version": "0.33.0", + "version": "0.33.1", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/darwin-x64/package.json b/bindings/nodejs/npm/darwin-x64/package.json index 9ddc7f7a05de..11554d2537e7 100644 --- a/bindings/nodejs/npm/darwin-x64/package.json +++ b/bindings/nodejs/npm/darwin-x64/package.json @@ -1,7 +1,7 @@ { "name": "@opendal/lib-darwin-x64", "repository": "git@github.com/apache/incubator-opendal.git", - "version": "0.33.0", + "version": "0.33.1", "os": [ "darwin" ], diff --git a/bindings/nodejs/npm/linux-x64-gnu/package.json b/bindings/nodejs/npm/linux-x64-gnu/package.json index c5e4cf88dd00..cf33f7c3c81c 100644 --- a/bindings/nodejs/npm/linux-x64-gnu/package.json +++ b/bindings/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@opendal/lib-linux-x64-gnu", - "version": "0.33.0", + "version": "0.33.1", "repository": "git@github.com/apache/incubator-opendal.git", "os": [ "linux" diff --git a/bindings/nodejs/npm/win32-x64-msvc/package.json b/bindings/nodejs/npm/win32-x64-msvc/package.json index cf9c50686b28..6ec9b54dce75 100644 --- a/bindings/nodejs/npm/win32-x64-msvc/package.json +++ b/bindings/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@opendal/lib-win32-x64-msvc", - "version": "0.33.0", + "version": "0.33.1", "repository": "git@github.com/apache/incubator-opendal.git", "os": [ "win32" diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index 7c36a607df31..39fc9ff5d639 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -1,7 +1,7 @@ { "name": "opendal", "author": "OpenDAL Contributors ", - "version": "0.33.0", + "version": "0.33.1", "license": "Apache-2.0", "main": "index.js", "types": "index.d.ts", diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs index e5a83f1408f5..208ebffbdd6d 100644 --- a/core/src/layers/logging.rs +++ b/core/src/layers/logging.rs @@ -754,7 +754,7 @@ impl LayeredAccessor for LoggingAccessor { .map(|(rp, w)| { debug!( target: LOGGING_TARGET, - "service={} operation={} path={} -> written", + "service={} operation={} path={} -> start writing", self.scheme, Operation::BlockingWrite, path, diff --git a/core/src/services/azblob/error.rs b/core/src/services/azblob/error.rs index c73e956b3c06..76eac0a30c04 100644 --- a/core/src/services/azblob/error.rs +++ b/core/src/services/azblob/error.rs @@ -90,7 +90,9 @@ pub async fn parse_error(resp: Response) -> Result { let (kind, retryable) = match parts.status { StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), - StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false), + StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => { + (ErrorKind::ConditionNotMatch, false) + } StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index 778c77812015..006c3b1bb1fc 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -374,22 +374,31 @@ impl Accessor for GcsBackend { .set_root(&self.core.root) .set_name(&self.core.bucket) .set_capability(Capability { + stat: true, + stat_with_if_match: true, + stat_with_if_none_match: true, + read: true, read_can_next: true, + write: true, write_without_content_length: true, + list: true, + list_with_limit: true, + list_with_start_after: true, scan: true, copy: true, + ..Default::default() }); am } async fn create_dir(&self, path: &str, _: OpCreate) -> Result { - let mut req = self - .core - .gcs_insert_object_request(path, Some(0), None, AsyncBody::Empty)?; + let mut req = + self.core + .gcs_insert_object_request(path, Some(0), None, None, AsyncBody::Empty)?; self.core.sign(&mut req).await?; @@ -435,13 +444,16 @@ impl Accessor for GcsBackend { } } - async fn stat(&self, path: &str, _: OpStat) -> Result { + async fn stat(&self, path: &str, args: OpStat) -> Result { // Stat root always returns a DIR. if path == "/" { return Ok(RpStat::new(Metadata::new(EntryMode::DIR))); } - let resp = self.core.gcs_get_object_metadata(path).await?; + let resp = self + .core + .gcs_get_object_metadata(path, args.if_match(), args.if_none_match()) + .await?; if resp.status().is_success() { // read http response body @@ -492,14 +504,20 @@ impl Accessor for GcsBackend { async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { Ok(( RpList::default(), - GcsPager::new(self.core.clone(), path, "/", args.limit()), + GcsPager::new( + self.core.clone(), + path, + "/", + args.limit(), + args.start_after(), + ), )) } async fn scan(&self, path: &str, args: OpScan) -> Result<(RpScan, Self::Pager)> { Ok(( RpScan::default(), - GcsPager::new(self.core.clone(), path, "", args.limit()), + GcsPager::new(self.core.clone(), path, "", args.limit(), None), )) } } diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs index 044d6332b841..323cd0158f96 100644 --- a/core/src/services/gcs/core.rs +++ b/core/src/services/gcs/core.rs @@ -23,6 +23,7 @@ use backon::ExponentialBuilder; use backon::Retryable; use bytes::Bytes; use bytes::BytesMut; +use http::header::CACHE_CONTROL; use http::header::CONTENT_LENGTH; use http::header::CONTENT_RANGE; use http::header::CONTENT_TYPE; @@ -150,6 +151,7 @@ impl GcsCore { path: &str, size: Option, content_type: Option<&str>, + cache_control: Option<&str>, body: AsyncBody, ) -> Result> { let p = build_abs_path(&self.root, path); @@ -172,8 +174,10 @@ impl GcsCore { let mut req = Request::post(&url); - if let Some(size) = size { - req = req.header(CONTENT_LENGTH, size) + req = req.header(CONTENT_LENGTH, size.unwrap_or_default()); + + if let Some(cache_control) = cache_control { + req = req.header(CACHE_CONTROL, cache_control) } if let Some(storage_class) = &self.default_storage_class { @@ -210,7 +214,12 @@ impl GcsCore { } } - pub async fn gcs_get_object_metadata(&self, path: &str) -> Result> { + pub async fn gcs_get_object_metadata( + &self, + path: &str, + if_match: Option<&str>, + if_none_match: Option<&str>, + ) -> Result> { let p = build_abs_path(&self.root, path); let url = format!( @@ -220,7 +229,15 @@ impl GcsCore { percent_encode_path(&p) ); - let req = Request::get(&url); + let mut req = Request::get(&url); + + if let Some(if_none_match) = if_none_match { + req = req.header(IF_NONE_MATCH, if_none_match); + } + + if let Some(if_match) = if_match { + req = req.header(IF_MATCH, if_match); + } let mut req = req .body(AsyncBody::Empty) @@ -281,6 +298,7 @@ impl GcsCore { page_token: &str, delimiter: &str, limit: Option, + start_after: Option, ) -> Result> { let p = build_abs_path(&self.root, path); @@ -296,6 +314,12 @@ impl GcsCore { if let Some(limit) = limit { write!(url, "&maxResults={limit}").expect("write into string must succeed"); } + if let Some(start_after) = start_after { + let start_after = build_abs_path(&self.root, &start_after); + write!(url, "&startOffset={}", percent_encode_path(&start_after)) + .expect("write into string must succeed"); + } + if !page_token.is_empty() { // NOTE: // @@ -325,9 +349,12 @@ impl GcsCore { "{}/upload/storage/v1/b/{}/o?uploadType=resumable&name={}", self.endpoint, self.bucket, p ); + let mut req = Request::post(&url) + .header(CONTENT_LENGTH, 0) .body(AsyncBody::Empty) .map_err(new_request_build_error)?; + self.sign(&mut req).await?; self.send(req).await } diff --git a/core/src/services/gcs/error.rs b/core/src/services/gcs/error.rs index 0f8e262d90ec..3f2e0b688a65 100644 --- a/core/src/services/gcs/error.rs +++ b/core/src/services/gcs/error.rs @@ -57,7 +57,9 @@ pub async fn parse_error(resp: Response) -> Result { let (kind, retryable) = match parts.status { StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), - StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false), + StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => { + (ErrorKind::ConditionNotMatch, false) + } StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE diff --git a/core/src/services/gcs/pager.rs b/core/src/services/gcs/pager.rs index bfa999811042..e6cffd6752ae 100644 --- a/core/src/services/gcs/pager.rs +++ b/core/src/services/gcs/pager.rs @@ -35,19 +35,30 @@ pub struct GcsPager { delimiter: String, limit: Option, + /// Filter results to objects whose names are lexicographically + /// **equal to or after** startOffset + start_after: Option, + page_token: String, done: bool, } impl GcsPager { /// Generate a new directory walker - pub fn new(core: Arc, path: &str, delimiter: &str, limit: Option) -> Self { + pub fn new( + core: Arc, + path: &str, + delimiter: &str, + limit: Option, + start_after: Option<&str>, + ) -> Self { Self { core, path: path.to_string(), delimiter: delimiter.to_string(), limit, + start_after: start_after.map(String::from), page_token: "".to_string(), done: false, @@ -64,7 +75,13 @@ impl oio::Page for GcsPager { let resp = self .core - .gcs_list_objects(&self.path, &self.page_token, &self.delimiter, self.limit) + .gcs_list_objects( + &self.path, + &self.page_token, + &self.delimiter, + self.limit, + self.start_after.clone(), + ) .await?; if !resp.status().is_success() { @@ -97,6 +114,12 @@ impl oio::Page for GcsPager { continue; } + // exclude the inclusive start_after itself + let path = &build_rel_path(&self.core.root, &object.name); + if self.start_after.as_ref() == Some(path) { + continue; + } + let mut meta = Metadata::new(EntryMode::FILE); // set metadata fields @@ -113,7 +136,7 @@ impl oio::Page for GcsPager { meta.set_last_modified(parse_datetime_from_rfc3339(object.updated.as_str())?); - let de = oio::Entry::new(&build_rel_path(&self.core.root, &object.name), meta); + let de = oio::Entry::new(path, meta); entries.push(de); } diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs index c4ed1f6e0449..1c48afbc231c 100644 --- a/core/src/services/gcs/writer.rs +++ b/core/src/services/gcs/writer.rs @@ -67,6 +67,7 @@ impl GcsWriter { &percent_encode_path(&self.path), Some(bs.len()), self.op.content_type(), + self.op.cache_control(), AsyncBody::Bytes(bs), )?; diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs index d045c0638f34..9ddd34f8a099 100644 --- a/core/src/services/hdfs/writer.rs +++ b/core/src/services/hdfs/writer.rs @@ -15,13 +15,10 @@ // specific language governing permissions and limitations // under the License. -use std::io::Seek; -use std::io::SeekFrom; use std::io::Write; use async_trait::async_trait; use bytes::Bytes; -use futures::AsyncSeekExt; use futures::AsyncWriteExt; use super::error::parse_io_error; @@ -30,7 +27,10 @@ use crate::*; pub struct HdfsWriter { f: F, - pos: u64, + /// The position of current written bytes in the buffer. + /// + /// We will maintain the posstion in pos to make sure the buffer is written correctly. + pos: usize, } impl HdfsWriter { @@ -41,17 +41,17 @@ impl HdfsWriter { #[async_trait] impl oio::Write for HdfsWriter { - /// # Notes - /// - /// File could be partial written, so we will seek to start to make sure - /// we write the same content. async fn write(&mut self, bs: Bytes) -> Result<()> { - self.f - .seek(SeekFrom::Start(self.pos)) - .await - .map_err(parse_io_error)?; - self.f.write_all(&bs).await.map_err(parse_io_error)?; - self.pos += bs.len() as u64; + while self.pos < bs.len() { + let n = self + .f + .write(&bs[self.pos..]) + .await + .map_err(parse_io_error)?; + self.pos += n; + } + // Reset pos to 0 for next write. + self.pos = 0; Ok(()) } @@ -71,16 +71,13 @@ impl oio::Write for HdfsWriter { } impl oio::BlockingWrite for HdfsWriter { - /// # Notes - /// - /// File could be partial written, so we will seek to start to make sure - /// we write the same content. fn write(&mut self, bs: Bytes) -> Result<()> { - self.f - .seek(SeekFrom::Start(self.pos)) - .map_err(parse_io_error)?; - self.f.write_all(&bs).map_err(parse_io_error)?; - self.pos += bs.len() as u64; + while self.pos < bs.len() { + let n = self.f.write(&bs[self.pos..]).map_err(parse_io_error)?; + self.pos += n; + } + // Reset pos to 0 for next write. + self.pos = 0; Ok(()) } diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index 1ca6776f3395..8a4d3fb6ab76 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -259,6 +259,10 @@ impl Accessor for HttpBackend { ma.set_scheme(Scheme::Http) .set_root(&self.root) .set_capability(Capability { + stat: true, + stat_with_if_match: true, + stat_with_if_none_match: true, + read: true, read_can_next: true, diff --git a/core/src/services/http/error.rs b/core/src/services/http/error.rs index e330e5720956..aed2e13d63b1 100644 --- a/core/src/services/http/error.rs +++ b/core/src/services/http/error.rs @@ -31,7 +31,9 @@ pub async fn parse_error(resp: Response) -> Result { let (kind, retryable) = match parts.status { StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), - StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false), + StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => { + (ErrorKind::ConditionNotMatch, false) + } StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index f9575e9e2456..c3222bff0b57 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -132,8 +132,8 @@ pub use webdav::Webdav; #[cfg(feature = "services-webhdfs")] mod webhdfs; - mod onedrive; +pub use onedrive::OneDrive; #[cfg(feature = "services-webhdfs")] pub use webhdfs::Webhdfs; diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 3ed1c55a7ddd..c9c77e004156 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -306,12 +306,18 @@ impl Accessor for ObsBackend { .set_root(&self.core.root) .set_name(&self.core.bucket) .set_capability(Capability { + stat: true, + stat_with_if_match: true, + stat_with_if_none_match: true, + read: true, read_can_next: true, + write: true, list: true, scan: true, copy: true, + ..Default::default() }); diff --git a/core/src/services/obs/error.rs b/core/src/services/obs/error.rs index 3bf328a91085..ddcca5f39047 100644 --- a/core/src/services/obs/error.rs +++ b/core/src/services/obs/error.rs @@ -45,7 +45,9 @@ pub async fn parse_error(resp: Response) -> Result { let (kind, retryable) = match parts.status { StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), - StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false), + StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => { + (ErrorKind::ConditionNotMatch, false) + } StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index aa9220043041..71bfd5131d65 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,12 +1,34 @@ use async_trait::async_trait; +use http::{header, Request, Response, StatusCode}; use std::fmt::Debug; -use crate::{raw::{Accessor, AccessorInfo, IncomingAsyncBody}, Capability}; +use crate::{ + ops::{OpRead, OpWrite}, + raw::{ + build_rooted_abs_path, new_request_build_error, parse_into_metadata, percent_encode_path, + Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, RpWrite, + }, + types::Result, + Capability, Error, ErrorKind, +}; + +use super::{error::parse_error, writer::OneDriveWriter}; #[derive(Clone)] pub struct OneDriveBackend { root: String, access_token: String, + client: HttpClient, +} + +impl OneDriveBackend { + pub(crate) fn new(root: String, access_token: String, http_client: HttpClient) -> Self { + Self { + root, + access_token, + client: http_client, + } + } } impl Debug for OneDriveBackend { @@ -22,7 +44,7 @@ impl Debug for OneDriveBackend { impl Accessor for OneDriveBackend { type Reader = IncomingAsyncBody; type BlockingReader = (); - type Writer = (); + type Writer = OneDriveWriter; type BlockingWriter = (); type Pager = (); type BlockingPager = (); @@ -43,4 +65,131 @@ impl Accessor for OneDriveBackend { ma } + + async fn read(&self, path: &str, _args: OpRead) -> Result<(RpRead, Self::Reader)> { + let resp = self.onedrive_get(path).await?; + + let status = resp.status(); + + if status.is_redirection() { + let location = resp + .headers() + .get(header::LOCATION) + .ok_or_else(|| { + Error::new( + ErrorKind::ContentIncomplete, + "redirect location not found in response", + ) + })? + .to_str() + .map_err(|e| { + Error::new( + ErrorKind::ContentIncomplete, + "redirect location is not valid utf-8", + ) + })?; + + let resp = self.onedrive_get_redirection(location).await?; + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } else { + match status { + StatusCode::OK | StatusCode::PARTIAL_CONTENT => { + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } + + _ => Err(parse_error(resp).await?), + } + } + } + + async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + if args.content_length().is_none() { + return Err(Error::new( + ErrorKind::Unsupported, + "write without content length is not supported", + )); + } + + let path = build_rooted_abs_path(&self.root, path); + + Ok(( + RpWrite::default(), + OneDriveWriter::new(self.clone(), args, path), + )) + } +} + +impl OneDriveBackend { + const ONEDRIVE_ENDPOINT_PREFIX: &'static str = + "https://graph.microsoft.com/v1.0/me/drive/root:"; + const ONEDRIVE_ENDPOINT_SUFFIX: &'static str = ":/content"; + + async fn onedrive_get(&self, path: &str) -> Result> { + let path = build_rooted_abs_path(&self.root, path); + + let url: String = format!( + "{}{}{}", + OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + percent_encode_path(&path), + OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + ); + + let mut req = Request::get(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } + + async fn onedrive_get_redirection(&self, url: &str) -> Result> { + let mut req = Request::get(url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + let req = req + .body(AsyncBody::Empty) + .map_err(new_request_build_error)?; + + self.client.send(req).await + } + + pub async fn onedrive_put( + &self, + path: &str, + size: Option, + content_type: Option<&str>, + body: AsyncBody, + ) -> Result> { + let url = format!( + "{}{}{}", + OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + percent_encode_path(&path), + OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + ); + + let mut req = Request::put(&url); + + let auth_header_content = format!("Bearer {}", self.access_token); + req = req.header(header::AUTHORIZATION, auth_header_content); + + if let Some(size) = size { + req = req.header(header::CONTENT_LENGTH, size) + } + + if let Some(mime) = content_type { + req = req.header(header::CONTENT_TYPE, mime) + } + + let req = req.body(body).map_err(new_request_build_error)?; + + self.client.send(req).await + } } diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index f1ae50ba3147..7598067da7d9 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -1,8 +1,18 @@ +use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use log::debug; + +use super::backend::OneDriveBackend; +use crate::raw::{normalize_root, HttpClient}; +use crate::Scheme; +use crate::*; + #[derive(Default)] pub struct OneDriveBuilder { access_token: Option, + root: Option, + http_client: Option, } impl Debug for OneDriveBuilder { @@ -14,18 +24,52 @@ impl Debug for OneDriveBuilder { } impl OneDriveBuilder { - pub fn new() -> Self { - Self::default() + pub fn access_token(&mut self, access_token: &str) -> &mut Self { + self.access_token = Some(access_token.to_string()); + self } - pub fn access_token(mut self, access_token: &str) -> Self { - self.access_token = Some(access_token.to_string()); + pub fn root(&mut self, root: &str) -> &mut Self { + self.root = Some(root.to_string()); self } - // pub fn build(self) -> OneDrive { - // OneDrive { - // access_token: self.access_token, - // } - // } + pub fn http_client(&mut self, http_client: HttpClient) -> &mut Self { + self.http_client = Some(http_client); + self + } +} + +impl Builder for OneDriveBuilder { + fn build(&mut self) -> Result { + let root = normalize_root(&self.root.take().unwrap_or_default()); + debug!("backend use root {}", root); + + let client = if let Some(client) = self.http_client.take() { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::Onedrive) + })? + }; + + match self.access_token.clone() { + Some(access_token) => Ok(OneDriveBackend::new(root, access_token, client)), + None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")), + } + } + + const SCHEME: Scheme = Scheme::Onedrive; + + type Accessor = OneDriveBackend; + + fn from_map(map: HashMap) -> Self { + let mut builder = Self::default(); + + map.get("root").map(|v| builder.root(v)); + map.get("access_token").map(|v| builder.access_token(v)); + + builder + } } diff --git a/core/src/services/onedrive/error.rs b/core/src/services/onedrive/error.rs new file mode 100644 index 000000000000..6fe6b57d42aa --- /dev/null +++ b/core/src/services/onedrive/error.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use http::Response; +use http::StatusCode; + +use crate::raw::*; +use crate::Error; +use crate::ErrorKind; +use crate::Result; + +/// Parse error response into Error. +pub async fn parse_error(resp: Response) -> Result { + let (parts, body) = resp.into_parts(); + let bs = body.bytes().await?; + + let (kind, retryable) = match parts.status { + StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), + StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), + StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true), + _ => (ErrorKind::Unexpected, false), + }; + + let mut err = Error::new(kind, &String::from_utf8_lossy(&bs)) + .with_context("response", format!("{parts:?}")); + + if retryable { + err = err.set_temporary(); + } + + Ok(err) +} diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs index 2abc218155ac..29c3e5b5a908 100644 --- a/core/src/services/onedrive/graph_model.rs +++ b/core/src/services/onedrive/graph_model.rs @@ -17,29 +17,38 @@ struct GraphApiOneDriveResponse { struct OneDriveItem { #[serde(rename = "createdDateTime")] created_date_time: String, - eTag: String, + #[serde(rename = "eTag")] + e_tag: String, id: String, - lastModifiedDateTime: String, + #[serde(rename = "lastModifiedDateTime")] + last_modified_date_time: String, name: String, + size: usize, - webUrl: String, - parentReference: ParentReference, - fileSystemInfo: FileSystemInfo, + #[serde(rename = "webUrl")] + web_url: String, + #[serde(rename = "parentReference")] + parent_reference: ParentReference, + #[serde(rename = "fileSystemInfo")] + file_system_info: FileSystemInfo, #[serde(flatten)] item_type: ItemType, } #[derive(Debug, Serialize, Deserialize)] struct ParentReference { - driveId: String, - driveType: String, + #[serde(rename = "driveId")] + drive_id: String, + #[serde(rename = "driveType")] + drive_type: String, id: String, path: String, } #[derive(Debug, Serialize, Deserialize)] struct FileSystemInfo { - createdDateTime: String, + #[serde(rename = "createdDateTime")] + created_date_time: String, #[serde[rename = "lastModifiedDateTime"]] last_modified_date_time: String, } @@ -151,12 +160,12 @@ fn test_parse_one_drive_json() { assert_eq!(response.value.len(), 2); let item = &response.value[0]; assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.eTag, "eTag"); + assert_eq!(item.e_tag, "eTag"); assert_eq!(item.id, "id"); - assert_eq!(item.lastModifiedDateTime, "2020-01-01T00:00:00Z"); + assert_eq!(item.last_modified_date_time, "2020-01-01T00:00:00Z"); assert_eq!(item.name, "name"); assert_eq!(item.size, 0); - assert_eq!(item.webUrl, "webUrl"); + assert_eq!(item.web_url, "webUrl"); assert_eq!( item.item_type, ItemType::Folder { diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 8e92f0b8419b..47ef1328a71a 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -1,3 +1,7 @@ mod backend; mod builder; +mod error; mod graph_model; + +pub use builder::OneDriveBuilder as OneDrive; +mod writer; diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs new file mode 100644 index 000000000000..abc33755e1f5 --- /dev/null +++ b/core/src/services/onedrive/writer.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_trait::async_trait; +use bytes::Bytes; +use http::StatusCode; + +use super::backend::OneDriveBackend; +use super::error::parse_error; +use crate::ops::OpWrite; +use crate::raw::*; +use crate::*; + +pub struct OneDriveWriter { + backend: OneDriveBackend, + + op: OpWrite, + path: String, +} + +impl OneDriveWriter { + pub fn new(backend: OneDriveBackend, op: OpWrite, path: String) -> Self { + OneDriveWriter { backend, op, path } + } +} + +#[async_trait] +impl oio::Write for OneDriveWriter { + async fn write(&mut self, bs: Bytes) -> Result<()> { + let resp = self + .backend + .onedrive_put( + &self.path, + Some(bs.len()), + self.op.content_type(), + AsyncBody::Bytes(bs), + ) + .await?; + + let status = resp.status(); + + match status { + StatusCode::CREATED | StatusCode::OK | StatusCode::NO_CONTENT => { + resp.into_body().consume().await?; + Ok(()) + } + _ => Err(parse_error(resp).await?), + } + } + + async fn abort(&mut self) -> Result<()> { + Ok(()) + } + + async fn close(&mut self) -> Result<()> { + Ok(()) + } +} diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index f6e213945a38..8ecb0c957134 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -424,16 +424,23 @@ impl Accessor for OssBackend { .set_root(&self.core.root) .set_name(&self.core.bucket) .set_capability(Capability { + stat: true, + stat_with_if_match: true, + stat_with_if_none_match: true, + read: true, read_can_next: true, + write: true, write_without_content_length: true, + list: true, scan: true, copy: true, presign: true, batch: true, batch_max_operations: Some(1000), + ..Default::default() }); diff --git a/core/src/services/oss/error.rs b/core/src/services/oss/error.rs index 4e86245febc1..715ad744f8c0 100644 --- a/core/src/services/oss/error.rs +++ b/core/src/services/oss/error.rs @@ -44,7 +44,9 @@ pub async fn parse_error(resp: Response) -> Result { let (kind, retryable) = match parts.status { StatusCode::NOT_FOUND => (ErrorKind::NotFound, false), StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false), - StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false), + StatusCode::PRECONDITION_FAILED | StatusCode::NOT_MODIFIED => { + (ErrorKind::ConditionNotMatch, false) + } StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE diff --git a/core/src/services/s3/pager.rs b/core/src/services/s3/pager.rs index c8e98c168551..3c5b15ed0337 100644 --- a/core/src/services/s3/pager.rs +++ b/core/src/services/s3/pager.rs @@ -35,6 +35,8 @@ pub struct S3Pager { path: String, delimiter: String, limit: Option, + + /// Amazon S3 starts listing **after** this specified key start_after: Option, token: String, diff --git a/core/src/services/wasabi/backend.rs b/core/src/services/wasabi/backend.rs index 3b85e9fbb58d..5190bb2133f6 100644 --- a/core/src/services/wasabi/backend.rs +++ b/core/src/services/wasabi/backend.rs @@ -902,8 +902,13 @@ impl Accessor for WasabiBackend { .set_root(&self.core.root) .set_name(&self.core.bucket) .set_capability(Capability { + stat: true, + stat_with_if_match: true, + stat_with_if_none_match: true, + read: true, read_can_next: true, + write: true, list: true, scan: true, @@ -911,6 +916,7 @@ impl Accessor for WasabiBackend { presign: true, batch: true, rename: true, + ..Default::default() }); diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 4a073d6e9fea..2e9bd7e7afbd 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -433,7 +433,7 @@ impl WebdavBackend { ) -> Result> { let p = build_rooted_abs_path(&self.root, path); - let url = format!("{}{}", self.endpoint, percent_encode_path(&p)); + let url: String = format!("{}{}", self.endpoint, percent_encode_path(&p)); let mut req = Request::get(&url); diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs index 6d3194ccce41..fdaf939e1f28 100644 --- a/core/tests/behavior/read_only.rs +++ b/core/tests/behavior/read_only.rs @@ -17,6 +17,7 @@ use anyhow::Result; use futures::AsyncReadExt; +use opendal::ops::OpStat; use opendal::EntryMode; use opendal::ErrorKind; use opendal::Operator; @@ -64,6 +65,8 @@ macro_rules! behavior_read_tests { test_stat_special_chars, test_stat_not_cleaned_path, test_stat_not_exist, + test_stat_with_if_match, + test_stat_with_if_none_match, test_stat_root, test_read_full, test_read_full_with_special_chars, @@ -122,6 +125,63 @@ pub async fn test_stat_not_exist(op: Operator) -> Result<()> { Ok(()) } +/// Stat with if_match should succeed, else get a ConditionNotMatch error. +pub async fn test_stat_with_if_match(op: Operator) -> Result<()> { + if !op.info().capability().stat_with_if_match { + return Ok(()); + } + + let path = "normal_file"; + + let meta = op.stat(path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 262144); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_match("invalid_etag"); + + let res = op.stat_with(path, op_stat).await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_match(meta.etag().expect("etag must exist")); + + let result = op.stat_with(path, op_stat).await; + assert!(result.is_ok()); + + Ok(()) +} + +/// Stat with if_none_match should succeed, else get a ConditionNotMatch. +pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> { + if !op.info().capability().stat_with_if_none_match { + return Ok(()); + } + + let path = "normal_file"; + + let meta = op.stat(path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 262144); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist")); + + let res = op.stat_with(path, op_stat).await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_none_match("invalid_etag"); + + let res = op.stat_with(path, op_stat).await?; + assert_eq!(res.mode(), meta.mode()); + assert_eq!(res.content_length(), meta.content_length()); + + Ok(()) +} + /// Root should be able to stat and returns DIR. pub async fn test_stat_root(op: Operator) -> Result<()> { let meta = op.stat("").await?; diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs index 6a2b66dd2463..1b5a468ce3fd 100644 --- a/core/tests/behavior/write.rs +++ b/core/tests/behavior/write.rs @@ -81,6 +81,7 @@ macro_rules! behavior_write_tests { test_stat_not_cleaned_path, test_stat_not_exist, test_stat_with_if_match, + test_stat_with_if_none_match, test_stat_root, test_read_full, test_read_range, @@ -283,6 +284,42 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> { Ok(()) } +/// Stat with if_none_match should succeed, else get a ConditionNotMatch. +pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> { + if !op.info().capability().stat_with_if_none_match { + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_none_match(meta.etag().expect("etag must exist")); + + let res = op.stat_with(&path, op_stat).await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let mut op_stat = OpStat::default(); + op_stat = op_stat.with_if_none_match("invalid_etag"); + + let res = op.stat_with(&path, op_stat).await?; + assert_eq!(res.mode(), meta.mode()); + assert_eq!(res.content_length(), meta.content_length()); + + op.delete(&path).await.expect("delete must succeed"); + Ok(()) +} + /// Root should be able to stat and returns DIR. pub async fn test_stat_root(op: Operator) -> Result<()> { let meta = op.stat("").await?; From 5b35b0d530b10733056be4badec7fc0caf43071a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:43:02 -0600 Subject: [PATCH 19/35] clean up --- core/src/services/onedrive/backend.rs | 2 +- core/src/services/onedrive/builder.rs | 58 +++++++ core/src/services/onedrive/graph_model.rs | 187 ---------------------- core/src/services/onedrive/mod.rs | 1 - 4 files changed, 59 insertions(+), 189 deletions(-) delete mode 100644 core/src/services/onedrive/graph_model.rs diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 71bfd5131d65..99fbcb331d88 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -85,7 +85,7 @@ impl Accessor for OneDriveBackend { .map_err(|e| { Error::new( ErrorKind::ContentIncomplete, - "redirect location is not valid utf-8", + format!("redirect location not valid utf8: {:?}", e).as_str(), ) })?; diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 7598067da7d9..f83e396531c0 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -8,6 +8,54 @@ use crate::raw::{normalize_root, HttpClient}; use crate::Scheme; use crate::*; +/// [OneDrive](https://onedrive.com) backend support. +/// +/// # Capabilities +/// +/// This service can be used to: +/// +/// - [x] read +/// - [x] write +/// - [ ] copy +/// - [ ] rename +/// - [ ] list +/// - [ ] ~~scan~~ +/// - [ ] ~~presign~~ +/// - [ ] blocking +/// +/// # Notes +/// +/// Currently, only OneDrive Personal is supported. +/// +/// # Configuration +/// +/// - `access_token`: set the access_token for Graph API +/// - `root`: Set the work directory for backend +/// +/// You can refer to [`OneDriveBuilder`]'s docs for more information +/// +/// # Example +/// +/// ## Via Builder +/// +/// ```no_run +/// use anyhow::Result; +/// use opendal::services::OneDrive; +/// use opendal::Operator; +/// +/// #[tokio::main] +/// async fn main() -> Result<()> { +/// // create backend builder +/// let mut builder = OneDrive::default(); +/// +/// builder +/// .access_token("xxx") +/// .root("/path/to/root"); +/// +/// let op: Operator = Operator::new(builder)?.finish(); +/// Ok(()) +/// } +/// ``` #[derive(Default)] pub struct OneDriveBuilder { access_token: Option, @@ -24,16 +72,26 @@ impl Debug for OneDriveBuilder { } impl OneDriveBuilder { + /// set the bearer access token for OneDrive + /// + /// default: no access token, which leads to failure pub fn access_token(&mut self, access_token: &str) -> &mut Self { self.access_token = Some(access_token.to_string()); self } + /// Set root path of OneDrive folder. pub fn root(&mut self, root: &str) -> &mut Self { self.root = Some(root.to_string()); self } + /// Specify the http client that used by this service. + /// + /// # Notes + /// + /// This API is part of OpenDAL's Raw API. `HttpClient` could be changed + /// during minor updates. pub fn http_client(&mut self, http_client: HttpClient) -> &mut Self { self.http_client = Some(http_client); self diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs deleted file mode 100644 index 29c3e5b5a908..000000000000 --- a/core/src/services/onedrive/graph_model.rs +++ /dev/null @@ -1,187 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::Result; -use std::collections::HashMap; - -#[derive(Debug, Serialize, Deserialize)] -struct GraphApiOneDriveResponse { - #[serde(rename = "@odata.context")] - odata_context: String, - - #[serde(rename = "@odata.count")] - odata_count: usize, - - value: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -struct OneDriveItem { - #[serde(rename = "createdDateTime")] - created_date_time: String, - #[serde(rename = "eTag")] - e_tag: String, - id: String, - #[serde(rename = "lastModifiedDateTime")] - last_modified_date_time: String, - name: String, - - size: usize, - #[serde(rename = "webUrl")] - web_url: String, - #[serde(rename = "parentReference")] - parent_reference: ParentReference, - #[serde(rename = "fileSystemInfo")] - file_system_info: FileSystemInfo, - #[serde(flatten)] - item_type: ItemType, -} - -#[derive(Debug, Serialize, Deserialize)] -struct ParentReference { - #[serde(rename = "driveId")] - drive_id: String, - #[serde(rename = "driveType")] - drive_type: String, - id: String, - path: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct FileSystemInfo { - #[serde(rename = "createdDateTime")] - created_date_time: String, - #[serde[rename = "lastModifiedDateTime"]] - last_modified_date_time: String, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq)] -#[serde(untagged)] -enum ItemType { - Folder { - folder: HashMap, - #[serde(rename = "specialFolder")] - special_folder: HashMap, - }, - File { - file: HashMap, - }, -} - -fn parse_one_drive_json(data: &str) -> Result { - let response: GraphApiOneDriveResponse = serde_json::from_str(data)?; - Ok(response) -} - -#[test] -fn test_parse_one_drive_json() { - let data = r#"{ - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children", - "@odata.count": 1, - "value": [ - { - "createdDateTime": "2020-01-01T00:00:00Z", - "cTag": "cTag", - "eTag": "eTag", - "id": "id", - "lastModifiedDateTime": "2020-01-01T00:00:00Z", - "name": "name", - "size": 0, - "webUrl": "webUrl", - "reactions": { - "like": 0 - }, - "parentReference": { - "driveId": "driveId", - "driveType": "driveType", - "id": "id", - "path": "/drive/root:" - }, - "fileSystemInfo": { - "createdDateTime": "2020-01-01T00:00:00Z", - "lastModifiedDateTime": "2020-01-01T00:00:00Z" - }, - "folder": { - "childCount": 0 - }, - "specialFolder": { - "name": "name" - } - }, - { - "@microsoft.graph.downloadUrl": "https://public.ch.files.1drv.com/y4mPh7u0QjYTl5j9aZDj77EoplXNhXFzSbakI4iYoUXMaGUOSmpx1d20AnCoU9G32nj6W2qsKNfecsgfmF6O8ZE89yUYj7qnhsIvfikcJjJ0_skDA12gl2cCScQ3opoza_RcG2Lb_Pa2jyqiqgruh0TJRcC1y7mtEw89wqXx2bgjOvmo0ozTAwopTtpti9yo43Zb7nBI1efm3IwWhFKcHUUKx7WlD_8VPXPB4Xffokz61NiXoxMeq0hbwrblcznywz2AcE71SprDyCi8E7kDRjwmiTNoyfZc_FuUMZDO29WUbA", - "createdDateTime": "2018-12-30T05:32:55.46Z", - "cTag": "aYzozMjIxN0ZDMTE1NEFFQzNEITEwMi4yNTc", - "eTag": "aMzIyMTdGQzExNTRBRUMzRCExMDIuMw", - "id": "32217FC1154AEC3D!102", - "lastModifiedDateTime": "2018-12-30T05:33:23.557Z", - "name": "Getting started with OneDrive.pdf", - "size": 1025867, - "webUrl": "https://1drv.ms/b/s!AD3sShXBfyEyZg", - "reactions": { - "commentCount": 0 - }, - "createdBy": { - "user": { - "displayName": "Great Cat", - "id": "32217fc1154aec3d" - } - }, - "lastModifiedBy": { - "user": { - "displayName": "Great Cat", - "id": "32217fc1154aec3d" - } - }, - "parentReference": { - "driveId": "32217fc1154aec3d", - "driveType": "personal", - "id": "32217FC1154AEC3D!101", - "path": "/drive/root:" - }, - "file": { - "mimeType": "application/pdf", - "hashes": { - "quickXorHash": "NIfFZIvQVZH260260iUuQN5GscM=", - "sha1Hash": "E8890F3D1CE6E3FCCE46D08B188275D6CAE3292C" - } - }, - "fileSystemInfo": { - "createdDateTime": "2018-12-30T05:32:55.46Z", - "lastModifiedDateTime": "2018-12-30T05:32:55.46Z" - } - } - ] - }"#; - let response = parse_one_drive_json(data).unwrap(); - assert_eq!( - response.odata_context, - "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children" - ); - assert_eq!(response.odata_count, 1); - assert_eq!(response.value.len(), 2); - let item = &response.value[0]; - assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.e_tag, "eTag"); - assert_eq!(item.id, "id"); - assert_eq!(item.last_modified_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.name, "name"); - assert_eq!(item.size, 0); - assert_eq!(item.web_url, "webUrl"); - assert_eq!( - item.item_type, - ItemType::Folder { - folder: { - let mut map = HashMap::new(); - map.insert( - "childCount".to_string(), - serde_json::Value::Number(0.into()), - ); - map - }, - special_folder: { - let mut map = HashMap::new(); - map.insert("name".to_string(), "name".to_string()); - map - }, - } - ); -} diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 47ef1328a71a..7f7b256fdee3 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -1,7 +1,6 @@ mod backend; mod builder; mod error; -mod graph_model; pub use builder::OneDriveBuilder as OneDrive; mod writer; From 45c46955d4818f85f6424ac7377bd486b45919e4 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:43:35 -0600 Subject: [PATCH 20/35] fix --- core/src/services/onedrive/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 99fbcb331d88..b09f64dbf49f 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -171,7 +171,7 @@ impl OneDriveBackend { let url = format!( "{}{}{}", OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, - percent_encode_path(&path), + percent_encode_path(path), OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX ); From 974f30450233a9a4a811fc18c9f1cfb8575ffb8b Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:46:18 -0600 Subject: [PATCH 21/35] update --- core/src/services/onedrive/builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index f83e396531c0..b1c33814bd5a 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -26,6 +26,7 @@ use crate::*; /// # Notes /// /// Currently, only OneDrive Personal is supported. +/// For uploading, only files under 4MB are supported via the Simple Upload API (). /// /// # Configuration /// From bdd57ee798f27284d4a58b952fcac1ef4ba7d1d0 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:48:49 -0600 Subject: [PATCH 22/35] update --- core/Cargo.toml | 68 +++++++++++++++++++++------------------- core/src/services/mod.rs | 2 ++ 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 52330e0b1eff..1d33c4b6904e 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -49,6 +49,7 @@ default = [ "services-s3", "services-webdav", "services-webhdfs", + "services-onedrive", ] # Build docs or not. @@ -121,6 +122,7 @@ services-obs = [ "reqsign?/services-huaweicloud", "reqsign?/reqwest_request", ] +services-onedrive = [] services-oss = [ "dep:reqsign", "reqsign?/services-aliyun", @@ -150,73 +152,73 @@ harness = false name = "ops" [dependencies] -anyhow = { version = "1.0.30", features = ["std"] } +anyhow = {version = "1.0.30", features = ["std"]} async-compat = "0.2" -async-tls = { version = "0.11", optional = true } +async-tls = {version = "0.11", optional = true} async-trait = "0.1.68" backon = "0.4.0" base64 = "0.21" -bb8 = { version = "0.8", optional = true } +bb8 = {version = "0.8", optional = true} bytes = "1.2" chrono = "0.4.24" -dashmap = { version = "5.4", optional = true } +dashmap = {version = "5.4", optional = true} flagset = "0.4" -futures = { version = "0.3", features = ["alloc"] } -hdrs = { version = "0.2", optional = true, features = ["async_file"] } +futures = {version = "0.3", features = ["alloc"]} +hdrs = {version = "0.2", optional = true, features = ["async_file"]} http = "0.2.5" hyper = "0.14" -lazy-regex = { version = "2.5.0", optional = true } +lazy-regex = {version = "2.5.0", optional = true} log = "0.4" -madsim = { version = "0.2.21", optional = true } +madsim = {version = "0.2.21", optional = true} md-5 = "0.10" -metrics = { version = "0.20", optional = true } -minitrace = { version = "0.4.0", optional = true } -moka = { version = "0.10", optional = true, features = ["future"] } +metrics = {version = "0.20", optional = true} +minitrace = {version = "0.4.0", optional = true} +moka = {version = "0.10", optional = true, features = ["future"]} once_cell = "1" -opentelemetry = { version = "0.19.0", optional = true } +opentelemetry = {version = "0.19.0", optional = true} parking_lot = "0.12" percent-encoding = "2" pin-project = "1" -prometheus = { version = "0.13", features = ["process"], optional = true } -prost = { version = "0.11", optional = true } -quick-xml = { version = "0.27", features = ["serialize", "overlapped-lists"] } -rand = { version = "0.8", optional = true } -redis = { version = "0.22", features = [ +prometheus = {version = "0.13", features = ["process"], optional = true} +prost = {version = "0.11", optional = true} +quick-xml = {version = "0.27", features = ["serialize", "overlapped-lists"]} +rand = {version = "0.8", optional = true} +redis = {version = "0.22", features = [ "tokio-comp", "connection-manager", -], optional = true } -reqsign = { version = "0.9.1", default-features = false, optional = true } -reqwest = { version = "0.11.13", features = [ +], optional = true} +reqsign = {version = "0.9.1", default-features = false, optional = true} +reqwest = {version = "0.11.13", features = [ "multipart", "stream", -], default-features = false } -rocksdb = { version = "0.20.1", default-features = false, optional = true } -serde = { version = "1", features = ["derive"] } +], default-features = false} +rocksdb = {version = "0.20.1", default-features = false, optional = true} +serde = {version = "1", features = ["derive"]} serde_json = "1" -sled = { version = "0.34.7", optional = true } -suppaftp = { version = "4.5", default-features = false, features = [ +sled = {version = "0.34.7", optional = true} +suppaftp = {version = "4.5", default-features = false, features = [ "async-secure", "async-rustls", -], optional = true } +], optional = true} tokio = "1.27" -tracing = { version = "0.1", optional = true } -uuid = { version = "1", features = ["serde", "v4"] } +tracing = {version = "0.1", optional = true} +uuid = {version = "1", features = ["serde", "v4"]} [dev-dependencies] cfg-if = "1" -criterion = { version = "0.4", features = ["async", "async_tokio"] } +criterion = {version = "0.4", features = ["async", "async_tokio"]} dotenvy = "0.15" env_logger = "0.10" -opentelemetry = { version = "0.19", default-features = false, features = [ +opentelemetry = {version = "0.19", default-features = false, features = [ "trace", -] } +]} opentelemetry-jaeger = "0.18" paste = "1" pretty_assertions = "1" rand = "0.8" sha2 = "0.10" size = "0.4" -tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] } +tokio = {version = "1.27", features = ["fs", "macros", "rt-multi-thread"]} tracing-opentelemetry = "0.17" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = {version = "0.3", features = ["env-filter"]} wiremock = "0.5" diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index c3222bff0b57..af8d25deb993 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -132,7 +132,9 @@ pub use webdav::Webdav; #[cfg(feature = "services-webhdfs")] mod webhdfs; +#[cfg(feature = "services-onedrive")] mod onedrive; +#[cfg(feature = "services-onedrive")] pub use onedrive::OneDrive; #[cfg(feature = "services-webhdfs")] From 229d7c04e887ce07831a75c31820f18410cc030b Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:50:13 -0600 Subject: [PATCH 23/35] onedrive read and write --- core/src/services/mod.rs | 2 +- core/src/services/onedrive/backend.rs | 18 +++++++++--------- core/src/services/onedrive/builder.rs | 18 +++++++++--------- core/src/services/onedrive/mod.rs | 2 +- core/src/services/onedrive/writer.rs | 6 +++--- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/core/src/services/mod.rs b/core/src/services/mod.rs index af8d25deb993..4a34f8b06788 100644 --- a/core/src/services/mod.rs +++ b/core/src/services/mod.rs @@ -135,7 +135,7 @@ mod webhdfs; #[cfg(feature = "services-onedrive")] mod onedrive; #[cfg(feature = "services-onedrive")] -pub use onedrive::OneDrive; +pub use onedrive::Onedrive; #[cfg(feature = "services-webhdfs")] pub use webhdfs::Webhdfs; diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index b09f64dbf49f..20ad9954b50c 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -15,13 +15,13 @@ use crate::{ use super::{error::parse_error, writer::OneDriveWriter}; #[derive(Clone)] -pub struct OneDriveBackend { +pub struct OnedriveBackend { root: String, access_token: String, client: HttpClient, } -impl OneDriveBackend { +impl OnedriveBackend { pub(crate) fn new(root: String, access_token: String, http_client: HttpClient) -> Self { Self { root, @@ -31,7 +31,7 @@ impl OneDriveBackend { } } -impl Debug for OneDriveBackend { +impl Debug for OnedriveBackend { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut de = f.debug_struct("OneDriveBackend"); de.field("root", &self.root); @@ -41,7 +41,7 @@ impl Debug for OneDriveBackend { } #[async_trait] -impl Accessor for OneDriveBackend { +impl Accessor for OnedriveBackend { type Reader = IncomingAsyncBody; type BlockingReader = (); type Writer = OneDriveWriter; @@ -121,7 +121,7 @@ impl Accessor for OneDriveBackend { } } -impl OneDriveBackend { +impl OnedriveBackend { const ONEDRIVE_ENDPOINT_PREFIX: &'static str = "https://graph.microsoft.com/v1.0/me/drive/root:"; const ONEDRIVE_ENDPOINT_SUFFIX: &'static str = ":/content"; @@ -131,9 +131,9 @@ impl OneDriveBackend { let url: String = format!( "{}{}{}", - OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + OnedriveBackend::ONEDRIVE_ENDPOINT_PREFIX, percent_encode_path(&path), - OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + OnedriveBackend::ONEDRIVE_ENDPOINT_SUFFIX ); let mut req = Request::get(&url); @@ -170,9 +170,9 @@ impl OneDriveBackend { ) -> Result> { let url = format!( "{}{}{}", - OneDriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + OnedriveBackend::ONEDRIVE_ENDPOINT_PREFIX, percent_encode_path(path), - OneDriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + OnedriveBackend::ONEDRIVE_ENDPOINT_SUFFIX ); let mut req = Request::put(&url); diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index b1c33814bd5a..7a610b48a498 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -3,7 +3,7 @@ use std::fmt::{Debug, Formatter}; use log::debug; -use super::backend::OneDriveBackend; +use super::backend::OnedriveBackend; use crate::raw::{normalize_root, HttpClient}; use crate::Scheme; use crate::*; @@ -41,13 +41,13 @@ use crate::*; /// /// ```no_run /// use anyhow::Result; -/// use opendal::services::OneDrive; +/// use opendal::services::Onedrive; /// use opendal::Operator; /// /// #[tokio::main] /// async fn main() -> Result<()> { /// // create backend builder -/// let mut builder = OneDrive::default(); +/// let mut builder = Onedrive::default(); /// /// builder /// .access_token("xxx") @@ -58,13 +58,13 @@ use crate::*; /// } /// ``` #[derive(Default)] -pub struct OneDriveBuilder { +pub struct OnedriveBuilder { access_token: Option, root: Option, http_client: Option, } -impl Debug for OneDriveBuilder { +impl Debug for OnedriveBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let mut de = f.debug_struct("Builder"); de.field("endpoint", &self.access_token); @@ -72,7 +72,7 @@ impl Debug for OneDriveBuilder { } } -impl OneDriveBuilder { +impl OnedriveBuilder { /// set the bearer access token for OneDrive /// /// default: no access token, which leads to failure @@ -99,7 +99,7 @@ impl OneDriveBuilder { } } -impl Builder for OneDriveBuilder { +impl Builder for OnedriveBuilder { fn build(&mut self) -> Result { let root = normalize_root(&self.root.take().unwrap_or_default()); debug!("backend use root {}", root); @@ -114,14 +114,14 @@ impl Builder for OneDriveBuilder { }; match self.access_token.clone() { - Some(access_token) => Ok(OneDriveBackend::new(root, access_token, client)), + Some(access_token) => Ok(OnedriveBackend::new(root, access_token, client)), None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")), } } const SCHEME: Scheme = Scheme::Onedrive; - type Accessor = OneDriveBackend; + type Accessor = OnedriveBackend; fn from_map(map: HashMap) -> Self { let mut builder = Self::default(); diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 7f7b256fdee3..8afbcdf9ac52 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -2,5 +2,5 @@ mod backend; mod builder; mod error; -pub use builder::OneDriveBuilder as OneDrive; +pub use builder::OnedriveBuilder as Onedrive; mod writer; diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs index abc33755e1f5..b450e82dcb03 100644 --- a/core/src/services/onedrive/writer.rs +++ b/core/src/services/onedrive/writer.rs @@ -19,21 +19,21 @@ use async_trait::async_trait; use bytes::Bytes; use http::StatusCode; -use super::backend::OneDriveBackend; +use super::backend::OnedriveBackend; use super::error::parse_error; use crate::ops::OpWrite; use crate::raw::*; use crate::*; pub struct OneDriveWriter { - backend: OneDriveBackend, + backend: OnedriveBackend, op: OpWrite, path: String, } impl OneDriveWriter { - pub fn new(backend: OneDriveBackend, op: OpWrite, path: String) -> Self { + pub fn new(backend: OnedriveBackend, op: OpWrite, path: String) -> Self { OneDriveWriter { backend, op, path } } } From b2b2f6e0169d0a2449bbca7e6fd09eb303885e8a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:54:48 -0600 Subject: [PATCH 24/35] update license header --- core/src/services/onedrive/backend.rs | 17 +++++++++++++++++ core/src/services/onedrive/builder.rs | 17 +++++++++++++++++ core/src/services/onedrive/mod.rs | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 20ad9954b50c..d9102fb3b16f 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use async_trait::async_trait; use http::{header, Request, Response, StatusCode}; use std::fmt::Debug; diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 7a610b48a498..83523ab398c2 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + use std::collections::HashMap; use std::fmt::{Debug, Formatter}; diff --git a/core/src/services/onedrive/mod.rs b/core/src/services/onedrive/mod.rs index 8afbcdf9ac52..0b58fc8cc8f3 100644 --- a/core/src/services/onedrive/mod.rs +++ b/core/src/services/onedrive/mod.rs @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + mod backend; mod builder; mod error; From 1cf31bd2ab0d2f5006d42b6294171509af80bfe3 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:56:25 -0600 Subject: [PATCH 25/35] update --- core/Cargo.toml | 67 ++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 1d33c4b6904e..34eb05c46301 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -122,7 +122,6 @@ services-obs = [ "reqsign?/services-huaweicloud", "reqsign?/reqwest_request", ] -services-onedrive = [] services-oss = [ "dep:reqsign", "reqsign?/services-aliyun", @@ -152,73 +151,73 @@ harness = false name = "ops" [dependencies] -anyhow = {version = "1.0.30", features = ["std"]} +anyhow = { version = "1.0.30", features = ["std"] } async-compat = "0.2" -async-tls = {version = "0.11", optional = true} +async-tls = { version = "0.11", optional = true } async-trait = "0.1.68" backon = "0.4.0" base64 = "0.21" -bb8 = {version = "0.8", optional = true} +bb8 = { version = "0.8", optional = true } bytes = "1.2" chrono = "0.4.24" -dashmap = {version = "5.4", optional = true} +dashmap = { version = "5.4", optional = true } flagset = "0.4" -futures = {version = "0.3", features = ["alloc"]} -hdrs = {version = "0.2", optional = true, features = ["async_file"]} +futures = { version = "0.3", features = ["alloc"] } +hdrs = { version = "0.2", optional = true, features = ["async_file"] } http = "0.2.5" hyper = "0.14" -lazy-regex = {version = "2.5.0", optional = true} +lazy-regex = { version = "2.5.0", optional = true } log = "0.4" -madsim = {version = "0.2.21", optional = true} +madsim = { version = "0.2.21", optional = true } md-5 = "0.10" -metrics = {version = "0.20", optional = true} -minitrace = {version = "0.4.0", optional = true} -moka = {version = "0.10", optional = true, features = ["future"]} +metrics = { version = "0.20", optional = true } +minitrace = { version = "0.4.0", optional = true } +moka = { version = "0.10", optional = true, features = ["future"] } once_cell = "1" -opentelemetry = {version = "0.19.0", optional = true} +opentelemetry = { version = "0.19.0", optional = true } parking_lot = "0.12" percent-encoding = "2" pin-project = "1" -prometheus = {version = "0.13", features = ["process"], optional = true} -prost = {version = "0.11", optional = true} -quick-xml = {version = "0.27", features = ["serialize", "overlapped-lists"]} -rand = {version = "0.8", optional = true} -redis = {version = "0.22", features = [ +prometheus = { version = "0.13", features = ["process"], optional = true } +prost = { version = "0.11", optional = true } +quick-xml = { version = "0.27", features = ["serialize", "overlapped-lists"] } +rand = { version = "0.8", optional = true } +redis = { version = "0.22", features = [ "tokio-comp", "connection-manager", -], optional = true} -reqsign = {version = "0.9.1", default-features = false, optional = true} -reqwest = {version = "0.11.13", features = [ +], optional = true } +reqsign = { version = "0.9.1", default-features = false, optional = true } +reqwest = { version = "0.11.13", features = [ "multipart", "stream", -], default-features = false} -rocksdb = {version = "0.20.1", default-features = false, optional = true} -serde = {version = "1", features = ["derive"]} +], default-features = false } +rocksdb = { version = "0.20.1", default-features = false, optional = true } +serde = { version = "1", features = ["derive"] } serde_json = "1" -sled = {version = "0.34.7", optional = true} -suppaftp = {version = "4.5", default-features = false, features = [ +sled = { version = "0.34.7", optional = true } +suppaftp = { version = "4.5", default-features = false, features = [ "async-secure", "async-rustls", -], optional = true} +], optional = true } tokio = "1.27" -tracing = {version = "0.1", optional = true} -uuid = {version = "1", features = ["serde", "v4"]} +tracing = { version = "0.1", optional = true } +uuid = { version = "1", features = ["serde", "v4"] } [dev-dependencies] cfg-if = "1" -criterion = {version = "0.4", features = ["async", "async_tokio"]} +criterion = { version = "0.4", features = ["async", "async_tokio"] } dotenvy = "0.15" env_logger = "0.10" -opentelemetry = {version = "0.19", default-features = false, features = [ +opentelemetry = { version = "0.19", default-features = false, features = [ "trace", -]} +] } opentelemetry-jaeger = "0.18" paste = "1" pretty_assertions = "1" rand = "0.8" sha2 = "0.10" size = "0.4" -tokio = {version = "1.27", features = ["fs", "macros", "rt-multi-thread"]} +tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] } tracing-opentelemetry = "0.17" -tracing-subscriber = {version = "0.3", features = ["env-filter"]} +tracing-subscriber = { version = "0.3", features = ["env-filter"] } wiremock = "0.5" From d06e113c72bd57acf44417516e4478aa2a56bb08 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 22:57:10 -0600 Subject: [PATCH 26/35] fix formatting --- core/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/core/Cargo.toml b/core/Cargo.toml index 34eb05c46301..5c1feb5cfbf6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -122,6 +122,7 @@ services-obs = [ "reqsign?/services-huaweicloud", "reqsign?/reqwest_request", ] +services-onedrive = [] services-oss = [ "dep:reqsign", "reqsign?/services-aliyun", From 0bd8d24b4257bbfb42b3cc72be1180b4f325b4b6 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 25 Apr 2023 23:06:20 -0600 Subject: [PATCH 27/35] revert --- core/src/services/webdav/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 2e9bd7e7afbd..4a073d6e9fea 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -433,7 +433,7 @@ impl WebdavBackend { ) -> Result> { let p = build_rooted_abs_path(&self.root, path); - let url: String = format!("{}{}", self.endpoint, percent_encode_path(&p)); + let url = format!("{}{}", self.endpoint, percent_encode_path(&p)); let mut req = Request::get(&url); From d91e07433df5c0a36b0519756f06009e17714b83 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 16:41:32 -0600 Subject: [PATCH 28/35] 201 --- core/src/services/onedrive/writer.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/services/onedrive/writer.rs b/core/src/services/onedrive/writer.rs index b450e82dcb03..4ac6062a4c75 100644 --- a/core/src/services/onedrive/writer.rs +++ b/core/src/services/onedrive/writer.rs @@ -54,7 +54,9 @@ impl oio::Write for OneDriveWriter { let status = resp.status(); match status { - StatusCode::CREATED | StatusCode::OK | StatusCode::NO_CONTENT => { + // Typical response code: 201 Created + // Reference: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online#response + StatusCode::CREATED => { resp.into_body().consume().await?; Ok(()) } From 2ac89077a7e79ca920f4c106f3c19f7c30f0d398 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 16:43:45 -0600 Subject: [PATCH 29/35] clean up --- core/src/services/onedrive/builder.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 83523ab398c2..8c29d4cfe227 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -117,6 +117,19 @@ impl OnedriveBuilder { } impl Builder for OnedriveBuilder { + const SCHEME: Scheme = Scheme::Onedrive; + + type Accessor = OnedriveBackend; + + fn from_map(map: HashMap) -> Self { + let mut builder = Self::default(); + + map.get("root").map(|v| builder.root(v)); + map.get("access_token").map(|v| builder.access_token(v)); + + builder + } + fn build(&mut self) -> Result { let root = normalize_root(&self.root.take().unwrap_or_default()); debug!("backend use root {}", root); @@ -135,17 +148,4 @@ impl Builder for OnedriveBuilder { None => Err(Error::new(ErrorKind::ConfigInvalid, "access_token not set")), } } - - const SCHEME: Scheme = Scheme::Onedrive; - - type Accessor = OnedriveBackend; - - fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("access_token").map(|v| builder.access_token(v)); - - builder - } } From 1e547fe413800e865be8bac44ab0d191691c4a12 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 16:48:21 -0600 Subject: [PATCH 30/35] use parse_location --- core/src/services/onedrive/backend.rs | 37 ++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index d9102fb3b16f..1086946384ca 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -22,8 +22,9 @@ use std::fmt::Debug; use crate::{ ops::{OpRead, OpWrite}, raw::{ - build_rooted_abs_path, new_request_build_error, parse_into_metadata, percent_encode_path, - Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, RpRead, RpWrite, + build_rooted_abs_path, new_request_build_error, parse_into_metadata, parse_location, + percent_encode_path, Accessor, AccessorInfo, AsyncBody, HttpClient, IncomingAsyncBody, + RpRead, RpWrite, }, types::Result, Capability, Error, ErrorKind, @@ -89,26 +90,22 @@ impl Accessor for OnedriveBackend { let status = resp.status(); if status.is_redirection() { - let location = resp - .headers() - .get(header::LOCATION) - .ok_or_else(|| { - Error::new( + let headers = resp.headers(); + let location = parse_location(headers)?; + + match location { + None => { + return Err(Error::new( ErrorKind::ContentIncomplete, "redirect location not found in response", - ) - })? - .to_str() - .map_err(|e| { - Error::new( - ErrorKind::ContentIncomplete, - format!("redirect location not valid utf8: {:?}", e).as_str(), - ) - })?; - - let resp = self.onedrive_get_redirection(location).await?; - let meta = parse_into_metadata(path, resp.headers())?; - Ok((RpRead::with_metadata(meta), resp.into_body())) + )); + } + Some(location) => { + let resp = self.onedrive_get_redirection(location).await?; + let meta = parse_into_metadata(path, resp.headers())?; + Ok((RpRead::with_metadata(meta), resp.into_body())) + } + } } else { match status { StatusCode::OK | StatusCode::PARTIAL_CONTENT => { From bc6f40951a3851a72c2905e992a438761a84b1b0 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 20:03:16 -0600 Subject: [PATCH 31/35] trigger ci From 0d8e16cd7407d8d59bbffd85c44c94e5ab9c2ae0 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 20:05:27 -0600 Subject: [PATCH 32/35] update --- core/src/services/onedrive/graph_model.rs | 187 ---------------------- 1 file changed, 187 deletions(-) delete mode 100644 core/src/services/onedrive/graph_model.rs diff --git a/core/src/services/onedrive/graph_model.rs b/core/src/services/onedrive/graph_model.rs deleted file mode 100644 index 29c3e5b5a908..000000000000 --- a/core/src/services/onedrive/graph_model.rs +++ /dev/null @@ -1,187 +0,0 @@ -use serde::{Deserialize, Serialize}; -use serde_json::Result; -use std::collections::HashMap; - -#[derive(Debug, Serialize, Deserialize)] -struct GraphApiOneDriveResponse { - #[serde(rename = "@odata.context")] - odata_context: String, - - #[serde(rename = "@odata.count")] - odata_count: usize, - - value: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -struct OneDriveItem { - #[serde(rename = "createdDateTime")] - created_date_time: String, - #[serde(rename = "eTag")] - e_tag: String, - id: String, - #[serde(rename = "lastModifiedDateTime")] - last_modified_date_time: String, - name: String, - - size: usize, - #[serde(rename = "webUrl")] - web_url: String, - #[serde(rename = "parentReference")] - parent_reference: ParentReference, - #[serde(rename = "fileSystemInfo")] - file_system_info: FileSystemInfo, - #[serde(flatten)] - item_type: ItemType, -} - -#[derive(Debug, Serialize, Deserialize)] -struct ParentReference { - #[serde(rename = "driveId")] - drive_id: String, - #[serde(rename = "driveType")] - drive_type: String, - id: String, - path: String, -} - -#[derive(Debug, Serialize, Deserialize)] -struct FileSystemInfo { - #[serde(rename = "createdDateTime")] - created_date_time: String, - #[serde[rename = "lastModifiedDateTime"]] - last_modified_date_time: String, -} - -#[derive(Debug, Serialize, Deserialize, PartialEq)] -#[serde(untagged)] -enum ItemType { - Folder { - folder: HashMap, - #[serde(rename = "specialFolder")] - special_folder: HashMap, - }, - File { - file: HashMap, - }, -} - -fn parse_one_drive_json(data: &str) -> Result { - let response: GraphApiOneDriveResponse = serde_json::from_str(data)?; - Ok(response) -} - -#[test] -fn test_parse_one_drive_json() { - let data = r#"{ - "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children", - "@odata.count": 1, - "value": [ - { - "createdDateTime": "2020-01-01T00:00:00Z", - "cTag": "cTag", - "eTag": "eTag", - "id": "id", - "lastModifiedDateTime": "2020-01-01T00:00:00Z", - "name": "name", - "size": 0, - "webUrl": "webUrl", - "reactions": { - "like": 0 - }, - "parentReference": { - "driveId": "driveId", - "driveType": "driveType", - "id": "id", - "path": "/drive/root:" - }, - "fileSystemInfo": { - "createdDateTime": "2020-01-01T00:00:00Z", - "lastModifiedDateTime": "2020-01-01T00:00:00Z" - }, - "folder": { - "childCount": 0 - }, - "specialFolder": { - "name": "name" - } - }, - { - "@microsoft.graph.downloadUrl": "https://public.ch.files.1drv.com/y4mPh7u0QjYTl5j9aZDj77EoplXNhXFzSbakI4iYoUXMaGUOSmpx1d20AnCoU9G32nj6W2qsKNfecsgfmF6O8ZE89yUYj7qnhsIvfikcJjJ0_skDA12gl2cCScQ3opoza_RcG2Lb_Pa2jyqiqgruh0TJRcC1y7mtEw89wqXx2bgjOvmo0ozTAwopTtpti9yo43Zb7nBI1efm3IwWhFKcHUUKx7WlD_8VPXPB4Xffokz61NiXoxMeq0hbwrblcznywz2AcE71SprDyCi8E7kDRjwmiTNoyfZc_FuUMZDO29WUbA", - "createdDateTime": "2018-12-30T05:32:55.46Z", - "cTag": "aYzozMjIxN0ZDMTE1NEFFQzNEITEwMi4yNTc", - "eTag": "aMzIyMTdGQzExNTRBRUMzRCExMDIuMw", - "id": "32217FC1154AEC3D!102", - "lastModifiedDateTime": "2018-12-30T05:33:23.557Z", - "name": "Getting started with OneDrive.pdf", - "size": 1025867, - "webUrl": "https://1drv.ms/b/s!AD3sShXBfyEyZg", - "reactions": { - "commentCount": 0 - }, - "createdBy": { - "user": { - "displayName": "Great Cat", - "id": "32217fc1154aec3d" - } - }, - "lastModifiedBy": { - "user": { - "displayName": "Great Cat", - "id": "32217fc1154aec3d" - } - }, - "parentReference": { - "driveId": "32217fc1154aec3d", - "driveType": "personal", - "id": "32217FC1154AEC3D!101", - "path": "/drive/root:" - }, - "file": { - "mimeType": "application/pdf", - "hashes": { - "quickXorHash": "NIfFZIvQVZH260260iUuQN5GscM=", - "sha1Hash": "E8890F3D1CE6E3FCCE46D08B188275D6CAE3292C" - } - }, - "fileSystemInfo": { - "createdDateTime": "2018-12-30T05:32:55.46Z", - "lastModifiedDateTime": "2018-12-30T05:32:55.46Z" - } - } - ] - }"#; - let response = parse_one_drive_json(data).unwrap(); - assert_eq!( - response.odata_context, - "https://graph.microsoft.com/v1.0/$metadata#users('user_id')/drive/root/children" - ); - assert_eq!(response.odata_count, 1); - assert_eq!(response.value.len(), 2); - let item = &response.value[0]; - assert_eq!(item.created_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.e_tag, "eTag"); - assert_eq!(item.id, "id"); - assert_eq!(item.last_modified_date_time, "2020-01-01T00:00:00Z"); - assert_eq!(item.name, "name"); - assert_eq!(item.size, 0); - assert_eq!(item.web_url, "webUrl"); - assert_eq!( - item.item_type, - ItemType::Folder { - folder: { - let mut map = HashMap::new(); - map.insert( - "childCount".to_string(), - serde_json::Value::Number(0.into()), - ); - map - }, - special_folder: { - let mut map = HashMap::new(); - map.insert("name".to_string(), "name".to_string()); - map - }, - } - ); -} From 5a793e09e94130bee528f53d81b19b0e048c0a87 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 20:18:38 -0600 Subject: [PATCH 33/35] fix debug fmt --- core/src/services/onedrive/builder.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 8c29d4cfe227..5bbbdd0013b0 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -83,9 +83,10 @@ pub struct OnedriveBuilder { impl Debug for OnedriveBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut de = f.debug_struct("Builder"); - de.field("endpoint", &self.access_token); - de.finish() + f.debug_struct("Backend") + .field("access_token", &self.access_token) + .field("root", &self.root) + .finish() } } From 16d70d3af76dde8b41ee47b09064ea40d50f7090 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 21:20:06 -0600 Subject: [PATCH 34/35] remove access token --- core/src/services/onedrive/builder.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 5bbbdd0013b0..b1d020aac9eb 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -83,10 +83,7 @@ pub struct OnedriveBuilder { impl Debug for OnedriveBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Backend") - .field("access_token", &self.access_token) - .field("root", &self.root) - .finish() + f.debug_struct("Backend").field("root", &self.root).finish() } } From 0cc42cf118c74cad63f04fc6ed32ee306a7c0de2 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 27 Apr 2023 21:21:41 -0600 Subject: [PATCH 35/35] update --- core/src/services/onedrive/backend.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/core/src/services/onedrive/backend.rs b/core/src/services/onedrive/backend.rs index 1086946384ca..ce15965e44ff 100644 --- a/core/src/services/onedrive/backend.rs +++ b/core/src/services/onedrive/backend.rs @@ -136,18 +136,11 @@ impl Accessor for OnedriveBackend { } impl OnedriveBackend { - const ONEDRIVE_ENDPOINT_PREFIX: &'static str = - "https://graph.microsoft.com/v1.0/me/drive/root:"; - const ONEDRIVE_ENDPOINT_SUFFIX: &'static str = ":/content"; - async fn onedrive_get(&self, path: &str) -> Result> { let path = build_rooted_abs_path(&self.root, path); - let url: String = format!( - "{}{}{}", - OnedriveBackend::ONEDRIVE_ENDPOINT_PREFIX, + "https://graph.microsoft.com/v1.0/me/drive/root:{}:/content", percent_encode_path(&path), - OnedriveBackend::ONEDRIVE_ENDPOINT_SUFFIX ); let mut req = Request::get(&url); @@ -183,10 +176,8 @@ impl OnedriveBackend { body: AsyncBody, ) -> Result> { let url = format!( - "{}{}{}", - OnedriveBackend::ONEDRIVE_ENDPOINT_PREFIX, - percent_encode_path(path), - OnedriveBackend::ONEDRIVE_ENDPOINT_SUFFIX + "https://graph.microsoft.com/v1.0/me/drive/root:{}:/content", + percent_encode_path(path) ); let mut req = Request::put(&url);