From 94e9a9f79c33f8cb03440fcedfc89f9d07b6672d Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 4 Jan 2024 14:21:02 +0800 Subject: [PATCH 1/2] fix(core): Handling content encoding correctly Signed-off-by: Xuanwo --- core/src/raw/http_util/client.rs | 9 ++++++--- core/src/raw/http_util/header.rs | 7 ++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs index 3bd7e826cd09..94cbd1dd3901 100644 --- a/core/src/raw/http_util/client.rs +++ b/core/src/raw/http_util/client.rs @@ -28,6 +28,7 @@ use http::Response; use super::body::IncomingAsyncBody; use super::parse_content_length; use super::AsyncBody; +use crate::raw::http_util::header::parse_content_encoding; use crate::raw::*; use crate::Error; use crate::ErrorKind; @@ -167,11 +168,13 @@ impl HttpClient { })?; // Get content length from header so that we can check it. - // If the request method is HEAD, we will ignore this. - let content_length = if is_head { + // + // - If the request method is HEAD, we will ignore content length. + // - If response contains content_encoding, we should omit it's content length. + let content_length = if is_head || parse_content_encoding(resp.headers())?.is_some() { None } else { - parse_content_length(resp.headers()).expect("response content length must be valid") + parse_content_length(resp.headers())? }; let mut hr = Response::builder() diff --git a/core/src/raw/http_util/header.rs b/core/src/raw/http_util/header.rs index f266f7c0c46d..ec28113a70dd 100644 --- a/core/src/raw/http_util/header.rs +++ b/core/src/raw/http_util/header.rs @@ -19,7 +19,6 @@ use base64::engine::general_purpose; use base64::Engine; use chrono::DateTime; use chrono::Utc; -use http::header::CACHE_CONTROL; use http::header::CONTENT_DISPOSITION; use http::header::CONTENT_LENGTH; use http::header::CONTENT_RANGE; @@ -27,6 +26,7 @@ use http::header::CONTENT_TYPE; use http::header::ETAG; use http::header::LAST_MODIFIED; use http::header::LOCATION; +use http::header::{CACHE_CONTROL, CONTENT_ENCODING}; use http::HeaderValue; use http::{HeaderMap, HeaderName}; use md5::Digest; @@ -77,6 +77,11 @@ pub fn parse_content_type(headers: &HeaderMap) -> Result> { parse_header_to_str(headers, CONTENT_TYPE) } +/// Parse content encoding from header map. +pub fn parse_content_encoding(headers: &HeaderMap) -> Result> { + parse_header_to_str(headers, CONTENT_ENCODING) +} + /// Parse content range from header map. pub fn parse_content_range(headers: &HeaderMap) -> Result> { parse_header_to_str(headers, CONTENT_RANGE)? From 1365dbb817c68fc5307ae3625f1d1ca29fedb962 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 4 Jan 2024 14:24:05 +0800 Subject: [PATCH 2/2] polish Signed-off-by: Xuanwo --- core/src/raw/http_util/client.rs | 2 +- core/src/raw/http_util/mod.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs index 94cbd1dd3901..ea1e9ea0b8b9 100644 --- a/core/src/raw/http_util/client.rs +++ b/core/src/raw/http_util/client.rs @@ -26,9 +26,9 @@ use http::Request; use http::Response; use super::body::IncomingAsyncBody; +use super::parse_content_encoding; use super::parse_content_length; use super::AsyncBody; -use crate::raw::http_util::header::parse_content_encoding; use crate::raw::*; use crate::Error; use crate::ErrorKind; diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs index 8dd9a58eead8..5a5f375e8dba 100644 --- a/core/src/raw/http_util/mod.rs +++ b/core/src/raw/http_util/mod.rs @@ -35,6 +35,7 @@ pub use header::format_authorization_by_basic; pub use header::format_authorization_by_bearer; pub use header::format_content_md5; pub use header::parse_content_disposition; +pub use header::parse_content_encoding; pub use header::parse_content_length; pub use header::parse_content_md5; pub use header::parse_content_range;