diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 8d2742aaafe5b..5df97aacca10f 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -112,19 +112,23 @@ use crate::{DataFusionError, Result}; macro_rules! config_namespace { ( $(#[doc = $struct_d:tt])* + $(#[allow($($struct_depr:tt)*)])? $vis:vis struct $struct_name:ident { $( $(#[doc = $d:tt])* + $(#[allow($($field_depr:tt)*)])? $field_vis:vis $field_name:ident : $field_type:ty, $(warn = $warn: expr,)? $(transform = $transform:expr,)? default = $default:expr )*$(,)* } ) => { $(#[doc = $struct_d])* + $(#[allow($($struct_depr)*)])? #[derive(Debug, Clone, PartialEq)] $vis struct $struct_name{ $( $(#[doc = $d])* + $(#[allow($($field_depr)*)])? $field_vis $field_name : $field_type, )* } @@ -378,6 +382,7 @@ config_namespace! { /// See also: [`SessionConfig`] /// /// [`SessionConfig`]: https://docs.rs/datafusion/latest/datafusion/prelude/struct.SessionConfig.html + // #[derive(Debug)] pub struct ParquetOptions { // The following options affect reading parquet files @@ -467,6 +472,7 @@ config_namespace! { /// (writing) Sets max statistics size for any column. If NULL, uses /// default parquet writer setting + #[allow(deprecated)] pub max_statistics_size: Option, default = Some(4096) /// (writing) Target maximum number of rows in each row group (defaults to 1M @@ -1598,19 +1604,23 @@ impl ConfigField for TableParquetOptions { macro_rules! config_namespace_with_hashmap { ( $(#[doc = $struct_d:tt])* + $(#[allow($($struct_depr:tt)*)])? $vis:vis struct $struct_name:ident { $( $(#[doc = $d:tt])* + $(#[allow($($field_depr:tt)*)])? $field_vis:vis $field_name:ident : $field_type:ty, $(transform = $transform:expr,)? default = $default:expr )*$(,)* } ) => { $(#[doc = $struct_d])* + $(#[allow($($struct_depr)*)])? #[derive(Debug, Clone, PartialEq)] $vis struct $struct_name{ $( $(#[doc = $d])* + $(#[allow($($field_depr)*)])? $field_vis $field_name : $field_type, )* } @@ -1669,6 +1679,7 @@ macro_rules! config_namespace_with_hashmap { $( let key = format!("{}.{field}::{}", key_prefix, column_name, field = stringify!($field_name)); let desc = concat!($($d),*).trim(); + #[allow(deprecated)] col_options.$field_name.visit(v, key.as_str(), desc); )* } @@ -1720,6 +1731,7 @@ config_namespace_with_hashmap! { /// Sets max statistics size for the column path. If NULL, uses /// default parquet options + #[allow(deprecated)] pub max_statistics_size: Option, default = None } } diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs index 46bce06470f38..b080ec3d22bc9 100644 --- a/datafusion/common/src/file_options/parquet_writer.rs +++ b/datafusion/common/src/file_options/parquet_writer.rs @@ -156,9 +156,13 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder { builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv); } + //max_statistics_size is deprecated as per latest arrow version. + #[allow(deprecated)] if let Some(max_statistics_size) = options.max_statistics_size { - builder = - builder.set_column_max_statistics_size(path, max_statistics_size); + builder = { + #[allow(deprecated)] + builder.set_column_max_statistics_size(path, max_statistics_size) + } } } @@ -207,6 +211,8 @@ impl ParquetOptions { dictionary_enabled, dictionary_page_size_limit, statistics_enabled, + + #[allow(deprecated)] max_statistics_size, max_row_group_size, created_by,