Store tags in compressed segments instead of in the metadata Delta Lake

README.md

@@ -16,7 +16,7 @@ ModelarDB is designed to be cross-platform and is currently automatically tested
 through [GitHub Actions](https://github.com/ModelarData/ModelarDB-RS/actions). It is also known to work on FreeBSD which
 is [currently not supported by GitHub Actions](https://github.com/actions/runner/issues/385). It is implemented in
 [Rust](https://www.rust-lang.org/) and uses [Apache Arrow Flight](https://github.com/apache/arrow-rs/tree/master/arrow-flight)
-for communicating with clients, [Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion) as its query
+for communicating with clients, [Apache DataFusion](https://github.com/apache/datafusion) as its query
 engine, [Apache Arrow](https://github.com/apache/arrow-rs) as its in-memory data format, and
 [Apache Parquet](https://github.com/apache/arrow-rs/tree/master/parquet) as its on-disk data format.
 

crates/modelardb_client/src/error.rs

@@ -23,8 +23,8 @@ use std::result::Result as StdResult;
 use arrow::error::ArrowError;
 use object_store::Error as ObjectStoreError;
 use rustyline::error::ReadlineError as RustyLineError;
-use tonic::transport::Error as TonicTransportError;
 use tonic::Status as TonicStatusError;
+use tonic::transport::Error as TonicTransportError;
 
 /// Result type used throughout `modelardb_client`.
 pub type Result<T> = StdResult<T, ModelarDbClientError>;

crates/modelardb_client/src/helper.rs

@@ -19,13 +19,13 @@
 
 use std::result::Result;
 
+use rustyline::Context;
+use rustyline::Helper;
 use rustyline::completion::{self, Completer};
 use rustyline::error::ReadlineError;
 use rustyline::highlight::Highlighter;
 use rustyline::hint::Hinter;
 use rustyline::validate::Validator;
-use rustyline::Context;
-use rustyline::Helper;
 
 /// Provides tab-completion for the client's read-eval-print loop.
 pub struct ClientHelper {

crates/modelardb_client/src/main.rs

@@ -27,17 +27,17 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use arrow::array::ArrayRef;
-use arrow::datatypes::{Schema, SchemaRef, ToByteSlice};
+use arrow::datatypes::{Schema, ToByteSlice};
 use arrow::ipc::convert;
 use arrow::util::pretty;
 use arrow_flight::flight_service_client::FlightServiceClient;
-use arrow_flight::{utils, Action, Criteria, FlightData, FlightDescriptor, Ticket};
+use arrow_flight::{Action, Criteria, FlightData, FlightDescriptor, Ticket, utils};
 use bytes::Bytes;
 use object_store::local::LocalFileSystem;
 use object_store::path::Path;
 use object_store::{ObjectMeta, ObjectStore};
-use rustyline::history::FileHistory;
 use rustyline::Editor;
+use rustyline::history::FileHistory;
 use tonic::transport::Channel;
 use tonic::{Request, Streaming};
 
@@ -387,7 +387,7 @@ async fn execute_query_and_print_result(
 /// Returns [`ModelarDbClientError`] if the batches in the result set could not be printed.
 async fn print_batches_with_confirmation(
     mut stream: Streaming<FlightData>,
-    schema: SchemaRef,
+    schema: Arc<Schema>,
     dictionaries_by_id: &HashMap<i64, ArrayRef>,
 ) -> Result<()> {
     let mut user_input = String::new();
@@ -424,7 +424,7 @@ async fn print_batches_with_confirmation(
 /// batches in the result set could not be printed.
 async fn print_batches_without_confirmation(
     mut stream: Streaming<FlightData>,
-    schema: SchemaRef,
+    schema: Arc<Schema>,
     dictionaries_by_id: &HashMap<i64, ArrayRef>,
 ) -> Result<()> {
     while let Some(flight_data) = stream.message().await? {

crates/modelardb_common/src/remote.rs

@@ -17,11 +17,12 @@
 
 use std::collections::HashMap;
 use std::error::Error;
+use std::sync::Arc;
 
 use arrow::array::ArrayRef;
-use arrow::datatypes::SchemaRef;
+use arrow::datatypes::Schema;
 use arrow::record_batch::RecordBatch;
-use arrow_flight::{utils, FlightData, FlightDescriptor};
+use arrow_flight::{FlightData, FlightDescriptor, utils};
 use tonic::Status;
 
 /// Return the table stored as the first element in [`FlightDescriptor.path`], otherwise a
@@ -39,7 +40,7 @@ pub fn table_name_from_flight_descriptor(
 /// could not be converted, [`Status`] is returned.
 pub fn flight_data_to_record_batch(
     flight_data: &FlightData,
-    schema: &SchemaRef,
+    schema: &Arc<Schema>,
     dictionaries_by_id: &HashMap<i64, ArrayRef>,
 ) -> Result<RecordBatch, Status> {
     debug_assert_eq!(flight_data.flight_descriptor, None);

crates/modelardb_common/src/test/mod.rs

@@ -24,7 +24,7 @@ pub const INGESTED_BUFFER_SIZE: usize = 1438392;
 pub const UNCOMPRESSED_BUFFER_SIZE: usize = 1048576;
 
 /// Expected size of the compressed segments produced in the tests.
-pub const COMPRESSED_SEGMENTS_SIZE: usize = 1437;
+pub const COMPRESSED_SEGMENTS_SIZE: usize = 1565;
 
 /// Number of bytes reserved for ingested data in tests.
 pub const INGESTED_RESERVED_MEMORY_IN_BYTES: usize = 5 * 1024 * 1024; // 5 MiB