From ed33aefe7f5d6b3d06baa90e6603965d1a32abd9 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Sun, 10 Oct 2021 11:57:13 -0500 Subject: [PATCH 01/29] refactor: v2 --- .gitignore | 1 + Cargo.lock | 11 + Cargo.toml | 5 +- src/buffer.rs | 168 ----------- src/lib.rs | 128 +-------- src/stream.rs | 778 -------------------------------------------------- src/u24.rs | 163 +++++++++++ 7 files changed, 182 insertions(+), 1072 deletions(-) delete mode 100644 src/buffer.rs delete mode 100644 src/stream.rs create mode 100644 src/u24.rs diff --git a/.gitignore b/.gitignore index ea8c4bf..f9645b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +old \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index d574710..e2dfc84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "binary_utils" version = "0.1.2" +dependencies = [ + "byteorder", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" diff --git a/Cargo.toml b/Cargo.toml index d13a6e4..bccde4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,7 @@ name = "binary_utils" version = "0.1.2" authors = ["Bavfalcon9 "] edition = "2018" -include = ["src/**/*", "README.md"] \ No newline at end of file +include = ["src/**/*", "README.md"] + +[dependencies] +byteorder = "1.4.3" diff --git a/src/buffer.rs b/src/buffer.rs deleted file mode 100644 index 04b691d..0000000 --- a/src/buffer.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::ops::Deref; -use std::convert::TryInto; -use std::string::FromUtf8Error; - -pub trait IBufferRead { - /// Reads a unsigned byte (0 - 255) - /// There is no `u` preceeding the function name - /// because unsigned bytes are default. - fn read_byte(&mut self) -> u8; - /// Reads a signed byte (-128 - 128) - fn read_ibyte(&mut self) -> i8; - /// Reads a boolean, either `true` or `false` - /// A boolean is represented as follows: - /// - **true** -> 0b01 - /// - **false** -> 0b00 - fn read_bool(&mut self) -> bool; - fn read_short(&mut self) -> i16; - fn read_ushort(&mut self) -> u16; - fn read_short_le(&mut self) -> i16; - fn read_ushort_le(&mut self) -> u16; - /// Reads a three byte unsigned integer. - /// A triad is defined as follows: - /// - **MAX:** 0x00FF_FFFF - /// - **MIN:** 0x0000_0000 - fn read_triad(&mut self) -> u32; - /// Same as `read_triad` except reads in a - /// [Big Endian](https://www.freecodecamp.org/news/what-is-endianness-big-endian-vs-little-endian/) format. - fn read_triad_be(&mut self) -> u32; - fn read_int(&mut self) -> i32; - fn read_int_le(&mut self) -> i32; - fn read_uint(&mut self) -> u32; - fn read_uint_le(&mut self) -> u32; - fn read_float(&mut self) -> f32; - fn read_float_le(&mut self) -> f32; - fn read_double(&mut self) -> f64; - fn read_double_le(&mut self) -> f64; - fn read_long(&mut self) -> i64; - fn read_ulong(&mut self) -> u64; - fn read_long_le(&mut self) -> i64; - fn read_ulong_le(&mut self) -> u64; - fn read_var_int(&mut self) -> i32; - fn read_uvar_int(&mut self) -> u32; - fn read_var_long(&mut self) -> i64; - fn read_uvar_long(&mut self) -> u64; - fn read_string(&mut self) -> Result; -} - -pub trait IBufferWrite { - fn write_byte(&mut self, v: u8); - fn write_ibyte(&mut self, v: i8); - fn write_bool(&mut self, v: bool); - fn write_short(&mut self, v: i16); - fn write_ushort(&mut self, v: u16); - fn write_short_le(&mut self, v: i16); - fn write_ushort_le(&mut self, v: u16); - // Any bytes exceeding the size of a 3 byte number, are automatically removed. - fn write_triad(&mut self, v: u32); - fn write_triad_be(&mut self, v: u32); - fn write_int(&mut self, v: i32); - fn write_int_le(&mut self, v: i32); - fn write_uint(&mut self, v: u32); - fn write_uint_le(&mut self, v: u32); - fn write_float(&mut self, v: f32); - fn write_float_le(&mut self, v: f32); - fn write_double(&mut self, v: f64); - fn write_double_le(&mut self, v: f64); - fn write_long(&mut self, v: i64); - fn write_ulong(&mut self, v: u64); - fn write_long_le(&mut self, v: i64); - fn write_ulong_le(&mut self, v: u64); - fn write_var_int(&mut self, v: i32); - fn write_uvar_int(&mut self, v: u32); - fn write_var_long(&mut self, v: i64); - fn write_uvar_long(&mut self, v: u64); - fn write_string(&mut self, v: String); -} - -pub fn is_u24(num: u32) -> bool { - // checks if num is within range - !(num > 0x00FF_FFFF) -} - -// /// Buffer implementation on Array (im lazy someone pls) -// impl IBufferRead for T where T: Deref { -// fn read_byte(&mut self) -> u16 { -// 0 -// } - -// fn read_signed_byte(&mut self) -> i8 { -// 0 -// } - -// fn read_bool(&mut self) -> bool { -// false -// } - -// fn read_short(&mut self) -> u16 { -// 0 -// } - -// fn read_signed_short(&mut self) -> u16 { -// 0 -// } - -// fn read_short_le(&mut self) -> u16 { -// 0 -// } - -// fn read_signed_short_le(&mut self) -> u16 { -// 0 -// } - -// fn read_triad(&mut self) -> usize { -// 0 -// } - -// fn read_triad_le(&mut self) -> usize { -// 0 -// } - -// fn read_int(&mut self) -> usize { -// 0 -// } - -// fn read_int_le(&mut self) -> usize { -// 0 -// } - -// fn read_float(&mut self) -> f32 { -// 0.0 -// } - -// fn read_float_le(&mut self) -> f32 { -// 0.0 -// } - -// fn read_double(&mut self) -> f64 { -// 0.0 -// } - -// fn read_double_le(&mut self) -> f64 { -// 0.0 -// } - -// fn read_long(&mut self) -> usize { -// 0 -// } - -// fn read_long_le(&mut self) -> usize { -// 0 -// } - -// fn read_var_int(&mut self) -> usize { -// 0 -// } - -// fn read_signed_var_int(&mut self) -> usize { -// 0 -// } - -// fn read_var_long(&mut self) -> usize { -// 0 -// } - -// fn read_signed_var_long(&mut self) -> usize { -// 0 -// } -// } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 3c0c590..fc145aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,126 +1,4 @@ -#![allow(unused_imports, dead_code)] -pub mod buffer; -pub mod stream; +use std::io::Cursor; +pub mod u24; -pub use buffer::*; -pub use stream::*; - -pub trait StreamEncoder { - fn into_stream(&self) -> BinaryStream; -} - -pub trait StreamDecoder { - fn from_stream(stream: BinaryStream) -> Self; -} - -#[cfg(test)] -mod tests { - use crate::*; - - #[test] - fn read_write_read_write_varint() { - let mut stream = stream::BinaryStream::new(); - stream.write_uvar_long(32432); - dbg!(stream.clone()); - stream.set_offset(0); - assert_eq!(stream.read_uvar_long(), 32432); - } - - #[test] - fn slice_test() { - let mut stream = stream::BinaryStream::init(&[132, 0, 0, 0, 64, 0, 144, 0, 0, 0, 9, 144, 81, 212, 113, 24, 50, 101, 140, 0, 0, 0, 0, 4, 43, 112, 111, 0].to_vec()); - stream.read_byte(); - stream.read_triad(); - let offset = stream.get_offset(); - let mut clamped = stream.clamp(offset, None); - assert_eq!(clamped.read_byte(), 64) - } - - #[test] - fn test_read_short() { - let mut bin_stream = stream::BinaryStream::new(); - bin_stream.write_short(12); - - print!("{:?}", bin_stream); - } - - #[test] - fn test_write_byte() { - let okay = vec![10]; - let mut stream = stream::BinaryStream::new(); - stream.write_byte(10); - assert_eq!(okay, stream.get_buffer()); - } - - #[test] - fn test_read_byte() { - let raw = vec![0, 10, 0, 13, 10]; - let mut stream = stream::BinaryStream::init(&raw); - stream.read_short(); - stream.read_short(); - println!("{}", stream.get_offset()); - let is_ten = stream.read_byte(); - assert_eq!(is_ten, 10); - } - - #[test] - fn read_slice_panic() { - let raw = vec![7, 0, 255, 255, 0, 254, 254, 254, 254, 253, 253, 253, 253, 18, 52, 86, 120, 4, 128, 255, 255, 254, 74, 188, 2, 65, 140, 131, 72, 201, 65, 219, 142, 52]; - let mut stream = stream::BinaryStream::init(&raw); - stream.read_byte(); - assert_eq!([0, 255, 255, 0].to_vec(), stream.read_slice(Some(4))); - assert_eq!(stream.get_offset(), 5); - } - - #[test] - fn test_read_triad() { - let buf = [ 233, 9, 27 ]; - // we need to read the first three bytes - let mut bin = stream::BinaryStream::init(&buf.to_vec()); - let num = bin.read_triad(); - - assert_eq!(1772009, num); - } - - #[test] - fn test_read_triad_zero() { - let buf = [ 0, 0, 0 ]; - let mut bin = stream::BinaryStream::init(&buf.to_vec()); - let num = bin.read_triad(); - - assert_eq!(num, 0); - } - - #[test] - fn test_read_index_at_1() { - let buf = [144, 0, 0, 0, 9, 143, 162, 116, 15, 10, 144, 162, 92, 0, 0, 0, 0, 21, 47, 173, 144, 0]; - let mut bin = stream::BinaryStream::init(&buf.to_vec()); - bin.read_byte(); - bin.read_triad(); - } - - #[test] - fn test_read_var_int() { - let buf = [236, 189, 203, 118, 242, 202, 214, 247, 247, 126, 189, 36, 151, 241, 166, 155, 253, 14, 73, 128, 183, 73, 207, 128, 132, 193, 72, 24, 161, 3, 82, 70, 198, 30, 128, 216, 6, 36, 48, 182, 49, 167, 140]; - let mut bin = stream::BinaryStream::init(&buf.to_vec()); - let v = bin.read_var_int(); - assert_eq!(v, 0) - } - - #[test] - fn test_write_triad() { - let okay = vec![0, 0, 0]; - let mut bin = stream::BinaryStream::new(); - bin.write_triad(0); - assert_eq!(okay, bin.get_buffer()); - } - - #[test] - fn test_read_int() { - let buf = [ 0, 0, 0, 7 ]; - let mut bin = stream::BinaryStream::init(&buf.to_vec()); - let num = bin.read_int(); - - assert_eq!(7, num); - } -} \ No newline at end of file +pub type Stream = Cursor>; \ No newline at end of file diff --git a/src/stream.rs b/src/stream.rs deleted file mode 100644 index 7beb86f..0000000 --- a/src/stream.rs +++ /dev/null @@ -1,778 +0,0 @@ -use std::convert::TryInto; -use std::string::FromUtf8Error; -use std::ops::{ Range, Index, IndexMut }; -use std::error::Error; -use std::fmt::{ Display, Formatter, Result as FResult }; -use std::mem; - -use crate::is_u24; -use super::buffer; - -// Errors for binarystream -pub struct AllocationError; - -#[derive(Debug)] -pub struct IllegalOffsetError { - legal: usize, - actual: usize, - cause_bounds: bool -} - -#[derive(Debug)] -pub enum ClampErrorCause { - AboveBounds, - BelowBounds, - InvalidBounds -} - -#[derive(Debug)] -pub struct ClampError { - cause: ClampErrorCause -} - -impl ClampError { - fn new(cause: ClampErrorCause) -> Self { - Self { - cause - } - } -} - -impl Display for AllocationError { - fn fmt(&self, f: &mut Formatter) -> FResult { - write!(f, "Attempted to allocate negative bytes.") - } -} - -impl Display for IllegalOffsetError { - fn fmt(&self, f: &mut Formatter) -> FResult { - if self.cause_bounds { - write!(f, "Offset: {} is outside of the BinaryStream's bounds", self.actual) - } else { - write!(f, "Offset: {} is outside of possible offset of: {}", self.actual, self.cause_bounds) - } - } -} - -impl Display for ClampError { - fn fmt(&self, f: &mut Formatter) -> FResult { - match self.cause { - ClampErrorCause::AboveBounds => write!(f, "Clamp start can not be bigger than the buffer's length."), - ClampErrorCause::BelowBounds => write!(f, "Clamp end is less than the start length."), - ClampErrorCause::InvalidBounds => write!(f, "Clamp start or end is mismatched.") - } - } -} - -pub trait IBinaryStream { - /// Increases the offset. If `None` is given in `amount`, 1 will be used. - fn increase_offset(&mut self, amount: Option) -> usize; - - /// Changes the offset of the stream to the new given offset. - /// returns `true` if the offset is in bounds and `false` if the offset is out of bounds. - fn set_offset(&mut self, offset: usize) -> bool; - - /// Returns the current offset at the given time when called. - fn get_offset(&mut self) -> usize; - - /// Returns the length of the current buffer. - fn get_length(&self) -> usize; - - /// Returns the current buffer as a clone of the original. - fn get_buffer(&self) -> Vec; - - /// Allocates more bytes to the binary stream. - /// Allocations can occur as many times as desired, however a negative allocation will cause - /// the stream to "drop" or "delete" bytes from the buffer. Discarded bytes are not recoverable. - /// - /// Useful when writing to a stream, allows for allocating for chunks, etc. - /// - /// **Example:** - /// - /// stream.allocate(1024); - /// stream.write_string(String::from("a random string, that can only be a max of 1024 bytes.")); - fn allocate(&mut self, bytes: usize); - - /// Allocates more bytes to the binary stream only **if** the given bytelength will exceed - /// the current binarystream's bounds. - fn allocate_if(&mut self, bytes: usize); - - /// Create a new Binary Stream from nothing. - fn new() -> Self; - - /// Create a new Binary Stream from a vector of bytes. - fn init(buf: &Vec) -> Self; - - /// Similar to slice, clamp, "grips" the buffer from a given offset, and changes the initial bounds. - /// Meaning that any previous bytes before the given bounds are no longer writable. - /// - /// Useful for cloning "part" of a stream, and only allowing certain "bytes" to be read. - /// Clamps can not be undone. - /// - /// **Example:** - /// - /// let stream = BinaryStream::new(vec!(([98,105,110,97,114,121,32,117,116,105,108,115])); - /// let shareable_stream = stream.clamp(7, None); // 32,117,116,105,108,115 are now the only bytes readable externally - fn clamp(&mut self, start: usize, end: Option) -> Self; - - /// Checks whether or not the given offset is in between the streams bounds and if the offset is valid. - /// - /// **Example:** - /// - /// if stream.is_within_bounds(100) { - /// println!("Can write to offset: 100"); - /// } else { - /// println!("100 is out of bounds."); - /// } - fn is_within_bounds(&self, offset: usize) -> bool; - - /// Reads a byte, updates the offset, clamps to last offset. - /// - /// **Example:** - /// - /// let mut fbytes = Vec::new(); - /// loop { - /// if fbytes.len() < 4 { - /// fbytes.push(stream.read()); - /// } - /// break; - /// } - fn read(&mut self) -> u8; - - /// Writes a byte ands returns it. - fn write_usize(&mut self, v: usize) -> usize; - - /// Reads a slice from the Binary Stream and automatically updates - /// the offset for the given slice's length. - /// - /// **Example:** - /// stream.read_slice(); - fn read_slice(&mut self, length: Option) -> Vec; - - /// Reads a slice from the Binary Stream and automatically updates - /// the offset for the given slice's length. - /// - /// ! This function indexes from 0 always! - /// - /// **Example:** - /// stream.read_slice(); - fn read_slice_exact(&mut self, length: Option) -> Vec; - - /// Writes a slice onto the Binary Stream and automatically allocates - /// memory for the slice. - /// - /// **Example:** - /// stream.write_slice(&[0, 38, 92, 10]); - fn write_slice(&mut self, v: &[u8]); -} - -#[derive(Debug, Clone)] -pub struct BinaryStream { - buffer: Vec, - offset: usize, - bounds: (usize, usize) -} - -impl BinaryStream { - /// Takes a slice out of the vector and makes a new binary stream with that slice or "segment". - pub fn slice(&mut self, start: usize, end: Option) -> Self { - if end.is_none() { - let buf = self.buffer[start..self.buffer.len()].to_vec(); - return BinaryStream::init(&buf); - } else { - let buf = self.buffer[start..end.unwrap()].to_vec(); - return BinaryStream::init(&buf); - } - } - - /// Increases the offset. If `None` is given in `amount`, 1 will be used. - pub fn increase_offset(&mut self, amount: Option) -> usize { - let amnt = match amount { - None => 1 as usize, - Some(n) => n - }; - - if (self.offset + amnt) > self.bounds.1 { - panic!(IllegalOffsetError { - actual: amnt, - legal: self.bounds.1, - cause_bounds: true - }) - } - - self.offset = self.offset + amnt; - self.offset - } - - /// Changes the offset of the stream to the new given offset. - /// returns `true` if the offset is in bounds and `false` if the offset is out of bounds. - pub fn set_offset(&mut self, offset: usize) -> bool { - if offset > self.bounds.1 { - false - } else { - self.offset = offset; - true - } - } - - /// Returns the current offset at the given time when called. - pub fn get_offset(&mut self) -> usize { - self.offset - } - - /// Returns the length of the current buffer. - pub fn get_length(&self) -> usize { - self.buffer.len() as usize - } - - /// Returns the current buffer as a clone of the original. - pub fn get_buffer(&self) -> Vec { - self.buffer.clone() - } - - /// Allocates more bytes to the binary stream. - /// Allocations can occur as many times as desired, however a negative allocation will cause - /// the stream to "drop" or "delete" bytes from the buffer. Discarded bytes are not recoverable. - /// - /// Useful when writing to a stream, allows for allocating for chunks, etc. - /// - /// **Example:** - /// - /// stream.allocate(1024); - /// stream.write_string(String::from("a random string, that can only be a max of 1024 bytes.")); - pub fn allocate(&mut self, bytes: usize) { - self.bounds.1 = self.buffer.len() + bytes; - // self.buffer.resize(self.bounds.1, 0) - } - - /// Allocates more bytes to the binary stream only **if** the given bytelength will exceed - /// the current binarystream's bounds. - pub fn allocate_if(&mut self, bytes: usize) { - if (self.buffer.len() + bytes > self.bounds.1) && (self.offset + bytes) >= self.bounds.1 { - self.allocate(bytes) - } - } - - /// Create a new Binary Stream from nothing. - pub fn new() -> Self { - Self { - buffer: Vec::new(), - bounds: (0, 0), - offset: 0 - } - } - - /// Create a new Binary Stream from a vector of bytes. - pub fn init(buf: &Vec) -> Self { - Self { - buffer: buf.clone(), - bounds: (0, buf.len()), - offset: 0 - } - } - - /// Similar to slice, clamp, "grips" the buffer from a given offset, and changes the initial bounds. - /// Meaning that any previous bytes before the given bounds are no longer writable. - /// - /// Useful for cloning "part" of a stream, and only allowing certain "bytes" to be read. - /// Clamps can not be undone. - /// - /// **Example:** - /// - /// let stream = BinaryStream::new(vec!(([98,105,110,97,114,121,32,117,116,105,108,115]))); - /// let shareable_stream = stream.clamp(7, None); // 32,117,116,105,108,115 are now the only bytes readable externally - pub fn clamp(&mut self, start: usize, end: Option) -> Self { - let mut new = self.clone(); - if start > self.buffer.len() { - panic!(ClampError::new(ClampErrorCause::AboveBounds)); - } else if start < self.bounds.0 { - panic!(ClampError::new(ClampErrorCause::BelowBounds)); - } - - new.bounds.0 = start; - new.set_offset(start); - - if match end { None => false, _ => true} { - if end.unwrap() < self.bounds.0 { - panic!(ClampError::new(ClampErrorCause::InvalidBounds)); - } - new.bounds.1 = end.unwrap(); - } - - // Dereferrenced for use by consumer. - new - } - - /// Checks whether or not the given offset is in between the streams bounds and if the offset is valid. - /// - /// **Example:** - /// - /// if stream.is_within_bounds(100) { - /// println!("Can write to offset: 100"); - /// } else { - /// println!("100 is out of bounds."); - /// } - pub fn is_within_bounds(&self, offset: usize) -> bool { - !(offset > self.bounds.1 || offset < self.bounds.0 || offset > self.buffer.len()) - } - - /// Reads a byte, updates the offset, clamps to last offset. - /// - /// **Example:** - /// - /// let mut fbytes = Vec::new(); - /// loop { - /// if fbytes.len() < 4 { - /// fbytes.push(stream.read()); - /// } - /// break; - /// } - pub fn read(&mut self) -> u8 { - let byte = self[self.offset]; - self.clamp(self.offset, None); - self.increase_offset(None); - byte - } - - /// Writes a byte ands returns it. - pub fn write_usize(&mut self, v: usize) -> usize { - self.allocate_if(1); - self.buffer.push(v as u8); - v - } - - /// Writes a slice onto the Binary Stream and automatically allocates - /// memory for the slice. - /// - /// **Example:** - /// stream.write_slice(&[0, 38, 92, 10]); - pub fn write_slice(&mut self, v: &[u8]) { - self.allocate_if(v.len()); - self.buffer.extend_from_slice(v); - } - - /// Reads a slice from the Binary Stream and automatically updates - /// the offset for the given slice's length. - /// - /// **Example:** - /// stream.read_slice(); - pub fn read_slice_exact(&mut self, length: Option) -> Vec { - let len = match length { - Some(v) => v, - None => 1 - }; - let vec = self[self.offset..len].to_vec(); - self.increase_offset(Some(len)); - vec - } - - /// Reads a slice from the Binary Stream and automatically updates - /// the offset for the given slice's length. - /// - /// **Example:** - /// stream.read_slice(); - pub fn read_slice(&mut self, length: Option) -> Vec { - let len = match length { - Some(v) => v, - None => 1 - }; - let vec = self[self.offset..len + self.offset].to_vec(); - self.increase_offset(Some(len)); - vec - } -} - -/// Implements indexing on BinaryStream. -/// When indexing you can access the bytes only readable by the streams bounds. -/// If the offset you're trying to index is "outside" of the "bounds" of the stream this will panic. -/// -/// **Example:** -/// -/// let first_byte = stream[0]; -impl std::ops::Index for BinaryStream { - type Output = u8; - fn index(&self, idx: usize) -> &u8 { - if !self.is_within_bounds(idx) { - if self.bounds.0 == 0 && self.bounds.1 == self.buffer.len() { - panic!("Index is out of bounds due to clamp."); - } else { - panic!("Index is out of bounds."); - } - } - - self.buffer.get(idx).unwrap() - } -} - -/// Implements indexing with slices on BinaryStream. -/// Operates exactly like indexing, except with slices. -/// -/// **Example:** -/// -/// let first_bytes = stream[0..3]; -impl Index> for BinaryStream { - type Output = [u8]; - fn index(&self, idx: Range) -> &[u8] { - if !self.is_within_bounds(idx.end) || !self.is_within_bounds(idx.start) { - if self.bounds.0 == 0 && self.bounds.1 == self.buffer.len() { - panic!("Index is out of bounds due to clamp."); - } else { - panic!("Index is out of bounds."); - } - } - - self.buffer.get(idx).unwrap() - } -} - -impl std::ops::IndexMut for BinaryStream { - fn index_mut(&mut self, offset: usize) -> &mut u8 { - if !self.is_within_bounds(offset) { - self.buffer.get_mut(offset).unwrap() - } else { - panic!("Offset: {} is out of bounds.", offset); - } - } -} - -impl buffer::IBufferRead for BinaryStream { - /// Literally, reads a byte - fn read_ibyte(&mut self) -> i8 { - // an i8 is only 1 byte - let b = i8::from_be_bytes(self.buffer[self.offset..self.offset + 1].try_into().unwrap()); - self.increase_offset(Some(1)); - b - } - - fn read_byte(&mut self) -> u8 { - let byte = self.buffer[self.offset]; - self.increase_offset(None); - byte - } - - fn read_bool(&mut self) -> bool { - self.read_byte() != 0 - } - - fn read_string(&mut self) -> Result { - let length = self.read_short(); - let string = String::from_utf8(self[self.offset..self.offset + length as usize].to_vec()); - self.increase_offset(Some(self.offset + length as usize)); - string - } - - fn read_short(&mut self) -> i16 { - let b = i16::from_be_bytes(self.buffer[self.offset..self.offset + 2].try_into().unwrap()); - self.increase_offset(Some(2)); - b - } - - fn read_ushort(&mut self) -> u16 { - // a short is 2 bytes and is a u16, - let b = u16::from_be_bytes(self.buffer[self.offset..self.offset + 2].try_into().unwrap()); - self.increase_offset(Some(2)); - b - } - - fn read_short_le(&mut self) -> i16 { - let b = i16::from_le_bytes(self.buffer[self.offset..self.offset + 2].try_into().unwrap()); - self.increase_offset(Some(2)); - b - } - - fn read_ushort_le(&mut self) -> u16 { - let b = u16::from_le_bytes(self.buffer[self.offset..self.offset + 2].try_into().unwrap()); - self.increase_offset(Some(2)); - b - } - - fn read_triad(&mut self) -> u32 { - // a triad is 3 bytes - // we read three bytes here, but don't increase the offest - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), 0]; - let b = u32::from_le_bytes(bytes); - b - } - - fn read_triad_be(&mut self) -> u32 { - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), 0]; - let b = u32::from_be_bytes(bytes); - b - } - - fn read_int(&mut self) -> i32 { - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), self.read_byte()]; - i32::from_be_bytes(bytes) - } - - - fn read_int_le(&mut self) -> i32 { - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), self.read_byte()]; - i32::from_le_bytes(bytes) - } - - fn read_uint(&mut self) -> u32 { - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), self.read_byte()]; - u32::from_be_bytes(bytes) - } - - - fn read_uint_le(&mut self) -> u32 { - let bytes = [self.read_byte(), self.read_byte(), self.read_byte(), self.read_byte()]; - u32::from_le_bytes(bytes) - } - - fn read_float(&mut self) -> f32 { - let b = f32::from_be_bytes(self.buffer[self.offset..self.offset + 4].try_into().unwrap()); - self.increase_offset(Some(4)); - b - } - - fn read_float_le(&mut self) -> f32 { - let b = f32::from_le_bytes(self.buffer[self.offset..self.offset + 4].try_into().unwrap()); - self.increase_offset(Some(4)); - b - } - - fn read_double(&mut self) -> f64 { - let b = f64::from_be_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_double_le(&mut self) -> f64 { - let b = f64::from_le_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_long(&mut self) -> i64 { - let b = i64::from_be_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_ulong(&mut self) -> u64 { - let b = u64::from_be_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_long_le(&mut self) -> i64 { - let b = i64::from_le_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_ulong_le(&mut self) -> u64 { - let b = u64::from_le_bytes(self.buffer[self.offset..self.offset + 8].try_into().unwrap()); - self.increase_offset(Some(8)); - b - } - - fn read_var_int(&mut self) -> i32 { - let var_int: u32 = self.read_uvar_int(); - let mut value; - - // to-do remove this hack - value = i32::from_be_bytes(var_int.to_be_bytes()); - if var_int & 1 != 0 { - value = value ^ value - } - - value - } - - fn read_uvar_int(&mut self) -> u32 { - let mut value: u32 = 0; - let mut i: u32 = 0; - - while i < 35 { - let byte = self.read_byte(); - value |= u32::from(byte & 0x7f) << i; - if byte & 0x80 == 0 { - return value; - } - i += 7; - } - panic!("Read uvarint did not terminate after fifth byte."); - } - - fn read_var_long(&mut self) -> i64 { - let var_int: u64 = self.read_uvar_long(); - let mut value; - - // to-do remove this hack - value = i64::from_be_bytes(var_int.to_be_bytes()); - if var_int & 1 != 0 { - value = value ^ value - } - - value - } - - fn read_uvar_long(&mut self) -> u64 { - let mut value: u64 = 0; - let mut i: u64 = 0; - - while i < 70 { - let byte = self.read_byte(); - value |= ((byte & 0x7f) << i) as u64; - - if byte & 0x80 == 0 { - return value; - } - i += 7; - } - panic!("Read uvarlong did not terminate after tenth byte."); - } -} - -impl buffer::IBufferWrite for BinaryStream { - fn write_ibyte(&mut self, v: i8) { - self.write_slice(&v.to_be_bytes()) - } - - fn write_byte(&mut self, v: u8) { - self.write_slice(&v.to_be_bytes()) - } - - fn write_bool(&mut self, v: bool) { - let byte = match v { - true => 1, - false => 0 - }; - self.write_byte(byte); - } - - fn write_short(&mut self, v: i16) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_ushort(&mut self, v: u16) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_short_le(&mut self, v: i16) { - self.write_slice(&v.to_le_bytes()); - } - - fn write_ushort_le(&mut self, v: u16) { - self.write_slice(&v.to_le_bytes()); - } - - fn write_triad(&mut self, v: u32) { - // this is actually a hack fix! - // we're actually writing a u24 here - // we don't care about the last byte - let bytes = &v.to_le_bytes()[0..3]; - self.write_slice(bytes); - } - - fn write_triad_be(&mut self, v: u32) { - let bytes = &v.to_be_bytes()[0..3]; - self.write_slice(bytes); - } - - fn write_int(&mut self, v: i32) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_int_le(&mut self, v: i32) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_uint(&mut self, v: u32) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_uint_le(&mut self, v: u32) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_float(&mut self, v: f32) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_float_le(&mut self, v: f32) { - self.write_slice(&v.to_le_bytes()); - } - - fn write_double(&mut self, v: f64) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_double_le(&mut self, v: f64) { - self.write_slice(&v.to_le_bytes()); - } - - fn write_long(&mut self, v: i64) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_ulong(&mut self, v: u64) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_long_le(&mut self, v: i64) { - self.write_slice(&v.to_be_bytes()); - } - - fn write_ulong_le(&mut self, v: u64) { - self.write_slice(&v.to_le_bytes()); - } - - fn write_var_int(&mut self, v: i32) { - let bytes = v.to_be_bytes().to_vec(); - for byte in bytes.iter() { - if *byte == 0 { - self.write_byte(*byte & 0x7f); - return; - } else { - self.write_byte(*byte); - } - } - } - - fn write_uvar_int(&mut self, v: u32) { - let bytes = v.to_be_bytes().to_vec(); - for byte in bytes.iter() { - if *byte == 0 { - // self.write_byte(*byte); - return; - } else { - self.write_byte(*byte); - } - } - } - - fn write_var_long(&mut self, v: i64) { - let bytes = v.to_be_bytes().to_vec(); - for byte in bytes.iter() { - if *byte == 0 { - self.write_byte(*byte); - return; - } else { - self.write_byte(*byte); - } - } - } - - fn write_uvar_long(&mut self, v: u64) { - let bytes = v.to_be_bytes().to_vec(); - for byte in bytes.iter() { - if *byte == 0 { - self.write_byte(*byte); - return; - } else { - self.write_byte(*byte); - } - } - } - - - fn write_string(&mut self, v: String) { - self.write_ushort(v.len() as u16); - self.write_slice(v.as_bytes()); - } -} \ No newline at end of file diff --git a/src/u24.rs b/src/u24.rs new file mode 100644 index 0000000..9b2d088 --- /dev/null +++ b/src/u24.rs @@ -0,0 +1,163 @@ +#![allow(non_camel_case_types)] + +use std::convert::{From, Into}; +use std::ops::{Add, Mul, Div, Sub, BitOr}; +use std::cmp::{PartialEq, PartialOrd, Ordering}; +use std::io; +use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; +/// Base Implementation for a u24 +pub struct u24(u32); // inner is validated + +impl u24 { + pub fn is_u24(num: usize) -> bool { + num < 0x00FF_FFFF + } + + pub fn from_be_bytes(bytes: &[u8]) -> Self { + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], 0]).into() + } + + pub fn from_le_bytes(bytes: &[u8]) -> Self { + u32::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() + } + + pub fn to_le_bytes(self) -> [u8; 3] { + let bytes = self.0.to_le_bytes(); + [bytes[0], bytes[1], bytes[2]] + } + + pub fn to_be_bytes(self) -> [u8; 3] { + let bytes = self.0.to_be_bytes(); + [bytes[0], bytes[1], bytes[2]] + } +} + +pub trait u24Writer: io::Write { + #[inline] + fn write_u24(&mut self, num: u24) -> io::Result { + self.write(&num.to_be_bytes()) + } +} + +pub trait u24Reader: io::Read { + #[inline] + fn read_u24(&mut self) -> io::Result { + let initial = [self.read_u8()?, self.read_u8()?, self.read_u8()?]; + Ok(u24::from_be_bytes(&initial)) + } +} + +impl Add for u24 { + type Output = Self; + + fn add(self, other: u24) -> Self::Output { + u24(self.0 + other.0) + } +} + +impl Mul for u24 { + type Output = Self; + + fn mul(self, other: u24) -> Self::Output { + u24(self.0 * other.0) + } +} + +impl Sub for u24 { + type Output = Self; + + fn sub(self, other: u24) -> Self::Output { + u24(self.0 - other.0) + } +} + +impl Div for u24 { + type Output = Self; + + fn div(self, other: u24) -> Self::Output { + u24(self.0 / other.0) + } +} + +impl PartialEq for u24 { + fn eq(&self, other: &u24) -> bool { + self.0 == other.0 + } +} + +impl PartialOrd for u24 { + fn partial_cmp(&self, other: &u24) -> Option { + self.0.partial_cmp(&other.0) + } +} + +macro_rules! impl_primitive_u24 { + ($ty:ty) => { + impl From<$ty> for u24 { + fn from(value: $ty) -> Self { + if !u24::is_u24(value as usize) { + panic!("Can not convert a number larger than the bounds of a u24 into a u24") + } else { + u24(value as u32) + } + } + } + + impl BitOr<$ty> for u24 { + type Output = Self; + + fn bitor(self, rhs: $ty) -> Self::Output { + u24(self.0 | rhs as u32) + } + } + + impl Into<$ty> for u24 { + fn into(self) -> $ty { + self.0 as $ty + } + } + + impl Add<$ty> for u24 { + type Output = Self; + + fn add(self, other: $ty) -> Self::Output { + u24(self.0 + other as u32) + } + } + + impl Mul<$ty> for u24 { + type Output = Self; + + fn mul(self, other: $ty) -> Self::Output { + u24(self.0 * other as u32) + } + } + + impl Sub<$ty> for u24 { + type Output = Self; + + fn sub(self, other: $ty) -> Self::Output { + u24(self.0 - other as u32) + } + } + + impl Div<$ty> for u24 { + type Output = Self; + + fn div(self, other: $ty) -> Self::Output { + u24(self.0 / other as u32) + } + } + }; +} + +impl_primitive_u24!(u8); +impl_primitive_u24!(u16); +impl_primitive_u24!(u32); +impl_primitive_u24!(u64); +impl_primitive_u24!(u128); +impl_primitive_u24!(i8); +impl_primitive_u24!(i16); +impl_primitive_u24!(i32); +impl_primitive_u24!(i64); +impl_primitive_u24!(i128); \ No newline at end of file From ac56cd3e81216d57d1a2784fee0f8aa256f64a90 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Sun, 10 Oct 2021 16:34:37 -0500 Subject: [PATCH 02/29] chore: work on macros --- .gitignore | 2 +- bin_macro/Cargo.lock | 47 +++++++++ bin_macro/Cargo.toml | 13 +++ bin_macro/src/lib.rs | 8 ++ bin_macro/src/stream.rs | 35 +++++++ src/lib.rs | 26 ++++- src/u24.rs | 226 ++++++++++++++++++++-------------------- src/varint.rs | 1 + 8 files changed, 242 insertions(+), 116 deletions(-) create mode 100644 bin_macro/Cargo.lock create mode 100644 bin_macro/Cargo.toml create mode 100644 bin_macro/src/lib.rs create mode 100644 bin_macro/src/stream.rs create mode 100644 src/varint.rs diff --git a/.gitignore b/.gitignore index f9645b1..dc2d6ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -/target +target old \ No newline at end of file diff --git a/bin_macro/Cargo.lock b/bin_macro/Cargo.lock new file mode 100644 index 0000000..8b34b44 --- /dev/null +++ b/bin_macro/Cargo.lock @@ -0,0 +1,47 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bin_macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" diff --git a/bin_macro/Cargo.toml b/bin_macro/Cargo.toml new file mode 100644 index 0000000..4b3c596 --- /dev/null +++ b/bin_macro/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bin_macro" +version = "0.1.0" +edition = "2021" +private = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.29" +quote = "1.0.10" +syn = { version = "1.0.80", features = [ "full" ] } diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs new file mode 100644 index 0000000..5d9038a --- /dev/null +++ b/bin_macro/src/lib.rs @@ -0,0 +1,8 @@ +use proc_macro::TokenStream; +use syn::{DeriveInput, parse_macro_input}; +mod stream; + +#[proc_macro_derive(BinaryStream)] +pub fn derive_stream(input: TokenStream) -> TokenStream { + stream::derive_stream(parse_macro_input!(input as DeriveInput)).into() +} \ No newline at end of file diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs new file mode 100644 index 0000000..27bb08b --- /dev/null +++ b/bin_macro/src/stream.rs @@ -0,0 +1,35 @@ +use proc_macro2::{Ident, Literal, TokenStream}; +use quote::quote; +use syn::{Data, DeriveInput, Error, Fields, Result}; + +pub fn stream_parse(input: DeriveInput) -> Result { + let name = &input.ident; + match input.data { + Data::Struct(v) => { + + }, + Data::Enum(v) => { + + }, + _ => panic!("BinaryStream does not support Type Unions. Use Enums instead.") + } + Ok(quote! {}) +} + +pub fn impl_struct_fields(fields: Fields) { + match fields { + Fields::Named(v) => { + for field in &v.named { + let field_id = field.ident.as_ref(); + let id = quote!(self.#field_id); + + } + }, + Fields::Unnamed(v) => { + + }, + Fields::Unit => { + panic!("Can not use uninitalized data values.") + } + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index fc145aa..c8c6ab1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,26 @@ -use std::io::Cursor; +use std::io; +use byteorder::{ReadBytesExt, WriteBytesExt}; pub mod u24; -pub type Stream = Cursor>; \ No newline at end of file +pub type Stream = io::Cursor>; + +pub trait Streamable { + /// Writes `self` to the given buffer. + fn write(&self, src: &mut Vec); + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self; +} + +pub trait BinWrite: io::Write { + #[inline] + fn write_bool(&mut self, value: bool) -> io::Result<()> { + self.write_u8(value.into()) + } +} + +pub trait BinRead: io::Read { + #[inline] + fn read_bool(&mut self) -> bool { + self.read_u8().unwrap_or(0) == 0 + } +} \ No newline at end of file diff --git a/src/u24.rs b/src/u24.rs index 9b2d088..5bd75ed 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -1,154 +1,154 @@ #![allow(non_camel_case_types)] +use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; +use std::cmp::{Ordering, PartialEq, PartialOrd}; use std::convert::{From, Into}; -use std::ops::{Add, Mul, Div, Sub, BitOr}; -use std::cmp::{PartialEq, PartialOrd, Ordering}; use std::io; -use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; +use std::ops::{Add, BitOr, Div, Mul, Sub}; /// Base Implementation for a u24 pub struct u24(u32); // inner is validated impl u24 { - pub fn is_u24(num: usize) -> bool { - num < 0x00FF_FFFF - } - - pub fn from_be_bytes(bytes: &[u8]) -> Self { - u32::from_be_bytes([bytes[0], bytes[1], bytes[2], 0]).into() - } - - pub fn from_le_bytes(bytes: &[u8]) -> Self { - u32::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() - } - - pub fn to_le_bytes(self) -> [u8; 3] { - let bytes = self.0.to_le_bytes(); - [bytes[0], bytes[1], bytes[2]] - } - - pub fn to_be_bytes(self) -> [u8; 3] { - let bytes = self.0.to_be_bytes(); - [bytes[0], bytes[1], bytes[2]] - } + pub fn is_u24(num: usize) -> bool { + num < 0x00FF_FFFF + } + + pub fn from_be_bytes(bytes: &[u8]) -> Self { + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], 0]).into() + } + + pub fn from_le_bytes(bytes: &[u8]) -> Self { + u32::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() + } + + pub fn to_le_bytes(self) -> [u8; 3] { + let bytes = self.0.to_le_bytes(); + [bytes[0], bytes[1], bytes[2]] + } + + pub fn to_be_bytes(self) -> [u8; 3] { + let bytes = self.0.to_be_bytes(); + [bytes[0], bytes[1], bytes[2]] + } } pub trait u24Writer: io::Write { - #[inline] - fn write_u24(&mut self, num: u24) -> io::Result { - self.write(&num.to_be_bytes()) - } + #[inline] + fn write_u24(&mut self, num: u24) -> io::Result { + self.write(&num.to_be_bytes()) + } } pub trait u24Reader: io::Read { - #[inline] - fn read_u24(&mut self) -> io::Result { - let initial = [self.read_u8()?, self.read_u8()?, self.read_u8()?]; - Ok(u24::from_be_bytes(&initial)) - } + #[inline] + fn read_u24(&mut self) -> io::Result { + let initial = [self.read_u8()?, self.read_u8()?, self.read_u8()?]; + Ok(u24::from_be_bytes(&initial)) + } } impl Add for u24 { - type Output = Self; + type Output = Self; - fn add(self, other: u24) -> Self::Output { - u24(self.0 + other.0) - } + fn add(self, other: u24) -> Self::Output { + u24(self.0 + other.0) + } } impl Mul for u24 { - type Output = Self; + type Output = Self; - fn mul(self, other: u24) -> Self::Output { - u24(self.0 * other.0) - } + fn mul(self, other: u24) -> Self::Output { + u24(self.0 * other.0) + } } impl Sub for u24 { - type Output = Self; + type Output = Self; - fn sub(self, other: u24) -> Self::Output { - u24(self.0 - other.0) - } + fn sub(self, other: u24) -> Self::Output { + u24(self.0 - other.0) + } } impl Div for u24 { - type Output = Self; + type Output = Self; - fn div(self, other: u24) -> Self::Output { - u24(self.0 / other.0) - } + fn div(self, other: u24) -> Self::Output { + u24(self.0 / other.0) + } } impl PartialEq for u24 { - fn eq(&self, other: &u24) -> bool { - self.0 == other.0 - } + fn eq(&self, other: &u24) -> bool { + self.0 == other.0 + } } impl PartialOrd for u24 { - fn partial_cmp(&self, other: &u24) -> Option { - self.0.partial_cmp(&other.0) - } + fn partial_cmp(&self, other: &u24) -> Option { + self.0.partial_cmp(&other.0) + } } macro_rules! impl_primitive_u24 { - ($ty:ty) => { - impl From<$ty> for u24 { - fn from(value: $ty) -> Self { - if !u24::is_u24(value as usize) { - panic!("Can not convert a number larger than the bounds of a u24 into a u24") - } else { - u24(value as u32) - } - } - } - - impl BitOr<$ty> for u24 { - type Output = Self; - - fn bitor(self, rhs: $ty) -> Self::Output { - u24(self.0 | rhs as u32) - } - } - - impl Into<$ty> for u24 { - fn into(self) -> $ty { - self.0 as $ty - } - } - - impl Add<$ty> for u24 { - type Output = Self; - - fn add(self, other: $ty) -> Self::Output { - u24(self.0 + other as u32) - } - } - - impl Mul<$ty> for u24 { - type Output = Self; - - fn mul(self, other: $ty) -> Self::Output { - u24(self.0 * other as u32) - } - } - - impl Sub<$ty> for u24 { - type Output = Self; - - fn sub(self, other: $ty) -> Self::Output { - u24(self.0 - other as u32) - } - } - - impl Div<$ty> for u24 { - type Output = Self; - - fn div(self, other: $ty) -> Self::Output { - u24(self.0 / other as u32) - } - } - }; + ($ty:ty) => { + impl From<$ty> for u24 { + fn from(value: $ty) -> Self { + if !u24::is_u24(value as usize) { + panic!("Can not convert a number larger than the bounds of a u24 into a u24") + } else { + u24(value as u32) + } + } + } + + impl BitOr<$ty> for u24 { + type Output = Self; + + fn bitor(self, rhs: $ty) -> Self::Output { + u24(self.0 | rhs as u32) + } + } + + impl Into<$ty> for u24 { + fn into(self) -> $ty { + self.0 as $ty + } + } + + impl Add<$ty> for u24 { + type Output = Self; + + fn add(self, other: $ty) -> Self::Output { + u24(self.0 + other as u32) + } + } + + impl Mul<$ty> for u24 { + type Output = Self; + + fn mul(self, other: $ty) -> Self::Output { + u24(self.0 * other as u32) + } + } + + impl Sub<$ty> for u24 { + type Output = Self; + + fn sub(self, other: $ty) -> Self::Output { + u24(self.0 - other as u32) + } + } + + impl Div<$ty> for u24 { + type Output = Self; + + fn div(self, other: $ty) -> Self::Output { + u24(self.0 / other as u32) + } + } + }; } impl_primitive_u24!(u8); @@ -160,4 +160,4 @@ impl_primitive_u24!(i8); impl_primitive_u24!(i16); impl_primitive_u24!(i32); impl_primitive_u24!(i64); -impl_primitive_u24!(i128); \ No newline at end of file +impl_primitive_u24!(i128); diff --git a/src/varint.rs b/src/varint.rs new file mode 100644 index 0000000..d7490db --- /dev/null +++ b/src/varint.rs @@ -0,0 +1 @@ +pub struct VarInt(T, usize); From 3dbd5fc3b5804c876ce94b5df09ca2a2ffc7aecd Mon Sep 17 00:00:00 2001 From: Olybear9 Date: Mon, 11 Oct 2021 15:40:40 -0500 Subject: [PATCH 03/29] chore: more macro stuff --- Cargo.lock | 73 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ bin_macro/src/lib.rs | 5 ++- bin_macro/src/stream.rs | 58 +++++++++++++++++++++++++------- src/lib.rs | 7 +++- src/u24.rs | 2 +- tests/macro_tests.rs | 15 +++++++++ tests/tests.rs | 3 ++ 8 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 tests/macro_tests.rs create mode 100644 tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index e2dfc84..67dffbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,11 +2,22 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "bin_macro" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "binary_utils" version = "0.1.2" dependencies = [ + "bin_macro", "byteorder", + "endiannezz", ] [[package]] @@ -14,3 +25,65 @@ name = "byteorder" version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "endiannezz" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f7b3cf44c8349510612cdf1dec49f395f5116e485fcac95d5b89e13ebcd2e0" +dependencies = [ + "endiannezz_derive", + "rustversion", +] + +[[package]] +name = "endiannezz_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf827adb763dfb7dae793493972bd980fea88eacd2e82087f47dcbbfaffc90" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" diff --git a/Cargo.toml b/Cargo.toml index bccde4b..99b5cbf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,3 +7,5 @@ include = ["src/**/*", "README.md"] [dependencies] byteorder = "1.4.3" +bin_macro = { path = "./bin_macro" } +endiannezz = "0.6.5" diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs index 5d9038a..2152677 100644 --- a/bin_macro/src/lib.rs +++ b/bin_macro/src/lib.rs @@ -1,8 +1,11 @@ +#![feature(trace_macros)] +trace_macros!(true); + use proc_macro::TokenStream; use syn::{DeriveInput, parse_macro_input}; mod stream; #[proc_macro_derive(BinaryStream)] pub fn derive_stream(input: TokenStream) -> TokenStream { - stream::derive_stream(parse_macro_input!(input as DeriveInput)).into() + stream::stream_parse(parse_macro_input!(input as DeriveInput)).unwrap().into() } \ No newline at end of file diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 27bb08b..00e7a3e 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -1,35 +1,69 @@ use proc_macro2::{Ident, Literal, TokenStream}; -use quote::quote; -use syn::{Data, DeriveInput, Error, Fields, Result}; +use quote::{ToTokens, quote}; +use syn::{Data, DeriveInput, Error, Fields, Result, Type, token::SelfType}; pub fn stream_parse(input: DeriveInput) -> Result { let name = &input.ident; match input.data { Data::Struct(v) => { + // iterate through struct fields + let (writes, reads) = impl_struct_fields(v.fields); + // return a quote for block impl + dbg!(quote! { + impl Streamable for #name { + fn write(&self, &mut source) { + ( #(#writes)* ) + } - }, - Data::Enum(v) => { + fn read(source: &[u8], position: &mut usize) -> Self { + Self { + ( #(#reads)* ) + } + } + } + }); + Ok(quote! { + impl Streamable for #name { + fn write(&self, &mut source) { + ( #(#writes)* ) + } + fn read(source: &[u8], position: &mut usize) -> Self { + Self { + ( #(#reads)* ) + } + } + } + }) }, - _ => panic!("BinaryStream does not support Type Unions. Use Enums instead.") + Data::Enum(v) => Err(syn::Error::new(name.span(), "BinaryStream does not support Enums. Use Structs instead.")), + Data::Union(_) => Err(syn::Error::new(name.span(), "BinaryStream does not support Type Unions. Use Enums instead.")) } - Ok(quote! {}) } -pub fn impl_struct_fields(fields: Fields) { +pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec) { + let mut writers = Vec::::new(); + let mut readers = Vec::::new(); match fields { Fields::Named(v) => { for field in &v.named { - let field_id = field.ident.as_ref(); - let id = quote!(self.#field_id); - + let field_id = field.ident.as_ref().unwrap(); + let (writer, reader) = impl_streamable_lazy(field_id, &field.ty); + writers.push(writer); + readers.push(reader); } }, Fields::Unnamed(v) => { - + panic!("Can not parse un-named fields at this current point in time.") }, Fields::Unit => { panic!("Can not use uninitalized data values.") } } -} \ No newline at end of file + (writers, readers) +} + +pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { + (quote!(self.#name.write(&mut source);), quote!(#name: #ty::read(&mut source, &mut offset))) +} + diff --git a/src/lib.rs b/src/lib.rs index c8c6ab1..44f6646 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,10 @@ +#![feature(log_syntax)] + use std::io; use byteorder::{ReadBytesExt, WriteBytesExt}; + +// pub use bin_macro::*; + pub mod u24; pub type Stream = io::Cursor>; @@ -8,7 +13,7 @@ pub trait Streamable { /// Writes `self` to the given buffer. fn write(&self, src: &mut Vec); /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self; + fn read(source: &[u8], position: &mut usize) -> Self where Self: Sized; } pub trait BinWrite: io::Write { diff --git a/src/u24.rs b/src/u24.rs index 5bd75ed..8c8cc04 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -160,4 +160,4 @@ impl_primitive_u24!(i8); impl_primitive_u24!(i16); impl_primitive_u24!(i32); impl_primitive_u24!(i64); -impl_primitive_u24!(i128); +impl_primitive_u24!(i128); \ No newline at end of file diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs new file mode 100644 index 0000000..c6af900 --- /dev/null +++ b/tests/macro_tests.rs @@ -0,0 +1,15 @@ +use bin_macro::*; +use binary_utils::Streamable; + +#[derive(BinaryStream)] +pub struct TestPacket { + pub some_int: u8, + pub some_string: u8 +} + +#[test] +fn construct_struct() { + let mut buf = vec![1, 30]; + let pk = TestPacket::read(&buf, &mut 0); + +} \ No newline at end of file diff --git a/tests/tests.rs b/tests/tests.rs new file mode 100644 index 0000000..46e445a --- /dev/null +++ b/tests/tests.rs @@ -0,0 +1,3 @@ +// make a test +mod macro_tests; + From f008cd847eb89a164024de4107f79053cc64b67c Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 11 Oct 2021 22:23:55 -0500 Subject: [PATCH 04/29] chore: Actual working macro --- bin_macro/src/stream.rs | 30 +++++++++++------------------- tests/macro_tests.rs | 2 -- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 00e7a3e..0fb944f 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -1,4 +1,4 @@ -use proc_macro2::{Ident, Literal, TokenStream}; +use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, quote}; use syn::{Data, DeriveInput, Error, Fields, Result, Type, token::SelfType}; @@ -7,30 +7,22 @@ pub fn stream_parse(input: DeriveInput) -> Result { match input.data { Data::Struct(v) => { // iterate through struct fields - let (writes, reads) = impl_struct_fields(v.fields); + let (w, r) = impl_struct_fields(v.fields); + let writes = quote!(#(#w)*); + let reads = quote!(#(#r),*); + // get the visibility etc on each field // return a quote for block impl - dbg!(quote! { - impl Streamable for #name { - fn write(&self, &mut source) { - ( #(#writes)* ) - } - - fn read(source: &[u8], position: &mut usize) -> Self { - Self { - ( #(#reads)* ) - } - } - } - }); + dbg!(&writes); Ok(quote! { + #[automatically_derived] impl Streamable for #name { - fn write(&self, &mut source) { - ( #(#writes)* ) + fn write(&self, &mut source: &mut Vec) { + #writes } fn read(source: &[u8], position: &mut usize) -> Self { Self { - ( #(#reads)* ) + #reads } } } @@ -64,6 +56,6 @@ pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec } pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!(self.#name.write(&mut source);), quote!(#name: #ty::read(&mut source, &mut offset))) + (quote!{ self.#name.write(&mut source); }, quote!(#name: #ty::read(&mut source, &mut position))) } diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index c6af900..8e7db9b 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -1,6 +1,5 @@ use bin_macro::*; use binary_utils::Streamable; - #[derive(BinaryStream)] pub struct TestPacket { pub some_int: u8, @@ -11,5 +10,4 @@ pub struct TestPacket { fn construct_struct() { let mut buf = vec![1, 30]; let pk = TestPacket::read(&buf, &mut 0); - } \ No newline at end of file From 9e9d5327f587cd23e3f70dbfdd67257b52753e3f Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 11 Oct 2021 23:19:02 -0500 Subject: [PATCH 05/29] chore: Finish up basic v2 bin-stream --- bin_macro/src/stream.rs | 8 ++++++-- src/lib.rs | 41 +++++++++++++++++++++++++++++------------ src/u24.rs | 15 +++++++++++++++ tests/macro_tests.rs | 1 + 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 0fb944f..5140029 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -16,11 +16,15 @@ pub fn stream_parse(input: DeriveInput) -> Result { Ok(quote! { #[automatically_derived] impl Streamable for #name { - fn write(&self, &mut source: &mut Vec) { + fn write(&self) -> Vec { + use ::std::io::Write; + let mut writer = Vec::new(); #writes + writer } fn read(source: &[u8], position: &mut usize) -> Self { + use ::std::io::Read; Self { #reads } @@ -56,6 +60,6 @@ pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec } pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ self.#name.write(&mut source); }, quote!(#name: #ty::read(&mut source, &mut position))) + (quote!{ writer.write(&self.#name.write()[..]).unwrap(); }, quote!(#name: #ty::read(&source, position))) } diff --git a/src/lib.rs b/src/lib.rs index 44f6646..d07ff90 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ use std::io; use byteorder::{ReadBytesExt, WriteBytesExt}; +use std::convert::TryInto; // pub use bin_macro::*; @@ -11,21 +12,37 @@ pub type Stream = io::Cursor>; pub trait Streamable { /// Writes `self` to the given buffer. - fn write(&self, src: &mut Vec); + fn write(&self) -> Vec; /// Reads `self` from the given buffer. fn read(source: &[u8], position: &mut usize) -> Self where Self: Sized; } -pub trait BinWrite: io::Write { - #[inline] - fn write_bool(&mut self, value: bool) -> io::Result<()> { - self.write_u8(value.into()) - } +macro_rules! impl_streamable_primitive { + ($ty: ty) => { + impl Streamable for $ty { + fn write(&self) -> Vec { + self.to_be_bytes().to_vec() + } + + fn read(source: &[u8], position: &mut usize) -> Self { + // get the size + let size = ::std::mem::size_of::<$ty>(); + let range = position.clone()..size; + let data = <$ty>::from_be_bytes(source[range].try_into().unwrap()); + *position += size; + data + } + } + }; } -pub trait BinRead: io::Read { - #[inline] - fn read_bool(&mut self) -> bool { - self.read_u8().unwrap_or(0) == 0 - } -} \ No newline at end of file +impl_streamable_primitive!(u8); +impl_streamable_primitive!(u16); +impl_streamable_primitive!(u32); +impl_streamable_primitive!(u64); +impl_streamable_primitive!(u128); +impl_streamable_primitive!(i8); +impl_streamable_primitive!(i16); +impl_streamable_primitive!(i32); +impl_streamable_primitive!(i64); +impl_streamable_primitive!(i128); \ No newline at end of file diff --git a/src/u24.rs b/src/u24.rs index 8c8cc04..82279d7 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -5,7 +5,10 @@ use std::cmp::{Ordering, PartialEq, PartialOrd}; use std::convert::{From, Into}; use std::io; use std::ops::{Add, BitOr, Div, Mul, Sub}; + +use crate::Streamable; /// Base Implementation for a u24 +#[derive(Clone, Copy, Debug)] pub struct u24(u32); // inner is validated impl u24 { @@ -32,6 +35,18 @@ impl u24 { } } +impl Streamable for u24 { + /// Writes `self` to the given buffer. + fn write(&self) -> Vec { + self.to_be_bytes().to_vec().clone() + } + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self { + *position += 2; + Self::from_be_bytes(source) + } +} + pub trait u24Writer: io::Write { #[inline] fn write_u24(&mut self, num: u24) -> io::Result { diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 8e7db9b..3f5a34e 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -10,4 +10,5 @@ pub struct TestPacket { fn construct_struct() { let mut buf = vec![1, 30]; let pk = TestPacket::read(&buf, &mut 0); + assert_eq!(buf, pk.write()) } \ No newline at end of file From 7214f8010a37ddc200e029c7cfc802ffe3de40b6 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 12 Oct 2021 21:00:34 -0500 Subject: [PATCH 06/29] chore: add varint --- Cargo.lock | 28 ------ Cargo.toml | 3 +- src/lib.rs | 54 +++++----- src/u24.rs | 8 +- src/varint.rs | 228 ++++++++++++++++++++++++++++++++++++++++++- tests/macro_tests.rs | 12 +-- tests/tests.rs | 2 +- tests/vec.rs | 10 ++ 8 files changed, 281 insertions(+), 64 deletions(-) create mode 100644 tests/vec.rs diff --git a/Cargo.lock b/Cargo.lock index 67dffbe..dc5cede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,7 +17,6 @@ version = "0.1.2" dependencies = [ "bin_macro", "byteorder", - "endiannezz", ] [[package]] @@ -26,27 +25,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "endiannezz" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f7b3cf44c8349510612cdf1dec49f395f5116e485fcac95d5b89e13ebcd2e0" -dependencies = [ - "endiannezz_derive", - "rustversion", -] - -[[package]] -name = "endiannezz_derive" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf827adb763dfb7dae793493972bd980fea88eacd2e82087f47dcbbfaffc90" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.29" @@ -65,12 +43,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rustversion" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b3909d758bb75c79f23d4736fac9433868679d3ad2ea7a61e3c25cfda9a088" - [[package]] name = "syn" version = "1.0.80" diff --git a/Cargo.toml b/Cargo.toml index 99b5cbf..445a8ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,5 @@ include = ["src/**/*", "README.md"] [dependencies] byteorder = "1.4.3" bin_macro = { path = "./bin_macro" } -endiannezz = "0.6.5" + +[features] diff --git a/src/lib.rs b/src/lib.rs index d07ff90..d7f3ac7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,39 +1,47 @@ #![feature(log_syntax)] -use std::io; -use byteorder::{ReadBytesExt, WriteBytesExt}; use std::convert::TryInto; +use std::io; // pub use bin_macro::*; pub mod u24; +pub mod varint; pub type Stream = io::Cursor>; pub trait Streamable { - /// Writes `self` to the given buffer. - fn write(&self) -> Vec; - /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self where Self: Sized; + /// Writes `self` to the given buffer. + fn write(&self) -> Vec; + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self + where + Self: Sized; } +/// Little Endian Encoding +pub struct LE(pub T); + +/// Big Endian Encoding +pub struct BE(pub T); + macro_rules! impl_streamable_primitive { - ($ty: ty) => { - impl Streamable for $ty { - fn write(&self) -> Vec { - self.to_be_bytes().to_vec() - } - - fn read(source: &[u8], position: &mut usize) -> Self { - // get the size - let size = ::std::mem::size_of::<$ty>(); - let range = position.clone()..size; - let data = <$ty>::from_be_bytes(source[range].try_into().unwrap()); - *position += size; - data - } - } - }; + ($ty: ty) => { + impl Streamable for $ty { + fn write(&self) -> Vec { + self.to_be_bytes().to_vec() + } + + fn read(source: &[u8], position: &mut usize) -> Self { + // get the size + let size = ::std::mem::size_of::<$ty>(); + let range = position.clone()..size; + let data = <$ty>::from_be_bytes(source[range].try_into().unwrap()); + *position += size; + data + } + } + }; } impl_streamable_primitive!(u8); @@ -45,4 +53,4 @@ impl_streamable_primitive!(i8); impl_streamable_primitive!(i16); impl_streamable_primitive!(i32); impl_streamable_primitive!(i64); -impl_streamable_primitive!(i128); \ No newline at end of file +impl_streamable_primitive!(i128); diff --git a/src/u24.rs b/src/u24.rs index 82279d7..58d4f53 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -37,11 +37,11 @@ impl u24 { impl Streamable for u24 { /// Writes `self` to the given buffer. - fn write(&self) -> Vec { + fn write(&self) -> Vec { self.to_be_bytes().to_vec().clone() } - /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self { + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self { *position += 2; Self::from_be_bytes(source) } @@ -175,4 +175,4 @@ impl_primitive_u24!(i8); impl_primitive_u24!(i16); impl_primitive_u24!(i32); impl_primitive_u24!(i64); -impl_primitive_u24!(i128); \ No newline at end of file +impl_primitive_u24!(i128); diff --git a/src/varint.rs b/src/varint.rs index d7490db..d871638 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -1 +1,227 @@ -pub struct VarInt(T, usize); +use crate::Streamable; +use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; +use std::cmp::{Ordering, PartialEq, PartialOrd}; +use std::convert::{From, Into}; +use std::io::{self, Cursor, Read, Write}; +use std::ops::{Add, BitOr, Div, Mul, Sub}; +/// A minecraft specific unsized integer +/// A varint can be one of `32` and `64` bits +#[derive(Clone, Copy, Debug)] +pub struct VarInt(pub T); + +pub const VAR_INT_32_BYTE_MAX: usize = 5; +pub const VAR_INT_64_BYTE_MAX: usize = 10; + +macro_rules! varint_impl_generic { + ($ty:ty) => { + impl VarInt<$ty> { + /// Encodes the var_int into Big Endian Bytes + pub fn to_be_bytes(self) -> Vec { + self.to_bytes_be() + } + + pub fn to_le_bytes(self) -> Vec { + let mut v = Vec::::new(); + let bytes = self.to_bytes_be(); + for x in (0..bytes.len()).rev() { + v.push(*bytes.get(x).unwrap()); + } + v + } + + pub fn get_byte_length(self) -> u8 { + self.to_be_bytes().len() as u8 + } + + fn to_bytes_be(self) -> Vec { + let mut to_write: $ty = self.0; + let mut buf: Vec = Vec::new(); + + // while there is more than a single byte to write + while to_write >= 0x80 { + // write at most a byte, to account for overflow + buf.write_u8(to_write as u8 | 0x80).unwrap(); + to_write >>= 7; + } + + buf.write_u8(to_write as u8).unwrap(); + buf + } + + pub fn from_be_bytes(stream: &mut Cursor>) -> Self { + let mut value: $ty = 0; + + for x in (0..35).step_by(7) { + let byte = stream.read_u8().unwrap(); + value |= (byte & 0x7f) as $ty << x; + + // if the byte is a full length of a byte + // we can assume we are done + if byte & 0x80 == 0 { + break; + } + } + + VarInt::<$ty>(value) + } + + // pub fn from_le_bytes(bytes: &[u8]) -> Self { + // <$ty>::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() + // } + pub fn is_var_int(_: $ty) -> bool { + true + } + } + + impl Streamable for VarInt<$ty> { + /// Writes `self` to the given buffer. + fn write(&self) -> Vec { + self.to_be_bytes().to_vec().clone() + } + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self { + *position += 2; + // Self::from_be_bytes(source) + Self(0) + } + } + }; +} +macro_rules! varint_impl_generic64 { + ($ty:ty) => { + impl VarInt<$ty> { + /// Encodes the var_int into Big Endian Bytes + pub fn to_be_bytes(self) -> Vec { + self.to_bytes_be() + } + + pub fn to_le_bytes(self) -> Vec { + let mut v = Vec::::new(); + let bytes = self.to_bytes_be(); + for x in (0..bytes.len()).rev() { + v.push(*bytes.get(x).unwrap()); + } + v + } + + pub fn get_byte_length(self) -> u8 { + self.to_be_bytes().len() as u8 + } + + fn to_bytes_be(self) -> Vec { + let mut to_write: $ty = self.0; + let mut buf: Vec = Vec::new(); + + // while there is more than a single byte to write + while to_write >= 0x80 { + // write at most a byte, to account for overflow + buf.write_u8(to_write as u8 | 0x80).unwrap(); + to_write >>= 7; + } + + buf.write_u8(to_write as u8).unwrap(); + buf + } + + // pub fn from_be_bytes(bytes: &[u8]) -> Self { + // <$ty>::from_be_bytes([bytes[0], bytes[1], bytes[2], 0]).into() + // } + + // pub fn from_le_bytes(bytes: &[u8]) -> Self { + // <$ty>::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() + // } + pub fn is_var_int(_: $ty) -> bool { + true + } + } + + impl Streamable for VarInt<$ty> { + /// Writes `self` to the given buffer. + fn write(&self) -> Vec { + self.to_be_bytes().to_vec().clone() + } + /// Reads `self` from the given buffer. + fn read(source: &[u8], position: &mut usize) -> Self { + *position += 2; + // Self::from_be_bytes(source) + Self(0) + } + } + }; +} +varint_impl_generic!(u32); +varint_impl_generic!(i32); +varint_impl_generic64!(u64); +varint_impl_generic64!(i64); + +macro_rules! impl_primitive_VarInt { + ($ty:ty, $vk:ty) => { + impl From<$ty> for VarInt<$vk> { + fn from(value: $ty) -> Self { + if !VarInt::<$vk>::is_var_int(value as $vk) { + panic!( + "Can not convert a number larger than the bounds of a VarInt into a VarInt" + ) + } else { + VarInt(value as $vk) + } + } + } + + impl BitOr<$ty> for VarInt<$vk> { + type Output = Self; + + fn bitor(self, rhs: $ty) -> Self::Output { + VarInt(self.0 | rhs as $vk) + } + } + + impl Into<$ty> for VarInt<$vk> { + fn into(self) -> $ty { + self.0 as $ty + } + } + + impl Add<$ty> for VarInt<$vk> { + type Output = Self; + + fn add(self, other: $ty) -> Self::Output { + VarInt(self.0 + other as $vk) + } + } + + impl Mul<$ty> for VarInt<$vk> { + type Output = Self; + + fn mul(self, other: $ty) -> Self::Output { + VarInt(self.0 * other as $vk) + } + } + + impl Sub<$ty> for VarInt<$vk> { + type Output = Self; + + fn sub(self, other: $ty) -> Self::Output { + VarInt(self.0 - other as $vk) + } + } + + impl Div<$ty> for VarInt<$vk> { + type Output = Self; + + fn div(self, other: $ty) -> Self::Output { + VarInt(self.0 / other as $vk) + } + } + }; +} +impl_primitive_VarInt!(u8, u32); +impl_primitive_VarInt!(u16, u32); +impl_primitive_VarInt!(u32, u32); +impl_primitive_VarInt!(u64, u32); +impl_primitive_VarInt!(u128, u32); +impl_primitive_VarInt!(i8, u64); +impl_primitive_VarInt!(i16, u64); +impl_primitive_VarInt!(i32, u64); +impl_primitive_VarInt!(i64, u64); +impl_primitive_VarInt!(i128, u64); diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 3f5a34e..0600cd1 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -2,13 +2,13 @@ use bin_macro::*; use binary_utils::Streamable; #[derive(BinaryStream)] pub struct TestPacket { - pub some_int: u8, - pub some_string: u8 + pub some_int: u8, + pub some_string: u8, } #[test] fn construct_struct() { - let mut buf = vec![1, 30]; - let pk = TestPacket::read(&buf, &mut 0); - assert_eq!(buf, pk.write()) -} \ No newline at end of file + let mut buf = vec![1, 30]; + let pk = TestPacket::read(&buf, &mut 0); + assert_eq!(buf, pk.write()) +} diff --git a/tests/tests.rs b/tests/tests.rs index 46e445a..8c1e280 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,3 +1,3 @@ // make a test mod macro_tests; - +mod vec; diff --git a/tests/vec.rs b/tests/vec.rs new file mode 100644 index 0000000..a29ecf2 --- /dev/null +++ b/tests/vec.rs @@ -0,0 +1,10 @@ +use bin_macro::*; +use binary_utils::varint::VarInt; +use std::io::Cursor; + +#[test] +fn test_varint() { + let v = VarInt::(25565); + let val: Vec = vec![221, 199, 1]; + dbg!(&v.to_be_bytes()); +} From c5456e459bc109470931ec3ef3efb6ef24376548 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 12 Oct 2021 21:07:52 -0500 Subject: [PATCH 07/29] chore: fixes --- bin_macro/src/stream.rs | 8 ++++---- src/varint.rs | 34 +++++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 5140029..5f8997c 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -1,6 +1,6 @@ use proc_macro2::{Ident, TokenStream}; -use quote::{ToTokens, quote}; -use syn::{Data, DeriveInput, Error, Fields, Result, Type, token::SelfType}; +use quote::quote; +use syn::{Data, DeriveInput, Fields, Result, Type}; pub fn stream_parse(input: DeriveInput) -> Result { let name = &input.ident; @@ -32,7 +32,7 @@ pub fn stream_parse(input: DeriveInput) -> Result { } }) }, - Data::Enum(v) => Err(syn::Error::new(name.span(), "BinaryStream does not support Enums. Use Structs instead.")), + Data::Enum(_v) => Err(syn::Error::new(name.span(), "BinaryStream does not support Enums. Use Structs instead.")), Data::Union(_) => Err(syn::Error::new(name.span(), "BinaryStream does not support Type Unions. Use Enums instead.")) } } @@ -49,7 +49,7 @@ pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec readers.push(reader); } }, - Fields::Unnamed(v) => { + Fields::Unnamed(_v) => { panic!("Can not parse un-named fields at this current point in time.") }, Fields::Unit => { diff --git a/src/varint.rs b/src/varint.rs index d871638..e7d7b3b 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -1,8 +1,7 @@ use crate::Streamable; -use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; -use std::cmp::{Ordering, PartialEq, PartialOrd}; +use byteorder::{ReadBytesExt, WriteBytesExt}; use std::convert::{From, Into}; -use std::io::{self, Cursor, Read, Write}; +use std::io::Cursor; use std::ops::{Add, BitOr, Div, Mul, Sub}; /// A minecraft specific unsized integer /// A varint can be one of `32` and `64` bits @@ -80,9 +79,9 @@ macro_rules! varint_impl_generic { } /// Reads `self` from the given buffer. fn read(source: &[u8], position: &mut usize) -> Self { - *position += 2; - // Self::from_be_bytes(source) - Self(0) + let v = Self::from_be_bytes(&mut Cursor::new(source.to_vec())); + *position += v.get_byte_length() as usize; + v } } }; @@ -123,6 +122,23 @@ macro_rules! varint_impl_generic64 { buf } + pub fn from_be_bytes(stream: &mut Cursor>) -> Self { + let mut value: $ty = 0; + + for x in (0..70).step_by(7) { + let byte = stream.read_u8().unwrap(); + value |= (byte & 0x7f) as $ty << x; + + // if the byte is a full length of a byte + // we can assume we are done + if byte & 0x80 == 0 { + break; + } + } + + VarInt::<$ty>(value) + } + // pub fn from_be_bytes(bytes: &[u8]) -> Self { // <$ty>::from_be_bytes([bytes[0], bytes[1], bytes[2], 0]).into() // } @@ -142,9 +158,9 @@ macro_rules! varint_impl_generic64 { } /// Reads `self` from the given buffer. fn read(source: &[u8], position: &mut usize) -> Self { - *position += 2; - // Self::from_be_bytes(source) - Self(0) + let v = Self::from_be_bytes(&mut Cursor::new(source.to_vec())); + *position += v.get_byte_length() as usize; + v } } }; From 68b51f86946c904315d96cd213efb331c8b6c080 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Wed, 13 Oct 2021 22:39:01 -0500 Subject: [PATCH 08/29] chore: Add write methods to varint --- src/lib.rs | 47 +++++++++++++++++++++++++++++++++- src/varint.rs | 60 +++++++++++++++++++++++++++++++++++++++++--- tests/macro_tests.rs | 3 ++- tests/vec.rs | 1 + 4 files changed, 106 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d7f3ac7..bf7308b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,8 @@ #![feature(log_syntax)] use std::convert::TryInto; -use std::io; +use std::io::{self, Read, Write}; +use byteorder::{ReadBytesExt, WriteBytesExt}; // pub use bin_macro::*; @@ -10,6 +11,8 @@ pub mod varint; pub type Stream = io::Cursor>; +use varint::{VarInt, VarIntReader, VarIntWriter}; + pub trait Streamable { /// Writes `self` to the given buffer. fn write(&self) -> Vec; @@ -54,3 +57,45 @@ impl_streamable_primitive!(i16); impl_streamable_primitive!(i32); impl_streamable_primitive!(i64); impl_streamable_primitive!(i128); + +// implements bools +impl Streamable for bool { + fn write(&self) -> Vec { + vec![if *self { 1 } else { 0 }] + } + + fn read(source: &[u8], position: &mut usize) -> Self { + let v = source[*position] == 1; + *position += 1; + v + } +} + +// impl Streamable for Vec +// where +// T: Streamable { +// fn write(&self) -> Vec { +// // write the length as a varint +// let mut v: Vec = Vec::new(); +// v.write_all(&VarInt(v.len() as u32).to_be_bytes()[..]).unwrap(); +// for x in self.iter() { +// v.extend(x.write().iter()); +// } +// v +// } + +// fn read(source: &[u8], position: &mut usize) -> Self { +// // read a var_int +// let mut ret: Vec = Vec::new(); +// let varint = VarInt::::from_be_bytes(source); +// let length: u32 = varint.into(); + +// *position += varint.get_byte_length() as usize; + +// // read each length +// for _ in 0..length { +// ret.push(T::read(&source, position)); +// } +// ret +// } +// } \ No newline at end of file diff --git a/src/varint.rs b/src/varint.rs index e7d7b3b..f40d117 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -1,13 +1,21 @@ use crate::Streamable; use byteorder::{ReadBytesExt, WriteBytesExt}; use std::convert::{From, Into}; -use std::io::Cursor; +use std::io::{self, Cursor}; use std::ops::{Add, BitOr, Div, Mul, Sub}; /// A minecraft specific unsized integer /// A varint can be one of `32` and `64` bits #[derive(Clone, Copy, Debug)] pub struct VarInt(pub T); +pub trait VarIntWriter: io::Write { + fn write_var_int(&mut self, num: VarInt) -> io::Result; +} + +pub trait VarIntReader: io::Read { + fn read_var_int(&mut self) -> io::Result>; +} + pub const VAR_INT_32_BYTE_MAX: usize = 5; pub const VAR_INT_64_BYTE_MAX: usize = 10; @@ -47,7 +55,7 @@ macro_rules! varint_impl_generic { buf } - pub fn from_be_bytes(stream: &mut Cursor>) -> Self { + pub fn from_be_bytes_cursor(stream: &mut Cursor>) -> Self { let mut value: $ty = 0; for x in (0..35).step_by(7) { @@ -64,6 +72,24 @@ macro_rules! varint_impl_generic { VarInt::<$ty>(value) } + pub fn from_be_bytes(bstream: &[u8]) -> Self { + let mut stream = Cursor::new(bstream); + let mut value: $ty = 0; + + for x in (0..35).step_by(7) { + let byte = stream.read_u8().unwrap(); + value |= (byte & 0x7f) as $ty << x; + + // if the byte is a full length of a byte + // we can assume we are done + if byte & 0x80 == 0 { + break; + } + } + + VarInt::<$ty>(value) + } + // pub fn from_le_bytes(bytes: &[u8]) -> Self { // <$ty>::from_be_bytes([0, bytes[1], bytes[2], bytes[3]]).into() // } @@ -79,11 +105,39 @@ macro_rules! varint_impl_generic { } /// Reads `self` from the given buffer. fn read(source: &[u8], position: &mut usize) -> Self { - let v = Self::from_be_bytes(&mut Cursor::new(source.to_vec())); + let v = Self::from_be_bytes(source); *position += v.get_byte_length() as usize; v } } + + impl VarIntReader<$ty> for dyn io::Read { + #[inline] + fn read_var_int(&mut self) -> io::Result> { + let mut value: $ty = 0; + + for x in (0..35).step_by(7) { + let byte = self.read_u8().unwrap(); + value |= (byte & 0x7f) as $ty << x; + + // if the byte is a full length of a byte + // we can assume we are done + if byte & 0x80 == 0 { + break; + } + } + + Ok(VarInt::<$ty>(value)) + } + } + + impl VarIntWriter<$ty> for dyn io::Write { + #[inline] + fn write_var_int(&mut self, num: VarInt<$ty>) -> io::Result { + self.write_all(&num.to_be_bytes()[..]).unwrap(); + Ok(num.get_byte_length() as usize) + } + } }; } macro_rules! varint_impl_generic64 { diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 0600cd1..0877089 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -1,9 +1,10 @@ use bin_macro::*; -use binary_utils::Streamable; +use binary_utils::{Streamable, varint::VarInt}; #[derive(BinaryStream)] pub struct TestPacket { pub some_int: u8, pub some_string: u8, + pub unknown_size: VarInt:: } #[test] diff --git a/tests/vec.rs b/tests/vec.rs index a29ecf2..6ad29e2 100644 --- a/tests/vec.rs +++ b/tests/vec.rs @@ -6,5 +6,6 @@ use std::io::Cursor; fn test_varint() { let v = VarInt::(25565); let val: Vec = vec![221, 199, 1]; + dbg!(VarInt::::from_be_bytes(&[255, 255, 255, 1][..])); dbg!(&v.to_be_bytes()); } From 3eade59313bb6bf8a5fd3cd6cfcb25f08f244bc7 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Fri, 15 Oct 2021 23:51:30 -0500 Subject: [PATCH 09/29] chore; fix --- src/lib.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bf7308b..5b61cae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,17 +1,19 @@ #![feature(log_syntax)] use std::convert::TryInto; -use std::io::{self, Read, Write}; -use byteorder::{ReadBytesExt, WriteBytesExt}; +use std::io; -// pub use bin_macro::*; +pub use bin_macro::*; pub mod u24; pub mod varint; -pub type Stream = io::Cursor>; +pub use self::{ + u24::*, + varint::* +}; -use varint::{VarInt, VarIntReader, VarIntWriter}; +pub type Stream = io::Cursor>; pub trait Streamable { /// Writes `self` to the given buffer. From 4b504d55c89c97d4a787acfc9c077fc288ba7721 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Sat, 16 Oct 2021 23:36:01 -0500 Subject: [PATCH 10/29] chore: Fix struct reading/writing and change method name conventions --- bin_macro/src/stream.rs | 74 ++++++++++++++++++++++++++++++++++++++--- src/lib.rs | 69 ++++++++++++++++++++++++++++++-------- src/u24.rs | 4 +-- src/util.rs | 2 ++ src/varint.rs | 8 ++--- tests/macro_tests.rs | 12 ++++--- 6 files changed, 141 insertions(+), 28 deletions(-) create mode 100644 src/util.rs diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 5f8997c..e5a3cc4 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -16,15 +16,56 @@ pub fn stream_parse(input: DeriveInput) -> Result { Ok(quote! { #[automatically_derived] impl Streamable for #name { - fn write(&self) -> Vec { + fn parse(&self) -> Vec { use ::std::io::Write; + use binary_utils::varint::{VarInt, VarIntWriter}; + use binary_utils::u24::{u24, u24Writer}; let mut writer = Vec::new(); #writes writer } - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { use ::std::io::Read; + use binary_utils::varint::{VarInt, VarIntReader}; + use binary_utils::u24::{u24, u24Reader}; + + Self { + #reads + } + } + } + }) + }, + Data::Enum(data) => { + let mut write_collective = Vec::::new(); + let mut read_collective = Vec::::new(); + for var in data.variants.iter() { + let (w, r) = impl_named_fields(&var.ident, var.fields.clone()); + write_collective.extend(w); + read_collective.extend(r); + } + let writes = quote!(#(#write_collective)*); + let reads = quote!(#(#read_collective),*); + // get the visibility etc on each field + // return a quote for block impl + Ok(quote! { + #[automatically_derived] + impl Streamable for #name { + fn parse(&self) -> Vec { + use ::std::io::Write; + use binary_utils::varint::{VarInt, VarIntWriter}; + use binary_utils::u24::{u24, u24Writer}; + let mut writer = Vec::new(); + #writes + writer + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + use ::std::io::Read; + use binary_utils::varint::{VarInt, VarIntReader}; + use binary_utils::u24::{u24, u24Reader}; + Self { #reads } @@ -32,7 +73,6 @@ pub fn stream_parse(input: DeriveInput) -> Result { } }) }, - Data::Enum(_v) => Err(syn::Error::new(name.span(), "BinaryStream does not support Enums. Use Structs instead.")), Data::Union(_) => Err(syn::Error::new(name.span(), "BinaryStream does not support Type Unions. Use Enums instead.")) } } @@ -59,7 +99,33 @@ pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec (writers, readers) } +pub fn impl_named_fields(name: &Ident, fields: Fields) -> (Vec, Vec) { + let mut writers = Vec::::new(); + let mut readers = Vec::::new(); + match fields { + Fields::Named(v) => { + for field in &v.named { + let field_id = field.ident.as_ref().unwrap(); + let (writer, reader) = impl_streamable_lazy_named(name, field_id, &field.ty); + writers.push(writer); + readers.push(reader); + } + }, + Fields::Unnamed(_v) => { + panic!("Can not parse un-named fields at this current point in time.") + }, + Fields::Unit => { + panic!("Can not use uninitalized data values.") + } + } + (writers, readers) +} + + pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ writer.write(&self.#name.write()[..]).unwrap(); }, quote!(#name: #ty::read(&source, position))) + (quote!{ writer.write(&self.#name.parse()[..]); }, quote!(#name: #ty::compose(&source, position))) } +pub fn impl_streamable_lazy_named(name: &Ident, named: &Ident, ty: &Type) -> (TokenStream, TokenStream) { + (quote!{ writer.write(&self.#name.named.parse()[..]); }, quote!(#name: #ty::compose(&source, position))) +} diff --git a/src/lib.rs b/src/lib.rs index 5b61cae..a1229e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,20 +6,18 @@ use std::io; pub use bin_macro::*; pub mod u24; +pub mod util; pub mod varint; -pub use self::{ - u24::*, - varint::* -}; +pub use self::{u24::*, varint::*}; pub type Stream = io::Cursor>; pub trait Streamable { /// Writes `self` to the given buffer. - fn write(&self) -> Vec; + fn parse(&self) -> Vec; /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self + fn compose(source: &[u8], position: &mut usize) -> Self where Self: Sized; } @@ -33,15 +31,15 @@ pub struct BE(pub T); macro_rules! impl_streamable_primitive { ($ty: ty) => { impl Streamable for $ty { - fn write(&self) -> Vec { + fn parse(&self) -> Vec { self.to_be_bytes().to_vec() } - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { // get the size let size = ::std::mem::size_of::<$ty>(); - let range = position.clone()..size; - let data = <$ty>::from_be_bytes(source[range].try_into().unwrap()); + let range = position.clone()..(size + position.clone()); + let data = <$ty>::from_be_bytes(source.get(range).unwrap().try_into().unwrap()); *position += size; data } @@ -62,17 +60,62 @@ impl_streamable_primitive!(i128); // implements bools impl Streamable for bool { - fn write(&self) -> Vec { + fn parse(&self) -> Vec { vec![if *self { 1 } else { 0 }] } - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { let v = source[*position] == 1; *position += 1; v } } +macro_rules! impl_streamable_vec_primitive { + ($ty: ty) => { + impl Streamable for Vec<$ty> { + fn parse(&self) -> Vec { + use ::std::io::Write; + // write the length as a varint + let mut v: Vec = Vec::new(); + v.write_all(&VarInt(v.len() as u32).to_be_bytes()[..]) + .unwrap(); + for x in self.iter() { + v.extend(x.parse().iter()); + } + v + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + // use ::std::io::Read; + // read a var_int + let mut ret: Vec<$ty> = Vec::new(); + let varint = VarInt::::from_be_bytes(source); + let length: u32 = varint.into(); + + *position += varint.get_byte_length() as usize; + + // read each length + for _ in 0..length { + ret.push(<$ty>::compose(&source, position)); + } + ret + } + } + }; +} + +impl_streamable_vec_primitive!(u8); +impl_streamable_vec_primitive!(u16); +impl_streamable_vec_primitive!(u32); +impl_streamable_vec_primitive!(u64); +impl_streamable_vec_primitive!(u128); +impl_streamable_vec_primitive!(i8); +impl_streamable_vec_primitive!(i16); +impl_streamable_vec_primitive!(i32); +impl_streamable_vec_primitive!(i64); +impl_streamable_vec_primitive!(i128); + // impl Streamable for Vec // where // T: Streamable { @@ -100,4 +143,4 @@ impl Streamable for bool { // } // ret // } -// } \ No newline at end of file +// } diff --git a/src/u24.rs b/src/u24.rs index 58d4f53..1c0a8aa 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -37,11 +37,11 @@ impl u24 { impl Streamable for u24 { /// Writes `self` to the given buffer. - fn write(&self) -> Vec { + fn parse(&self) -> Vec { self.to_be_bytes().to_vec().clone() } /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { *position += 2; Self::from_be_bytes(source) } diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..4f6457f --- /dev/null +++ b/src/util.rs @@ -0,0 +1,2 @@ +// This file contains utilities for encoding and decoding streams. +use crate::Streamable; \ No newline at end of file diff --git a/src/varint.rs b/src/varint.rs index f40d117..ee32e3e 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -100,11 +100,11 @@ macro_rules! varint_impl_generic { impl Streamable for VarInt<$ty> { /// Writes `self` to the given buffer. - fn write(&self) -> Vec { + fn parse(&self) -> Vec { self.to_be_bytes().to_vec().clone() } /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { let v = Self::from_be_bytes(source); *position += v.get_byte_length() as usize; v @@ -207,11 +207,11 @@ macro_rules! varint_impl_generic64 { impl Streamable for VarInt<$ty> { /// Writes `self` to the given buffer. - fn write(&self) -> Vec { + fn parse(&self) -> Vec { self.to_be_bytes().to_vec().clone() } /// Reads `self` from the given buffer. - fn read(source: &[u8], position: &mut usize) -> Self { + fn compose(source: &[u8], position: &mut usize) -> Self { let v = Self::from_be_bytes(&mut Cursor::new(source.to_vec())); *position += v.get_byte_length() as usize; v diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 0877089..70a2b7d 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -1,15 +1,17 @@ use bin_macro::*; use binary_utils::{Streamable, varint::VarInt}; -#[derive(BinaryStream)] +#[derive(Debug, BinaryStream)] pub struct TestPacket { pub some_int: u8, pub some_string: u8, - pub unknown_size: VarInt:: + // pub unknown_size: VarInt } #[test] fn construct_struct() { - let mut buf = vec![1, 30]; - let pk = TestPacket::read(&buf, &mut 0); - assert_eq!(buf, pk.write()) + let mut buf = vec![1, 30, 1, 30]; + let pk = TestPacket::compose(&buf, &mut 0); + dbg!(&pk); + dbg!(pk.parse()); + assert_eq!(buf, pk.parse()) } From b93a49fe8d809d72f1a929a172283a0281f83ae8 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Sun, 17 Oct 2021 00:21:00 -0500 Subject: [PATCH 11/29] chore: Bump version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc5cede..327baa4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "binary_utils" -version = "0.1.2" +version = "0.2.0" dependencies = [ "bin_macro", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 445a8ae..fd8e239 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "binary_utils" -version = "0.1.2" +version = "0.2.0" authors = ["Bavfalcon9 "] edition = "2018" include = ["src/**/*", "README.md"] From aba94e802940c0fe38c78a1e596d7b28b2a6f1aa Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 17:41:58 -0500 Subject: [PATCH 12/29] feat(util): Add read/writing for String" --- bin_macro/src/stream.rs | 4 ++-- src/util.rs | 27 +++++++++++++++++++++- tests/macro_tests.rs | 51 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index e5a3cc4..4ea8a69 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -123,9 +123,9 @@ pub fn impl_named_fields(name: &Ident, fields: Fields) -> (Vec, Vec pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ writer.write(&self.#name.parse()[..]); }, quote!(#name: #ty::compose(&source, position))) + (quote!{ writer.write(&self.#name.parse()[..]).unwrap(); }, quote!(#name: #ty::compose(&source, position))) } pub fn impl_streamable_lazy_named(name: &Ident, named: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ writer.write(&self.#name.named.parse()[..]); }, quote!(#name: #ty::compose(&source, position))) + (quote!{ writer.write(&self.#name.named.parse()[..]).unwrap(); }, quote!(#name: #ty::compose(&source, position))) } diff --git a/src/util.rs b/src/util.rs index 4f6457f..f7f8724 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,2 +1,27 @@ // This file contains utilities for encoding and decoding streams. -use crate::Streamable; \ No newline at end of file +use crate::Streamable; +use std::io::{Cursor, Write}; +use byteorder::{BE, ReadBytesExt, WriteBytesExt}; + +impl Streamable for String { + fn parse(&self) -> Vec { + let mut buffer = Vec::::new(); + buffer.write_u16::(self.len() as u16).unwrap(); + buffer.write_all(self.as_bytes()).unwrap(); + buffer + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + let mut stream = Cursor::new(source); + stream.set_position(position.clone() as u64); + // Maybe do this in the future? + let len: usize = stream.read_u16::().unwrap().into(); + + unsafe { + // todo: Remove this nasty hack. + // todo: The hack being, remove the 2 from indexing on read_short + // todo: And utilize stream. + String::from_utf8_unchecked(stream.get_ref()[2..len].to_vec()) + } + } +} \ No newline at end of file diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 70a2b7d..84bfceb 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -1,5 +1,5 @@ use bin_macro::*; -use binary_utils::{Streamable, varint::VarInt}; +use binary_utils::{Streamable}; #[derive(Debug, BinaryStream)] pub struct TestPacket { pub some_int: u8, @@ -9,9 +9,52 @@ pub struct TestPacket { #[test] fn construct_struct() { - let mut buf = vec![1, 30, 1, 30]; + let buf = vec![1, 30]; let pk = TestPacket::compose(&buf, &mut 0); - dbg!(&pk); - dbg!(pk.parse()); assert_eq!(buf, pk.parse()) } + + +#[test] +fn write_string() { + let string = String::from("Hello world!"); + let hello_world_vec= vec![ + 0, + 12, + 72, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, + 33 + ]; + assert_eq!(hello_world_vec, string.parse()); +} + +#[test] +fn read_string() { + let hello_world_vec= vec![ + 0, + 12, + 72, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, + 33 + ]; + let string = String::compose(&hello_world_vec[..], &mut 0); + assert_eq!("Hello world!".to_string(), string); +} \ No newline at end of file From c9fd943167e345d048af8ce52b0b243a53cf156b Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 17:44:45 -0500 Subject: [PATCH 13/29] chore: Move from util to lib --- src/lib.rs | 54 ++++++++++++++++++++++++++++++++++++++++------------- src/util.rs | 25 ------------------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a1229e1..4e95710 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,9 @@ use std::io; pub use bin_macro::*; +use std::io::{Cursor, Write}; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; + pub mod u24; pub mod util; pub mod varint; @@ -58,19 +61,6 @@ impl_streamable_primitive!(i32); impl_streamable_primitive!(i64); impl_streamable_primitive!(i128); -// implements bools -impl Streamable for bool { - fn parse(&self) -> Vec { - vec![if *self { 1 } else { 0 }] - } - - fn compose(source: &[u8], position: &mut usize) -> Self { - let v = source[*position] == 1; - *position += 1; - v - } -} - macro_rules! impl_streamable_vec_primitive { ($ty: ty) => { impl Streamable for Vec<$ty> { @@ -116,6 +106,44 @@ impl_streamable_vec_primitive!(i32); impl_streamable_vec_primitive!(i64); impl_streamable_vec_primitive!(i128); + +// implements bools +impl Streamable for bool { + fn parse(&self) -> Vec { + vec![if *self { 1 } else { 0 }] + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + let v = source[*position] == 1; + *position += 1; + v + } +} + +impl Streamable for String { + fn parse(&self) -> Vec { + let mut buffer = Vec::::new(); + buffer.write_u16::(self.len() as u16).unwrap(); + buffer.write_all(self.as_bytes()).unwrap(); + buffer + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + let mut stream = Cursor::new(source); + stream.set_position(position.clone() as u64); + // Maybe do this in the future? + let len: usize = stream.read_u16::().unwrap().into(); + + unsafe { + // todo: Remove this nasty hack. + // todo: The hack being, remove the 2 from indexing on read_short + // todo: And utilize stream. + String::from_utf8_unchecked(stream.get_ref()[2..len].to_vec()) + } + } +} + + // impl Streamable for Vec // where // T: Streamable { diff --git a/src/util.rs b/src/util.rs index f7f8724..11a4c2f 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,27 +1,2 @@ // This file contains utilities for encoding and decoding streams. use crate::Streamable; -use std::io::{Cursor, Write}; -use byteorder::{BE, ReadBytesExt, WriteBytesExt}; - -impl Streamable for String { - fn parse(&self) -> Vec { - let mut buffer = Vec::::new(); - buffer.write_u16::(self.len() as u16).unwrap(); - buffer.write_all(self.as_bytes()).unwrap(); - buffer - } - - fn compose(source: &[u8], position: &mut usize) -> Self { - let mut stream = Cursor::new(source); - stream.set_position(position.clone() as u64); - // Maybe do this in the future? - let len: usize = stream.read_u16::().unwrap().into(); - - unsafe { - // todo: Remove this nasty hack. - // todo: The hack being, remove the 2 from indexing on read_short - // todo: And utilize stream. - String::from_utf8_unchecked(stream.get_ref()[2..len].to_vec()) - } - } -} \ No newline at end of file From e799bb355ef985765f2f29a8f1d0ef733c73b8ef Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 17:58:23 -0500 Subject: [PATCH 14/29] chore: Fix read/write for string --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 4e95710..e5637f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -138,7 +138,7 @@ impl Streamable for String { // todo: Remove this nasty hack. // todo: The hack being, remove the 2 from indexing on read_short // todo: And utilize stream. - String::from_utf8_unchecked(stream.get_ref()[2..len].to_vec()) + String::from_utf8_unchecked(stream.get_ref()[2..len + stream.position() as usize].to_vec()) } } } From 9708ce736b8456a7a5d03ae926d7ee3e48f35075 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 19:14:21 -0500 Subject: [PATCH 15/29] feat: Add socketaddr support --- src/lib.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e5637f6..1aca5c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,12 @@ #![feature(log_syntax)] -use std::convert::TryInto; +use std::{convert::TryInto, net::SocketAddr}; use std::io; +use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; pub use bin_macro::*; -use std::io::{Cursor, Write}; +use std::io::{Cursor, Write, Read}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; pub mod u24; @@ -144,6 +145,87 @@ impl Streamable for String { } +impl Streamable for SocketAddr { + fn parse(&self) -> Vec { + let mut stream = Vec::::new(); + match *self { + Self::V4(_) => { + stream.write_u8(4).unwrap(); + let partstr = self.to_string(); + let parts: Vec<&str> = partstr.split(".").collect(); + for part in parts { + let mask = u8::from_str_radix(part, 10).unwrap(); + stream.write_u8(mask).unwrap(); + } + stream.write_u16::(self.port()).expect("Could not write port to stream."); + stream + }, + Self::V6(addr) => { + stream.write_u8(6).unwrap(); + // family? or length?? + stream.write_u16::(0).unwrap(); + // port + stream.write_u16::(self.port()).unwrap(); + // flow + stream.write_u32::(addr.flowinfo()).unwrap(); + // actual address here + stream.write(&addr.ip().octets()).unwrap(); + // scope + stream.write_u32::(addr.scope_id()).unwrap(); + stream + } + } + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + let mut stream = Cursor::new(source); + stream.set_position(*position as u64); + match stream.read_u8().unwrap() { + 4 => { + let from = stream.position() as usize; + let to = stream.position() as usize + 4; + let parts = &source[from..to]; + stream.set_position(to as u64); + let port = stream.read_u16::().unwrap(); + SocketAddr::new(IpAddr::from([parts[0], parts[1], parts[2], parts[3]]), port) + }, + 6 => { + let _family = stream.read_u16::().unwrap(); + let port = stream.read_u16::().unwrap(); + let flow = stream.read_u32::().unwrap(); + let mut parts: [u8; 16] = [0; 16]; + stream.read(&mut parts).unwrap(); + // we need to read parts into address + let address = { + let mut s = Cursor::new(parts); + let (a, b, c, d, e, f, g, h) = ( + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + s.read_u16::().unwrap(), + ); + Ipv6Addr::new(a, b, c, d, e, f, g, h) + }; + let scope = stream.read_u32::().unwrap(); + SocketAddr::from(SocketAddrV6::new(address, port, flow, scope)) + }, + _ => panic!("Unknown Address type!") + } + // let addr_type = self.read_byte(); + // if addr_type == 4 { + // let parts = self.read_slice(Some(4 as usize)); + // let port = self.read_ushort(); + // SocketAddr::new(IpAddr::from([parts[0], parts[1], parts[2], parts[3]]), port) + // } else { + // SocketAddr::new(IpAddr::from([0, 0, 0, 0]), 0) + // } + } +} + // impl Streamable for Vec // where // T: Streamable { From 18ca29ed921f94855622a01704bf6413601a935e Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 19:38:50 -0500 Subject: [PATCH 16/29] chore: remove debugging --- bin_macro/src/lib.rs | 4 ++-- src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs index 2152677..b393b5c 100644 --- a/bin_macro/src/lib.rs +++ b/bin_macro/src/lib.rs @@ -1,5 +1,5 @@ -#![feature(trace_macros)] -trace_macros!(true); +// #![feature(trace_macros)] +// trace_macros!(true); use proc_macro::TokenStream; use syn::{DeriveInput, parse_macro_input}; diff --git a/src/lib.rs b/src/lib.rs index 1aca5c4..153a963 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(log_syntax)] +// #![feature(log_syntax)] use std::{convert::TryInto, net::SocketAddr}; use std::io; From 233a995aad22d4cb908cd8e04c947b40972f5b14 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 19:40:41 -0500 Subject: [PATCH 17/29] chore: remove debugging --- bin_macro/src/stream.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 4ea8a69..fca74c6 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -12,7 +12,6 @@ pub fn stream_parse(input: DeriveInput) -> Result { let reads = quote!(#(#r),*); // get the visibility etc on each field // return a quote for block impl - dbg!(&writes); Ok(quote! { #[automatically_derived] impl Streamable for #name { From 3af1bca05741154176164d5e926a9e92426b53db Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 20:26:43 -0500 Subject: [PATCH 18/29] chore: Format --- bin_macro/src/lib.rs | 8 +- bin_macro/src/stream.rs | 230 +++++++++++++++++++++------------------- src/lib.rs | 51 +++++---- src/u24.rs | 2 +- src/util.rs | 2 +- 5 files changed, 156 insertions(+), 137 deletions(-) diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs index b393b5c..a1e4fdc 100644 --- a/bin_macro/src/lib.rs +++ b/bin_macro/src/lib.rs @@ -2,10 +2,12 @@ // trace_macros!(true); use proc_macro::TokenStream; -use syn::{DeriveInput, parse_macro_input}; +use syn::{parse_macro_input, DeriveInput}; mod stream; #[proc_macro_derive(BinaryStream)] pub fn derive_stream(input: TokenStream) -> TokenStream { - stream::stream_parse(parse_macro_input!(input as DeriveInput)).unwrap().into() -} \ No newline at end of file + stream::stream_parse(parse_macro_input!(input as DeriveInput)) + .unwrap() + .into() +} diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index fca74c6..f0098c8 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -3,128 +3,140 @@ use quote::quote; use syn::{Data, DeriveInput, Fields, Result, Type}; pub fn stream_parse(input: DeriveInput) -> Result { - let name = &input.ident; - match input.data { - Data::Struct(v) => { - // iterate through struct fields - let (w, r) = impl_struct_fields(v.fields); - let writes = quote!(#(#w)*); - let reads = quote!(#(#r),*); - // get the visibility etc on each field - // return a quote for block impl - Ok(quote! { - #[automatically_derived] - impl Streamable for #name { - fn parse(&self) -> Vec { - use ::std::io::Write; - use binary_utils::varint::{VarInt, VarIntWriter}; - use binary_utils::u24::{u24, u24Writer}; - let mut writer = Vec::new(); - #writes - writer - } + let name = &input.ident; + match input.data { + Data::Struct(v) => { + // iterate through struct fields + let (w, r) = impl_struct_fields(v.fields); + let writes = quote!(#(#w)*); + let reads = quote!(#(#r),*); + // get the visibility etc on each field + // return a quote for block impl + Ok(quote! { + #[automatically_derived] + impl Streamable for #name { + fn parse(&self) -> Vec { + use ::std::io::Write; + use binary_utils::varint::{VarInt, VarIntWriter}; + use binary_utils::u24::{u24, u24Writer}; + let mut writer = Vec::new(); + #writes + writer + } - fn compose(source: &[u8], position: &mut usize) -> Self { - use ::std::io::Read; - use binary_utils::varint::{VarInt, VarIntReader}; - use binary_utils::u24::{u24, u24Reader}; + fn compose(source: &[u8], position: &mut usize) -> Self { + use ::std::io::Read; + use binary_utils::varint::{VarInt, VarIntReader}; + use binary_utils::u24::{u24, u24Reader}; - Self { - #reads - } - } - } - }) - }, - Data::Enum(data) => { - let mut write_collective = Vec::::new(); - let mut read_collective = Vec::::new(); - for var in data.variants.iter() { - let (w, r) = impl_named_fields(&var.ident, var.fields.clone()); - write_collective.extend(w); - read_collective.extend(r); - } - let writes = quote!(#(#write_collective)*); - let reads = quote!(#(#read_collective),*); - // get the visibility etc on each field - // return a quote for block impl - Ok(quote! { - #[automatically_derived] - impl Streamable for #name { - fn parse(&self) -> Vec { - use ::std::io::Write; - use binary_utils::varint::{VarInt, VarIntWriter}; - use binary_utils::u24::{u24, u24Writer}; - let mut writer = Vec::new(); - #writes - writer - } + Self { + #reads + } + } + } + }) + } + Data::Enum(data) => { + let mut write_collective = Vec::::new(); + let mut read_collective = Vec::::new(); + for var in data.variants.iter() { + let (w, r) = impl_named_fields(&var.ident, var.fields.clone()); + write_collective.extend(w); + read_collective.extend(r); + } + let writes = quote!(#(#write_collective)*); + let reads = quote!(#(#read_collective),*); + // get the visibility etc on each field + // return a quote for block impl + Ok(quote! { + #[automatically_derived] + impl Streamable for #name { + fn parse(&self) -> Vec { + use ::std::io::Write; + use binary_utils::varint::{VarInt, VarIntWriter}; + use binary_utils::u24::{u24, u24Writer}; + let mut writer = Vec::new(); + #writes + writer + } - fn compose(source: &[u8], position: &mut usize) -> Self { - use ::std::io::Read; - use binary_utils::varint::{VarInt, VarIntReader}; - use binary_utils::u24::{u24, u24Reader}; + fn compose(source: &[u8], position: &mut usize) -> Self { + use ::std::io::Read; + use binary_utils::varint::{VarInt, VarIntReader}; + use binary_utils::u24::{u24, u24Reader}; - Self { - #reads - } - } - } - }) - }, - Data::Union(_) => Err(syn::Error::new(name.span(), "BinaryStream does not support Type Unions. Use Enums instead.")) - } + Self { + #reads + } + } + } + }) + } + Data::Union(_) => Err(syn::Error::new( + name.span(), + "BinaryStream does not support Type Unions. Use Enums instead.", + )), + } } pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec) { - let mut writers = Vec::::new(); - let mut readers = Vec::::new(); - match fields { - Fields::Named(v) => { - for field in &v.named { - let field_id = field.ident.as_ref().unwrap(); - let (writer, reader) = impl_streamable_lazy(field_id, &field.ty); - writers.push(writer); - readers.push(reader); - } - }, - Fields::Unnamed(_v) => { - panic!("Can not parse un-named fields at this current point in time.") - }, - Fields::Unit => { - panic!("Can not use uninitalized data values.") - } - } - (writers, readers) + let mut writers = Vec::::new(); + let mut readers = Vec::::new(); + match fields { + Fields::Named(v) => { + for field in &v.named { + let field_id = field.ident.as_ref().unwrap(); + let (writer, reader) = impl_streamable_lazy(field_id, &field.ty); + writers.push(writer); + readers.push(reader); + } + } + Fields::Unnamed(_v) => { + panic!("Can not parse un-named fields at this current point in time.") + } + Fields::Unit => { + panic!("Can not use uninitalized data values.") + } + } + (writers, readers) } pub fn impl_named_fields(name: &Ident, fields: Fields) -> (Vec, Vec) { - let mut writers = Vec::::new(); - let mut readers = Vec::::new(); - match fields { - Fields::Named(v) => { - for field in &v.named { - let field_id = field.ident.as_ref().unwrap(); - let (writer, reader) = impl_streamable_lazy_named(name, field_id, &field.ty); - writers.push(writer); - readers.push(reader); - } - }, - Fields::Unnamed(_v) => { - panic!("Can not parse un-named fields at this current point in time.") - }, - Fields::Unit => { - panic!("Can not use uninitalized data values.") - } - } - (writers, readers) + let mut writers = Vec::::new(); + let mut readers = Vec::::new(); + match fields { + Fields::Named(v) => { + for field in &v.named { + let field_id = field.ident.as_ref().unwrap(); + let (writer, reader) = impl_streamable_lazy_named(name, field_id, &field.ty); + writers.push(writer); + readers.push(reader); + } + } + Fields::Unnamed(_v) => { + panic!("Can not parse un-named fields at this current point in time.") + } + Fields::Unit => { + panic!("Can not use uninitalized data values.") + } + } + (writers, readers) } - pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ writer.write(&self.#name.parse()[..]).unwrap(); }, quote!(#name: #ty::compose(&source, position))) + ( + quote! { writer.write(&self.#name.parse()[..]).unwrap(); }, + quote!(#name: #ty::compose(&source, position)), + ) } -pub fn impl_streamable_lazy_named(name: &Ident, named: &Ident, ty: &Type) -> (TokenStream, TokenStream) { - (quote!{ writer.write(&self.#name.named.parse()[..]).unwrap(); }, quote!(#name: #ty::compose(&source, position))) +pub fn impl_streamable_lazy_named( + name: &Ident, + _named: &Ident, + ty: &Type, +) -> (TokenStream, TokenStream) { + ( + quote! { writer.write(&self.#name.named.parse()[..]).unwrap(); }, + quote!(#name: #ty::compose(&source, position)), + ) } diff --git a/src/lib.rs b/src/lib.rs index 153a963..7a828c9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,13 @@ // #![feature(log_syntax)] -use std::{convert::TryInto, net::SocketAddr}; use std::io; use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::{convert::TryInto, net::SocketAddr}; pub use bin_macro::*; -use std::io::{Cursor, Write, Read}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use std::io::{Cursor, Read, Write}; pub mod u24; pub mod util; @@ -107,7 +107,6 @@ impl_streamable_vec_primitive!(i32); impl_streamable_vec_primitive!(i64); impl_streamable_vec_primitive!(i128); - // implements bools impl Streamable for bool { fn parse(&self) -> Vec { @@ -123,28 +122,30 @@ impl Streamable for bool { impl Streamable for String { fn parse(&self) -> Vec { - let mut buffer = Vec::::new(); - buffer.write_u16::(self.len() as u16).unwrap(); - buffer.write_all(self.as_bytes()).unwrap(); - buffer + let mut buffer = Vec::::new(); + buffer.write_u16::(self.len() as u16).unwrap(); + buffer.write_all(self.as_bytes()).unwrap(); + buffer } fn compose(source: &[u8], position: &mut usize) -> Self { - let mut stream = Cursor::new(source); - stream.set_position(position.clone() as u64); - // Maybe do this in the future? - let len: usize = stream.read_u16::().unwrap().into(); + let mut stream = Cursor::new(source); + stream.set_position(position.clone() as u64); + // Maybe do this in the future? + let len: usize = stream.read_u16::().unwrap().into(); + *position = (stream.position() as usize) + len; - unsafe { - // todo: Remove this nasty hack. - // todo: The hack being, remove the 2 from indexing on read_short - // todo: And utilize stream. - String::from_utf8_unchecked(stream.get_ref()[2..len + stream.position() as usize].to_vec()) - } + unsafe { + // todo: Remove this nasty hack. + // todo: The hack being, remove the 2 from indexing on read_short + // todo: And utilize stream. + String::from_utf8_unchecked( + stream.get_ref()[2..len + stream.position() as usize].to_vec(), + ) + } } } - impl Streamable for SocketAddr { fn parse(&self) -> Vec { let mut stream = Vec::::new(); @@ -157,9 +158,11 @@ impl Streamable for SocketAddr { let mask = u8::from_str_radix(part, 10).unwrap(); stream.write_u8(mask).unwrap(); } - stream.write_u16::(self.port()).expect("Could not write port to stream."); stream - }, + .write_u16::(self.port()) + .expect("Could not write port to stream."); + stream + } Self::V6(addr) => { stream.write_u8(6).unwrap(); // family? or length?? @@ -187,8 +190,9 @@ impl Streamable for SocketAddr { let parts = &source[from..to]; stream.set_position(to as u64); let port = stream.read_u16::().unwrap(); + *position = stream.position() as usize; SocketAddr::new(IpAddr::from([parts[0], parts[1], parts[2], parts[3]]), port) - }, + } 6 => { let _family = stream.read_u16::().unwrap(); let port = stream.read_u16::().unwrap(); @@ -211,9 +215,10 @@ impl Streamable for SocketAddr { Ipv6Addr::new(a, b, c, d, e, f, g, h) }; let scope = stream.read_u32::().unwrap(); + *position = stream.position() as usize; SocketAddr::from(SocketAddrV6::new(address, port, flow, scope)) - }, - _ => panic!("Unknown Address type!") + } + _ => panic!("Unknown Address type!"), } // let addr_type = self.read_byte(); // if addr_type == 4 { diff --git a/src/u24.rs b/src/u24.rs index 1c0a8aa..9f354b1 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -1,6 +1,6 @@ #![allow(non_camel_case_types)] -use byteorder::{ReadBytesExt, WriteBytesExt, BE, LE}; +use byteorder::ReadBytesExt; use std::cmp::{Ordering, PartialEq, PartialOrd}; use std::convert::{From, Into}; use std::io; diff --git a/src/util.rs b/src/util.rs index 11a4c2f..01b89aa 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,2 +1,2 @@ // This file contains utilities for encoding and decoding streams. -use crate::Streamable; +// use crate::Streamable; From 8f31ecf6c7f30075fe1ca96629e48b2d7795eb00 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 20:29:22 -0500 Subject: [PATCH 19/29] chore: Possibly fix a bug --- src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7a828c9..5d4b5b3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -155,7 +155,7 @@ impl Streamable for SocketAddr { let partstr = self.to_string(); let parts: Vec<&str> = partstr.split(".").collect(); for part in parts { - let mask = u8::from_str_radix(part, 10).unwrap(); + let mask = u8::from_str_radix(part, 10).unwrap_or(0); stream.write_u8(mask).unwrap(); } stream @@ -203,14 +203,14 @@ impl Streamable for SocketAddr { let address = { let mut s = Cursor::new(parts); let (a, b, c, d, e, f, g, h) = ( - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), - s.read_u16::().unwrap(), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), + s.read_u16::().unwrap_or(0), ); Ipv6Addr::new(a, b, c, d, e, f, g, h) }; From 6fbfb2d1ca98edca3d75b14f64c94f0636b9f0a6 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Mon, 18 Oct 2021 23:31:36 -0500 Subject: [PATCH 20/29] chore: Add forced LE --- src/lib.rs | 26 +++++++++++++++++++++++++- src/util.rs | 2 -- 2 files changed, 25 insertions(+), 3 deletions(-) delete mode 100644 src/util.rs diff --git a/src/lib.rs b/src/lib.rs index 5d4b5b3..38c88d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,7 +10,6 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use std::io::{Cursor, Read, Write}; pub mod u24; -pub mod util; pub mod varint; pub use self::{u24::*, varint::*}; @@ -29,6 +28,31 @@ pub trait Streamable { /// Little Endian Encoding pub struct LE(pub T); +impl Streamable for LE +where + T: Streamable { + fn parse(&self) -> Vec { + reverse_vec(self.0.parse()) + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + // if the source is expected to be LE we can swap it + // hehe... + let stream = reverse_vec(source.to_vec()); + LE(T::compose(&stream[..], position)) + } +} + +/// Reverses the bytes in a given vector +fn reverse_vec(bytes: Vec) -> Vec { + let mut ret: Vec = Vec::new(); + + for x in (0..bytes.len()).rev() { + ret.push(*bytes.get(x).unwrap()); + } + ret +} + /// Big Endian Encoding pub struct BE(pub T); diff --git a/src/util.rs b/src/util.rs deleted file mode 100644 index 01b89aa..0000000 --- a/src/util.rs +++ /dev/null @@ -1,2 +0,0 @@ -// This file contains utilities for encoding and decoding streams. -// use crate::Streamable; From cefeaec8b841350350457466c983d46ac884024f Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 12:26:59 -0500 Subject: [PATCH 21/29] chore: Add method to capsulate inner LE element --- bin_macro/src/lib.rs | 4 +-- src/lib.rs | 69 ++++++++++++++++++++++++-------------------- tests/macro_tests.rs | 31 +++++++++++++++++++- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs index a1e4fdc..a423632 100644 --- a/bin_macro/src/lib.rs +++ b/bin_macro/src/lib.rs @@ -1,5 +1,5 @@ -// #![feature(trace_macros)] -// trace_macros!(true); +#![feature(trace_macros)] +trace_macros!(true); use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput}; diff --git a/src/lib.rs b/src/lib.rs index 38c88d2..f101379 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,8 @@ // #![feature(log_syntax)] use std::io; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; -use std::{convert::TryInto, net::SocketAddr}; +use std::convert::{From, Into, TryInto}; +use std::net::{IpAddr, Ipv6Addr, SocketAddrV6, SocketAddr}; pub use bin_macro::*; @@ -36,15 +36,22 @@ where } fn compose(source: &[u8], position: &mut usize) -> Self { - // if the source is expected to be LE we can swap it + // If the source is expected to be LE we can swap it to BE bytes + // Doing this makes the byte stream officially BE. // hehe... let stream = reverse_vec(source.to_vec()); LE(T::compose(&stream[..], position)) } } +impl LE { + pub fn inner(self) -> T { + self.0 + } +} + /// Reverses the bytes in a given vector -fn reverse_vec(bytes: Vec) -> Vec { +pub fn reverse_vec(bytes: Vec) -> Vec { let mut ret: Vec = Vec::new(); for x in (0..bytes.len()).rev() { @@ -255,31 +262,29 @@ impl Streamable for SocketAddr { } } -// impl Streamable for Vec -// where -// T: Streamable { -// fn write(&self) -> Vec { -// // write the length as a varint -// let mut v: Vec = Vec::new(); -// v.write_all(&VarInt(v.len() as u32).to_be_bytes()[..]).unwrap(); -// for x in self.iter() { -// v.extend(x.write().iter()); -// } -// v -// } - -// fn read(source: &[u8], position: &mut usize) -> Self { -// // read a var_int -// let mut ret: Vec = Vec::new(); -// let varint = VarInt::::from_be_bytes(source); -// let length: u32 = varint.into(); - -// *position += varint.get_byte_length() as usize; - -// // read each length -// for _ in 0..length { -// ret.push(T::read(&source, position)); -// } -// ret -// } -// } +/// Writes a vector whose length is written with a short +impl Streamable for Vec { + fn parse(&self) -> Vec { + // write the length as a varint + let mut v: Vec = Vec::new(); + v.write_u16::(v.len() as u16).unwrap(); + for x in self.iter() { + v.extend(LE(x.clone()).parse().iter()); + } + v + } + + fn compose(source: &[u8], position: &mut usize) -> Self { + // read a var_int + let mut stream = Cursor::new(source); + let mut ret: Vec> = Vec::new(); + let length = stream.read_u16::().unwrap(); + + *position = stream.position() as usize; + // read each length + for _ in 0..length { + ret.push(LE::::compose(&source, position)); + } + ret.iter().map(|v| v.0.clone()).collect() + } +} diff --git a/tests/macro_tests.rs b/tests/macro_tests.rs index 84bfceb..b681c6b 100644 --- a/tests/macro_tests.rs +++ b/tests/macro_tests.rs @@ -1,5 +1,5 @@ use bin_macro::*; -use binary_utils::{Streamable}; +use binary_utils::{LE, Streamable, reverse_vec}; #[derive(Debug, BinaryStream)] pub struct TestPacket { pub some_int: u8, @@ -57,4 +57,33 @@ fn read_string() { ]; let string = String::compose(&hello_world_vec[..], &mut 0); assert_eq!("Hello world!".to_string(), string); +} + +#[derive(BinaryStream)] +pub struct HelloWorld { + data: LE:: +} + +#[test] +fn endianness() { + let hello_world_vec_le= reverse_vec(vec![ + 0, + 12, + 72, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, + 33 + ]); + + let data = HelloWorld::compose(&hello_world_vec_le, &mut 0); + + assert_eq!("Hello world!".to_string(), data.data.inner()); } \ No newline at end of file From 013e5ae89af84fba8a700d1bb51b9d5256d35f4d Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 12:50:17 -0500 Subject: [PATCH 22/29] chore: Add more LE stablizations, disable macro tracing --- bin_macro/src/lib.rs | 4 ++-- src/lib.rs | 16 +++++++++------- tests/vec.rs | 21 ++++++++++++++++++++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/bin_macro/src/lib.rs b/bin_macro/src/lib.rs index a423632..a1e4fdc 100644 --- a/bin_macro/src/lib.rs +++ b/bin_macro/src/lib.rs @@ -1,5 +1,5 @@ -#![feature(trace_macros)] -trace_macros!(true); +// #![feature(trace_macros)] +// trace_macros!(true); use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput}; diff --git a/src/lib.rs b/src/lib.rs index f101379..54d7df7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ pub trait Streamable { } /// Little Endian Encoding +#[derive(Debug, Clone, Copy)] pub struct LE(pub T); impl Streamable for LE @@ -263,13 +264,15 @@ impl Streamable for SocketAddr { } /// Writes a vector whose length is written with a short -impl Streamable for Vec { +impl Streamable for Vec> +where + T: Streamable { fn parse(&self) -> Vec { // write the length as a varint let mut v: Vec = Vec::new(); - v.write_u16::(v.len() as u16).unwrap(); + v.write_u16::(self.len() as u16).unwrap(); for x in self.iter() { - v.extend(LE(x.clone()).parse().iter()); + v.extend(x.parse().iter()); } v } @@ -277,14 +280,13 @@ impl Streamable for Vec { fn compose(source: &[u8], position: &mut usize) -> Self { // read a var_int let mut stream = Cursor::new(source); - let mut ret: Vec> = Vec::new(); + let mut ret: Vec> = Vec::new(); let length = stream.read_u16::().unwrap(); - *position = stream.position() as usize; // read each length for _ in 0..length { - ret.push(LE::::compose(&source, position)); + ret.push(LE::::compose(&source[*position..], &mut 0)); } - ret.iter().map(|v| v.0.clone()).collect() + ret } } diff --git a/tests/vec.rs b/tests/vec.rs index 6ad29e2..2b97fff 100644 --- a/tests/vec.rs +++ b/tests/vec.rs @@ -1,5 +1,5 @@ use bin_macro::*; -use binary_utils::varint::VarInt; +use binary_utils::{LE, varint::VarInt, Streamable}; use std::io::Cursor; #[test] @@ -9,3 +9,22 @@ fn test_varint() { dbg!(VarInt::::from_be_bytes(&[255, 255, 255, 1][..])); dbg!(&v.to_be_bytes()); } + +// test a string +#[test] +fn test_le_vec() { + // LE bytes for "Netrex" + let le_bytes_netrex: Vec = vec![120, 101, 114, 116, 101, 78, 6, 0]; + let str_bytes = LE("Netrex".to_string()); + + assert_eq!(str_bytes.parse(), le_bytes_netrex); + + let mut test: Vec> = Vec::new(); + test.push(str_bytes.clone()); + + // Vectors store length {stream, stream } + // where "stream" in this case is [length, string bytes] + let vector = test.parse(); + let restored = Vec::>::compose(&vector[..], &mut 0); + assert_eq!(restored[0].clone().inner(), str_bytes.inner()) +} \ No newline at end of file From 45e711517ff8de710badc7e0f02bd78e99f61ec8 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 14:26:18 -0500 Subject: [PATCH 23/29] feat: Add float primitives --- src/lib.rs | 14 ++++++++++---- src/u24.rs | 2 ++ src/varint.rs | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 54d7df7..deb8970 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,8 @@ // #![feature(log_syntax)] -use std::io; use std::convert::{From, Into, TryInto}; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6, SocketAddr}; +use std::io; +use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; pub use bin_macro::*; @@ -31,7 +31,8 @@ pub struct LE(pub T); impl Streamable for LE where - T: Streamable { + T: Streamable, +{ fn parse(&self) -> Vec { reverse_vec(self.0.parse()) } @@ -86,7 +87,9 @@ macro_rules! impl_streamable_primitive { impl_streamable_primitive!(u8); impl_streamable_primitive!(u16); impl_streamable_primitive!(u32); +impl_streamable_primitive!(f32); impl_streamable_primitive!(u64); +impl_streamable_primitive!(f64); impl_streamable_primitive!(u128); impl_streamable_primitive!(i8); impl_streamable_primitive!(i16); @@ -131,6 +134,8 @@ macro_rules! impl_streamable_vec_primitive { impl_streamable_vec_primitive!(u8); impl_streamable_vec_primitive!(u16); impl_streamable_vec_primitive!(u32); +impl_streamable_vec_primitive!(f32); +impl_streamable_vec_primitive!(f64); impl_streamable_vec_primitive!(u64); impl_streamable_vec_primitive!(u128); impl_streamable_vec_primitive!(i8); @@ -266,7 +271,8 @@ impl Streamable for SocketAddr { /// Writes a vector whose length is written with a short impl Streamable for Vec> where - T: Streamable { + T: Streamable, +{ fn parse(&self) -> Vec { // write the length as a varint let mut v: Vec = Vec::new(); diff --git a/src/u24.rs b/src/u24.rs index 9f354b1..09c1b92 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -170,6 +170,8 @@ impl_primitive_u24!(u8); impl_primitive_u24!(u16); impl_primitive_u24!(u32); impl_primitive_u24!(u64); +impl_primitive_u24!(f32); +impl_primitive_u24!(f64); impl_primitive_u24!(u128); impl_primitive_u24!(i8); impl_primitive_u24!(i16); diff --git a/src/varint.rs b/src/varint.rs index ee32e3e..1978570 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -289,9 +289,13 @@ impl_primitive_VarInt!(u8, u32); impl_primitive_VarInt!(u16, u32); impl_primitive_VarInt!(u32, u32); impl_primitive_VarInt!(u64, u32); +impl_primitive_VarInt!(f32, u32); +impl_primitive_VarInt!(f64, u32); impl_primitive_VarInt!(u128, u32); impl_primitive_VarInt!(i8, u64); impl_primitive_VarInt!(i16, u64); impl_primitive_VarInt!(i32, u64); impl_primitive_VarInt!(i64, u64); +impl_primitive_VarInt!(f32, u64); +impl_primitive_VarInt!(f64, u64); impl_primitive_VarInt!(i128, u64); From 188aded1454bb49bc527c41a9651ac0a81163cb3 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 19:15:36 -0500 Subject: [PATCH 24/29] feat: Add enums --- bin_macro/src/stream.rs | 134 +++++++++++++++++++++------------------- src/lib.rs | 2 +- tests/enum.rs | 30 +++++++++ tests/vec.rs | 4 +- 4 files changed, 102 insertions(+), 68 deletions(-) create mode 100644 tests/enum.rs diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index f0098c8..07d5018 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -1,13 +1,14 @@ use proc_macro2::{Ident, TokenStream}; use quote::quote; -use syn::{Data, DeriveInput, Fields, Result, Type}; +use syn::{Attribute, Data, DeriveInput, Error, Fields, Result, Type}; pub fn stream_parse(input: DeriveInput) -> Result { let name = &input.ident; + let attrs = input.attrs; match input.data { Data::Struct(v) => { // iterate through struct fields - let (w, r) = impl_struct_fields(v.fields); + let (w, r) = impl_named_fields(v.fields); let writes = quote!(#(#w)*); let reads = quote!(#(#r),*); // get the visibility etc on each field @@ -37,39 +38,64 @@ pub fn stream_parse(input: DeriveInput) -> Result { }) } Data::Enum(data) => { - let mut write_collective = Vec::::new(); - let mut read_collective = Vec::::new(); - for var in data.variants.iter() { - let (w, r) = impl_named_fields(&var.ident, var.fields.clone()); - write_collective.extend(w); - read_collective.extend(r); + let representation = find_one_attr("repr", attrs) + .expect("Enums must have a #[repr] attribute"); + let enum_ty = representation.parse_args::() + .expect("Enums can only have types as attributes"); + + if !enum_ty.to_string().starts_with(|v| v == 'u' || v == 'i' || v == 'f') { + return Err(Error::new_spanned(representation, "Representation must be a primitive number")); } - let writes = quote!(#(#write_collective)*); - let reads = quote!(#(#read_collective),*); - // get the visibility etc on each field - // return a quote for block impl - Ok(quote! { - #[automatically_derived] - impl Streamable for #name { - fn parse(&self) -> Vec { - use ::std::io::Write; - use binary_utils::varint::{VarInt, VarIntWriter}; - use binary_utils::u24::{u24, u24Writer}; - let mut writer = Vec::new(); - #writes - writer - } - fn compose(source: &[u8], position: &mut usize) -> Self { - use ::std::io::Read; - use binary_utils::varint::{VarInt, VarIntReader}; - use binary_utils::u24::{u24, u24Reader}; + let (mut writers, mut readers) = (Vec::::new(), Vec::::new()); - Self { - #reads - } - } - } + if !data.variants.iter().all(|v| match v.fields.clone() { Fields::Unit => true, Fields::Unnamed(_) => true, _ => false}) { + return Err(Error::new_spanned(data.variants, "Enum Fields must be Uninitialized or Named")); + } + + for variant in &data.variants { + // for each field... + let da = variant.discriminant.as_ref().expect("All Fields must have a explicit assignment.").clone(); + let discrim = da.1; + let var_name = variant.ident.clone(); + match &variant.fields { + Fields::Unit => { + // writers + dbg!(&enum_ty); + writers.push(quote!(Self::#var_name => (#discrim as #enum_ty).parse(),)); + // readers + readers.push(quote!(#discrim => Self::#var_name,)); + }, + Fields::Unnamed(_fields) => { + return Err(Error::new_spanned(variant, "Variant fields are not explicitly supported yet.")); + // for field in fields.unnamed.iter() { + // dbg!("I am here 2\n\n\\nn\n\n"); + // } + }, + _ => return Err(Error::new_spanned(variant.clone(), "Variant invalid")) + } + } + + Ok(quote!{ + #[automatically_derived] + impl Streamable for #name { + fn parse(&self) -> Vec { + match self { + #(#writers)* + } + } + + fn compose(source: &[u8], offset: &mut usize) -> Self { + // get the repr type and read it + let v = <#enum_ty>::compose(source, offset); + + match v { + #(#readers)* + _ => panic!("Will not fit in enum!") + } + + } + } }) } Data::Union(_) => Err(syn::Error::new( @@ -79,7 +105,7 @@ pub fn stream_parse(input: DeriveInput) -> Result { } } -pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec) { +pub fn impl_named_fields(fields: Fields) -> (Vec, Vec) { let mut writers = Vec::::new(); let mut readers = Vec::::new(); match fields { @@ -101,27 +127,10 @@ pub fn impl_struct_fields(fields: Fields) -> (Vec, Vec (writers, readers) } -pub fn impl_named_fields(name: &Ident, fields: Fields) -> (Vec, Vec) { - let mut writers = Vec::::new(); - let mut readers = Vec::::new(); - match fields { - Fields::Named(v) => { - for field in &v.named { - let field_id = field.ident.as_ref().unwrap(); - let (writer, reader) = impl_streamable_lazy_named(name, field_id, &field.ty); - writers.push(writer); - readers.push(reader); - } - } - Fields::Unnamed(_v) => { - panic!("Can not parse un-named fields at this current point in time.") - } - Fields::Unit => { - panic!("Can not use uninitalized data values.") - } - } - (writers, readers) -} +// pub fn impl_unnamed_fields(_fields: FieldsUnnamed) -> (TokenStream, TokenStream) { + +// todo!() +// } pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStream) { ( @@ -130,13 +139,10 @@ pub fn impl_streamable_lazy(name: &Ident, ty: &Type) -> (TokenStream, TokenStrea ) } -pub fn impl_streamable_lazy_named( - name: &Ident, - _named: &Ident, - ty: &Type, -) -> (TokenStream, TokenStream) { - ( - quote! { writer.write(&self.#name.named.parse()[..]).unwrap(); }, - quote!(#name: #ty::compose(&source, position)), - ) -} +fn find_one_attr(name: &str, attrs: Vec) -> Option { + let mut iter = attrs.iter().filter(|a| a.path.is_ident(name)); + match (iter.next(), iter.next()) { + (Some(v), None) => Some(v.clone()), + _ => None + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index deb8970..a427594 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -// #![feature(log_syntax)] +#![feature(log_syntax)] use std::convert::{From, Into, TryInto}; use std::io; diff --git a/tests/enum.rs b/tests/enum.rs new file mode 100644 index 0000000..9cf6865 --- /dev/null +++ b/tests/enum.rs @@ -0,0 +1,30 @@ +use bin_macro::*; +use binary_utils::Streamable; + +#[derive(Debug, BinaryStream, PartialEq)] +#[repr(u8)] +pub enum Test { + Apple = 0, + Pair = 1 +} + +#[test] +fn read_test_from_buffer() { + let buffer: &[u8] = &[0]; + let result = Test::compose(buffer, &mut 0); + assert_eq!(Test::Apple, result); +} + +#[test] +fn write_read_buffer() { + // write first + let variant = Test::Pair; + let buffer = variant.parse(); + + assert_eq!(buffer, vec![1]); + + // read now + let compose = Test::compose(&buffer[..], &mut 0); + + assert!(match compose { Test::Pair => true, _ => false }, "Reconstruction was not equivelant to Test::Pair"); +} \ No newline at end of file diff --git a/tests/vec.rs b/tests/vec.rs index 2b97fff..6018479 100644 --- a/tests/vec.rs +++ b/tests/vec.rs @@ -1,11 +1,9 @@ -use bin_macro::*; use binary_utils::{LE, varint::VarInt, Streamable}; -use std::io::Cursor; #[test] fn test_varint() { let v = VarInt::(25565); - let val: Vec = vec![221, 199, 1]; + let _val: Vec = vec![221, 199, 1]; dbg!(VarInt::::from_be_bytes(&[255, 255, 255, 1][..])); dbg!(&v.to_be_bytes()); } From 5cd67993b3f26eb773d1601198979e4cddc1b011 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 21:04:27 -0500 Subject: [PATCH 25/29] chore: disable logging --- src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index a427594..c9a4ae0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(log_syntax)] +// #![feature(log_syntax)] use std::convert::{From, Into, TryInto}; use std::io; @@ -16,6 +16,7 @@ pub use self::{u24::*, varint::*}; pub type Stream = io::Cursor>; + pub trait Streamable { /// Writes `self` to the given buffer. fn parse(&self) -> Vec; From fccedf3575e388c560e9d5c6b9e65bba96cf68b2 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 21:05:15 -0500 Subject: [PATCH 26/29] chore: disable logging --- bin_macro/src/stream.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin_macro/src/stream.rs b/bin_macro/src/stream.rs index 07d5018..271240b 100644 --- a/bin_macro/src/stream.rs +++ b/bin_macro/src/stream.rs @@ -61,7 +61,6 @@ pub fn stream_parse(input: DeriveInput) -> Result { match &variant.fields { Fields::Unit => { // writers - dbg!(&enum_ty); writers.push(quote!(Self::#var_name => (#discrim as #enum_ty).parse(),)); // readers readers.push(quote!(#discrim => Self::#var_name,)); From 3498de9feb2ad750797c8032617f75cace313016 Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 19 Oct 2021 21:38:07 -0500 Subject: [PATCH 27/29] chore: Add var_int tests --- tests/tests.rs | 1 + tests/var_int.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 tests/var_int.rs diff --git a/tests/tests.rs b/tests/tests.rs index 8c1e280..bd00c27 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,3 +1,4 @@ // make a test mod macro_tests; mod vec; +mod var_int; \ No newline at end of file diff --git a/tests/var_int.rs b/tests/var_int.rs new file mode 100644 index 0000000..a40b7f6 --- /dev/null +++ b/tests/var_int.rs @@ -0,0 +1,17 @@ +use binary_utils::*; + +#[test] +fn read_write_var_int() { + let one = VarInt::(2147483647); + let two = VarInt::(255); + let buf_one = one.parse(); + let buf_two = two.parse(); + + assert_eq!(buf_one, vec![255, 255, 255, 255, 7]); + assert_eq!(buf_two, vec![255, 1]); + + let buf_long_one = VarInt::(9223372036854775807).parse(); + assert_eq!(buf_long_one, vec![255, 255, 255, 255, 255, 255, 255, 255, 127]); + + assert_eq!(one.0, VarInt::::compose(&buf_one[..], &mut 0).0); +} \ No newline at end of file From 11d687bacca126e425ec6409501111edb2644c6c Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 26 Oct 2021 17:12:47 -0500 Subject: [PATCH 28/29] chore: Docs --- src/lib.rs | 48 +++++++++++++++++++++++++++++++++++++++++++++--- src/u24.rs | 1 + 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c9a4ae0..3df6e59 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,34 @@ pub use self::{u24::*, varint::*}; pub type Stream = io::Cursor>; - +/// A trait to parse and unparse header structs from a given buffer. +/// +/// **Example:** +/// ```rust notest +/// struct Foo { +/// bar: u8, +/// foo_bar: u16 +/// } +/// +/// impl Streamable for Foo { +/// fn parse(&self) -> Vec { +/// use std::io::Write; +/// let mut stream = Vec::::new(); +/// stream.write_all(bar.parse()[..]); +/// stream.write_all(bar.parse()[..]); +/// stream +/// } +/// +/// fn compose(source: &[u8], position: &mut usize) -> Self { +/// // Streamable is implemented for all primitives, so we can +/// // just use this implementation to read our properties. +/// Self { +/// bar: u8::compose(&source, &mut position), +/// foo_bar: u16::compose(&source, &mut position) +/// } +/// } +/// } +/// ``` pub trait Streamable { /// Writes `self` to the given buffer. fn parse(&self) -> Vec; @@ -26,7 +53,21 @@ pub trait Streamable { Self: Sized; } -/// Little Endian Encoding +/// Little Endian Type +/// +/// **Notice:** +/// This struct assumes the incoming buffer is BE and needs to be transformed. +/// +/// For LE decoding in BE streams use: +/// ```rust notest +/// fn read_u16_le(source: &[u8], offset: &mut usize) { +/// // get the size of your type, in this case it's 2 bytes. +/// let be_source = &source[offset..2]; +/// *offset += 2; +/// // now we can form the little endian +/// return LE::::compose(&be_source, &mut 0); +/// } +/// ``` #[derive(Debug, Clone, Copy)] pub struct LE(pub T); @@ -41,7 +82,8 @@ where fn compose(source: &[u8], position: &mut usize) -> Self { // If the source is expected to be LE we can swap it to BE bytes // Doing this makes the byte stream officially BE. - // hehe... + // We actually need to do some hacky stuff here, + // we need to get the size of `T` (in bytes) let stream = reverse_vec(source.to_vec()); LE(T::compose(&stream[..], position)) } diff --git a/src/u24.rs b/src/u24.rs index 09c1b92..e99ac71 100644 --- a/src/u24.rs +++ b/src/u24.rs @@ -8,6 +8,7 @@ use std::ops::{Add, BitOr, Div, Mul, Sub}; use crate::Streamable; /// Base Implementation for a u24 +/// A u24 is 3 bytes (24 bits) wide number. #[derive(Clone, Copy, Debug)] pub struct u24(u32); // inner is validated From 0318c3f0aa9f820d206422fe9ed4cb6eb961214b Mon Sep 17 00:00:00 2001 From: Bavfalcon9 Date: Tue, 26 Oct 2021 19:03:01 -0500 Subject: [PATCH 29/29] feat(LE): A few hack fixes and more tests --- src/lib.rs | 47 +++++++++++++++++++++++++++++++++++++++++++---- tests/le_test.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ tests/tests.rs | 3 ++- tests/vec.rs | 2 ++ 4 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tests/le_test.rs diff --git a/src/lib.rs b/src/lib.rs index 3df6e59..cf5f4b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ // #![feature(log_syntax)] +use std::any::type_name; use std::convert::{From, Into, TryInto}; use std::io; use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; @@ -16,10 +17,17 @@ pub use self::{u24::*, varint::*}; pub type Stream = io::Cursor>; +macro_rules! includes { + ($var: ident, $method: ident, $values: expr) => {{ + let v = &$values; + v.iter().filter(|&v| $var.$method(v)).count() > 0 + }}; +} + /// A trait to parse and unparse header structs from a given buffer. /// /// **Example:** -/// ```rust notest +/// ```rust ignore /// struct Foo { /// bar: u8, /// foo_bar: u16 @@ -59,7 +67,7 @@ pub trait Streamable { /// This struct assumes the incoming buffer is BE and needs to be transformed. /// /// For LE decoding in BE streams use: -/// ```rust notest +/// ```rust ignore /// fn read_u16_le(source: &[u8], offset: &mut usize) { /// // get the size of your type, in this case it's 2 bytes. /// let be_source = &source[offset..2]; @@ -73,7 +81,7 @@ pub struct LE(pub T); impl Streamable for LE where - T: Streamable, + T: Streamable + Sized, { fn parse(&self) -> Vec { reverse_vec(self.0.parse()) @@ -84,7 +92,23 @@ where // Doing this makes the byte stream officially BE. // We actually need to do some hacky stuff here, // we need to get the size of `T` (in bytes) - let stream = reverse_vec(source.to_vec()); + let stream = { + // if we can get the value of the type we do so here. + let name = type_name::(); + + if includes!( + name, + contains, + [ + "u8", "u16", "u32", "u64", "u128", "i8", "i16", "i32", "i64", "i128", "f32", + "f64" + ] + ) { + reverse_vec(source[*position..(*position + ::std::mem::size_of::())].to_vec()) + } else { + reverse_vec(source[*position..].to_vec()) + } + }; LE(T::compose(&stream[..], position)) } } @@ -124,6 +148,21 @@ macro_rules! impl_streamable_primitive { data } } + + // impl Streamable for LE<$ty> { + // fn parse(&self) -> Vec { + // reverse_vec(self.0.parse()) + // } + + // fn compose(source: &[u8], position: &mut usize) -> Self { + // // If the source is expected to be LE we can swap it to BE bytes + // // Doing this makes the byte stream officially BE. + // // We actually need to do some hacky stuff here, + // // we need to get the size of `T` (in bytes) + // let stream = reverse_vec(source[*position..(*position + ::std::mem::size_of::<$ty>())].to_vec()); + // LE(<$ty>::compose(&stream[..], position)) + // } + // } }; } diff --git a/tests/le_test.rs b/tests/le_test.rs new file mode 100644 index 0000000..d3e427e --- /dev/null +++ b/tests/le_test.rs @@ -0,0 +1,44 @@ +use std::io::Write; + +use binary_utils::*; + +#[test] +fn read_write_le_mixed() { + // LE encoded size, but BE data + // The first 3 bytes are dummy bytes + let buff_one: Vec = vec![32, 32, 32, 12, 0, 0, 0, 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100, 33, 32, 32, 32, 32]; + + // read first 3 bytes. + // this makes our offset 3 + let le_header = &buff_one[3..]; + let mut offset: usize = 0; + + // get the length + let length = LE::::compose(&le_header, &mut offset).0 as usize; + assert_eq!(12, length); + + // now get the rest of the buffer based on the length + let decoded = String::from_utf8(le_header[offset..(length + offset)].to_vec()).unwrap(); + assert_eq!(decoded, "Hello World!".to_string()); + + // get the rest of the buffer, there should be 4 more bytes + assert_eq!(&le_header[(length + offset)..], &[32, 32, 32, 32]); + + // Writing test + let mut buff_two: Vec = vec![32, 32, 32, 32]; + let to_encode = "Hello World!".to_string(); + + // The length of the string in LE: + let length = LE::(to_encode.len() as u32); + buff_two.write_all(&length.parse()[..]).unwrap(); + // we should now have the length of 12 written in LE + + // write the contents of the string now... + buff_two.write_all(&to_encode.as_bytes()).unwrap(); + + // Write magic to buffer. + buff_two.write_all(&[32, 32, 32, 32]).unwrap(); + + // Now check + assert_eq!(buff_one, &buff_one[..]); +} \ No newline at end of file diff --git a/tests/tests.rs b/tests/tests.rs index bd00c27..ca911f4 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,4 +1,5 @@ // make a test mod macro_tests; mod vec; -mod var_int; \ No newline at end of file +mod var_int; +mod le_test; \ No newline at end of file diff --git a/tests/vec.rs b/tests/vec.rs index 6018479..b21dd74 100644 --- a/tests/vec.rs +++ b/tests/vec.rs @@ -14,6 +14,7 @@ fn test_le_vec() { // LE bytes for "Netrex" let le_bytes_netrex: Vec = vec![120, 101, 114, 116, 101, 78, 6, 0]; let str_bytes = LE("Netrex".to_string()); + println!("{:?}", str_bytes.parse()); assert_eq!(str_bytes.parse(), le_bytes_netrex); @@ -23,6 +24,7 @@ fn test_le_vec() { // Vectors store length {stream, stream } // where "stream" in this case is [length, string bytes] let vector = test.parse(); + println!("{:?}", vector); let restored = Vec::>::compose(&vector[..], &mut 0); assert_eq!(restored[0].clone().inner(), str_bytes.inner()) } \ No newline at end of file