diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md index 1d5e2cb0..d7bacdf8 100644 --- a/.github/ISSUE_TEMPLATE/proposal.md +++ b/.github/ISSUE_TEMPLATE/proposal.md @@ -1,7 +1,7 @@ --- -name: Proposal -about: A proposal for a new feature or change -title: 'Proposal:' +name: Improvement Proposal +about: A formal proposal discussing any new features, changes, or improvements to the project. +title: 'CNC-0000:' labels: ['proposal'] projects: ['@FL03/concision:features', '@FL03/concision:roadmap'] assignees: @@ -10,3 +10,6 @@ assignees: --- +### Resources + +- [Google](https://google.com) \ No newline at end of file diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 8ad78abc..39d91bd1 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -1,4 +1,4 @@ -name: Clippy +name: clippy on: pull_request: diff --git a/.github/workflows/crates.yml b/.github/workflows/crates.yml index 467e3a1c..b7c226d0 100644 --- a/.github/workflows/crates.yml +++ b/.github/workflows/crates.yml @@ -1,4 +1,4 @@ -name: crates.io +name: crates concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2ed6c641..d8afa861 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,4 @@ -name: Rust +name: rust concurrency: cancel-in-progress: false diff --git a/Cargo.toml b/Cargo.toml index 87b42c15..32f3103c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,12 +13,13 @@ version = "0.1.14" [workspace.dependencies] # acme = { features = ["full"], branch = "v0.3.2", git = "https://github.com/FL03/acme", version = "0.3.2" } # ndtensor = { features = ["full"], branch = "v0.1.1", git = "https://github.com/FL03/ndtensor", version = "0.1" } -# scsys = { features = ["full"], branch = "v0.2.2", git = "https://github.com/scattered-systems/scsys", version = "0.2" } +scsys = { default-features = false, branch = "v0.2.3", features = ["derive"], git = "https://github.com/scattered-systems/scsys.git", version = "0.2" } approx = "0.5" -itertools = "0.12" +itertools = "0.13" lazy_static = "1" ndarray = { default-features = false, version = "0.15" } +ndarray-stats = "0.5" num = { default-features = false, version = "0.4" } paste = "1" smart-default = "0.7" diff --git a/concision/benches/default.rs b/concision/benches/default.rs index 937f2387..9e07b340 100644 --- a/concision/benches/default.rs +++ b/concision/benches/default.rs @@ -3,50 +3,97 @@ extern crate test; -use std::mem::replace; use test::Bencher; // bench: find the `BENCH_SIZE` first terms of the fibonacci sequence -static BENCH_SIZE: usize = 20; - -// recursive fibonacci -fn fibonacci(n: usize) -> u32 { - if n < 2 { - 1 - } else { - fibonacci(n - 1) + fibonacci(n - 2) - } -} - -// iterative fibonacci -struct Fibonacci { - curr: u32, - next: u32, -} - -impl Iterator for Fibonacci { - type Item = u32; - fn next(&mut self) -> Option { - let new_next = self.curr + self.next; - let new_curr = replace(&mut self.next, new_next); +const BENCH_SIZE: u32 = 20; - Some(replace(&mut self.curr, new_curr)) - } +#[bench] +fn fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fib::fibonacci).collect::>()) } -fn fibonacci_sequence() -> Fibonacci { - Fibonacci { curr: 1, next: 1 } +#[bench] +fn iter_fibonacci(b: &mut Bencher) { + b.iter(|| { + fib::Fibonacci::new() + .take(BENCH_SIZE as usize) + .collect::>() + }) } -// function to benchmark must be annotated with `#[bench]` #[bench] fn recursive_fibonacci(b: &mut Bencher) { // exact code to benchmark must be passed as a closure to the iter // method of Bencher - b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) + b.iter(|| { + (0..BENCH_SIZE) + .map(fib::recursive_fibonacci) + .collect::>() + }) } -#[bench] -fn iterative_fibonacci(b: &mut Bencher) { - b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +mod fib { + /// fibonacci(n) returns the nth fibonacci number + /// This function uses the definition of Fibonacci where: + /// F(0) = F(1) = 1 and F(n+1) = F(n) + F(n-1) for n>0 + /// + /// Warning: This will overflow the 128-bit unsigned integer at n=186 + pub fn fibonacci(n: u32) -> u128 { + // Use a and b to store the previous two values in the sequence + let mut a = 0; + let mut b = 1; + for _i in 0..n { + // As we iterate through, move b's value into a and the new computed + // value into b. + let c = a + b; + a = b; + b = c; + } + b + } + + /// fibonacci(n) returns the nth fibonacci number + /// This function uses the definition of Fibonacci where: + /// F(0) = F(1) = 1 and F(n+1) = F(n) + F(n-1) for n>0 + /// + /// Warning: This will overflow the 128-bit unsigned integer at n=186 + pub fn recursive_fibonacci(n: u32) -> u128 { + // Call the actual tail recursive implementation, with the extra + // arguments set up. + _recursive_fibonacci(n, 0, 1) + } + + fn _recursive_fibonacci(n: u32, previous: u128, current: u128) -> u128 { + if n == 0 { + current + } else { + _recursive_fibonacci(n - 1, current, current + previous) + } + } + + pub struct Fibonacci { + curr: u32, + next: u32, + } + + impl Fibonacci { + pub fn new() -> Fibonacci { + Fibonacci { curr: 0, next: 1 } + } + } + + impl Iterator for Fibonacci { + type Item = u32; + + fn next(&mut self) -> Option { + use core::mem::replace; + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } + } } diff --git a/concision/examples/linear.rs b/concision/examples/linear.rs index 5a063c8c..64d77b4c 100644 --- a/concision/examples/linear.rs +++ b/concision/examples/linear.rs @@ -4,7 +4,8 @@ */ extern crate concision as cnc; -use cnc::prelude::{linarr, Linear, Result, Sigmoid}; +use cnc::linear::Features; +use cnc::prelude::{linarr, InitializeExt, Linear, Result, Sigmoid}; use ndarray::Ix2; fn tracing() { @@ -23,16 +24,17 @@ fn tracing() { fn main() -> Result<()> { tracing(); tracing::info!("Starting linear model example"); + let samples = 20; + let (dm, dn) = (5, 3); + let features = Features::new(dn, dm); + let data = linarr::((samples, dm)).unwrap(); - let (samples, d_in, d_out) = (20, 5, 3); - let data = linarr::((samples, d_in)).unwrap(); - - let model = Linear::::from_features(d_in, d_out).uniform(); + let model = Linear::::lecun_normal(features, dm); assert!(model.is_biased()); let y = model.activate(&data, Sigmoid::sigmoid).unwrap(); - assert_eq!(y.dim(), (samples, d_out)); - println!("Predictions:\n{:?}", &y); + assert_eq!(y.dim(), (samples, dn)); + println!("Predictions:\n{:#?}", &y); Ok(()) } diff --git a/core/Cargo.toml b/core/Cargo.toml index a710e1d0..b0891b64 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -30,6 +30,7 @@ alloc = [ "num/alloc", "rand?/alloc", "rand_distr?/alloc", + "scsys/alloc", "serde?/alloc", ] @@ -55,7 +56,7 @@ rand-ext = [ "uuid/v4", ] -rng_std = [ +std-rng = [ "rand?/std", "rand?/std_rng", ] @@ -66,6 +67,7 @@ serde = [ "num/serde", "rand?/serde1", "rand_distr?/serde1", + "scsys/serde", "uuid/serde" ] @@ -80,15 +82,18 @@ tracing = [ # ********* [FF] Environments ********* std = [ "alloc", + "std-rng", "ndarray/std", "num/std", - "rng_std", + "scsys/std", "serde/std", "strum/std", "uuid/std" ] -wasm = [] +wasm = [ + "getrandom/js", +] wasi = [] @@ -111,6 +116,8 @@ required-features = ["approx"] [dependencies] ndarray.workspace = true num.workspace = true +paste.workspace = true +scsys.workspace = true smart-default.workspace = true strum.workspace = true @@ -154,6 +161,7 @@ lazy_static = "1" all-features = true rustc-args = ["--cfg", "docsrs"] -[target.wasm32-unknown-unknown] +[target.wasm32-unknown-unknown.dependencies] +getrandom = "0.2" [target.wasm32-wasi] diff --git a/core/src/error/kinds/external.rs b/core/src/error/kinds/external.rs index 89bdc0e0..eb5d289f 100644 --- a/core/src/error/kinds/external.rs +++ b/core/src/error/kinds/external.rs @@ -67,4 +67,4 @@ impl From> for ExternalError { } } -error_from!(ExternalError::Error<&str, String>); +from_variant!(ExternalError::Error {<&str>.to_string(), .to_string()}); diff --git a/core/src/error/kinds/predict.rs b/core/src/error/kinds/predict.rs index 114657c1..de39d1d8 100644 --- a/core/src/error/kinds/predict.rs +++ b/core/src/error/kinds/predict.rs @@ -2,6 +2,7 @@ Appellation: error Contrib: FL03 */ +use scsys::VariantConstructors; use smart_default::SmartDefault; use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; @@ -20,6 +21,7 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN PartialEq, PartialOrd, SmartDefault, + VariantConstructors, VariantNames, )] #[cfg_attr( @@ -34,11 +36,3 @@ pub enum PredictError { ShapeMismatch, TypeError, } - -impl PredictError { - variant_constructor!( - ArithmeticError.arithmetic_error, - ShapeMismatch.shape_mismatch, - TypeError.type_error - ); -} diff --git a/core/src/error/kinds/shape.rs b/core/src/error/kinds/shape.rs index 9e493334..d7c2c831 100644 --- a/core/src/error/kinds/shape.rs +++ b/core/src/error/kinds/shape.rs @@ -27,10 +27,9 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN )] #[strum(serialize_all = "snake_case")] pub enum ShapeError { - LayoutError, + IncompatibleLayout, + IncompatibleRank, ShapeMismatch, - RankMismatch, SizeMismatch, - Unknown, } diff --git a/core/src/func/activate.rs b/core/src/func/activate.rs deleted file mode 100644 index 5ab8d035..00000000 --- a/core/src/func/activate.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - Appellation: activate - Contrib: FL03 -*/ -pub use self::{binary::*, nl::*}; - -pub mod binary; -pub mod nl; - -pub fn linear(x: &T) -> T -where - T: Clone, -{ - x.clone() -} - -build_unary_trait!(LinearActivation.linear); - -impl LinearActivation for T -where - T: Clone, -{ - type Output = T; - - fn linear(&self) -> Self::Output { - linear(self) - } -} - -pub(crate) mod prelude { - pub use super::binary::*; - pub use super::nl::*; - pub use super::{linear, LinearActivation}; -} - -pub trait Activator { - type Output; -} diff --git a/core/src/func/activate/binary.rs b/core/src/func/activate/binary.rs index b45a0588..5b417dc0 100644 --- a/core/src/func/activate/binary.rs +++ b/core/src/func/activate/binary.rs @@ -6,18 +6,18 @@ use nd::{Array, ArrayBase, Data, Dimension}; use num::{One, Zero}; /// -pub fn heavyside(x: &T) -> T +pub fn heavyside(x: T) -> T where T: One + PartialOrd + Zero, { - if x > &T::zero() { + if x > T::zero() { T::one() } else { T::zero() } } -build_unary_trait!(Heavyside.heavyside,); +unary!(Heavyside::heavyside(self),); macro_rules! impl_heavyside { ($($ty:ty),* $(,)*) => { @@ -27,7 +27,7 @@ macro_rules! impl_heavyside { impl Heavyside for $ty { type Output = $ty; - fn heavyside(&self) -> Self::Output { + fn heavyside(self) -> Self::Output { heavyside(self) } } @@ -36,15 +36,28 @@ macro_rules! impl_heavyside { impl_heavyside!(f32, f64, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize,); -impl Heavyside for ArrayBase +impl Heavyside for ArrayBase where - A: Heavyside, + A: Clone + Heavyside, D: Dimension, S: Data, { - type Output = Array<::Output, D>; + type Output = Array; - fn heavyside(&self) -> Self::Output { - self.map(Heavyside::heavyside) + fn heavyside(self) -> Self::Output { + self.mapv(Heavyside::heavyside) + } +} + +impl<'a, A, B, S, D> Heavyside for &'a ArrayBase +where + A: Clone + Heavyside, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn heavyside(self) -> Self::Output { + self.mapv(Heavyside::heavyside) } } diff --git a/core/src/func/activate/linear.rs b/core/src/func/activate/linear.rs new file mode 100644 index 00000000..d4c9dc7e --- /dev/null +++ b/core/src/func/activate/linear.rs @@ -0,0 +1,21 @@ +/* + Appellation: linear + Contrib: FL03 +*/ + +pub fn linear(x: T) -> T { + x +} + +unary!(LinearActivation::linear(self)); + +impl<'a, T> LinearActivation for &'a T +where + T: Clone, +{ + type Output = T; + + fn linear(self) -> Self::Output { + self.clone() + } +} diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs new file mode 100644 index 00000000..cd1c2f0b --- /dev/null +++ b/core/src/func/activate/mod.rs @@ -0,0 +1,32 @@ +/* + Appellation: activate + Contrib: FL03 +*/ +pub use self::{binary::*, linear::*, nl::*}; + +pub mod binary; +pub mod linear; +pub mod nl; + +pub(crate) mod prelude { + pub use super::binary::*; + pub use super::linear::*; + pub use super::nl::*; + pub use super::{Activate, Evaluate}; +} + +#[doc(hidden)] +pub trait Activate { + type Output; + + fn activate(&self, args: &T) -> Self::Output; +} + +#[doc(hidden)] +pub trait Evaluate { + type Output; + + fn eval(&self, args: T) -> Self::Output; +} + +activator!(LinearActor::(T::clone) where T: Clone); diff --git a/core/src/func/activate/nl.rs b/core/src/func/activate/nl.rs index 93341e6f..694145c7 100644 --- a/core/src/func/activate/nl.rs +++ b/core/src/func/activate/nl.rs @@ -2,59 +2,50 @@ Appellation: sigmoid Contrib: FL03 */ +use crate::math::Exp; use ndarray::*; use num::complex::{Complex, ComplexFloat}; -use num::{Float, Zero}; +use num::traits::Zero; -pub fn relu(args: &T) -> T +pub fn relu(args: T) -> T where - T: Clone + PartialOrd + Zero, + T: PartialOrd + Zero, { - if args > &T::zero() { - return args.clone(); + if args > T::zero() { + return args; } T::zero() } -pub fn sigmoid(args: &T) -> T +pub fn sigmoid(args: T) -> T where T: ComplexFloat, { - (T::one() + (*args).neg().exp()).recip() + (T::one() + args.neg().exp()).recip() } -pub fn softmax(args: &Array) -> Array +pub fn softmax(args: &ArrayBase) -> Array where + A: ComplexFloat + ScalarOperand, D: Dimension, - T: Float, + S: Data, { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) + args.exp() / args.exp().sum() } -pub fn softmax_axis(args: &Array, axis: Option) -> Array -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } -} - -pub fn tanh(args: &T) -> T +pub fn tanh(args: T) -> T where T: ComplexFloat, { args.tanh() } -build_unary_trait!(ReLU.relu, Sigmoid.sigmoid, Softmax.softmax, Tanh.tanh,); +unary!( + ReLU::relu(self), + Sigmoid::sigmoid(self), + Softmax::softmax(self), + Tanh::tanh(self), +); /* ********** Implementations ********** @@ -76,7 +67,7 @@ macro_rules! nonlinear { impl $rho for $T { type Output = $T; - fn $call(&self) -> Self::Output { + fn $call(self) -> Self::Output { $call(self) } } @@ -84,7 +75,7 @@ macro_rules! nonlinear { impl<'a> $rho for &'a $T { type Output = $T; - fn $call(&self) -> Self::Output { + fn $call(self) -> Self::Output { $call(*self) } } @@ -99,12 +90,24 @@ macro_rules! nonlinear { { type Output = Array<::Output, D>; - fn $call(&self) -> Self::Output { - self.map($name::$call) + fn $call(self) -> Self::Output { + self.mapv($name::$call) } } - }; + impl<'a, A, S, D> $name for &'a ArrayBase + where + A: Clone + $name, + D: Dimension, + S: Data + { + type Output = Array<::Output, D>; + + fn $call(self) -> Self::Output { + self.mapv($name::$call) + } + } + }; } nonlinear!( @@ -134,6 +137,32 @@ nonlinear!( f32, f64, Complex, - Complex < f64 > + Complex ]>, ); + +impl Softmax for ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn softmax(self) -> Self::Output { + softmax(&self) + } +} + +impl<'a, A, S, D> Softmax for &'a ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn softmax(self) -> Self::Output { + softmax(self) + } +} diff --git a/core/src/func/dropout.rs b/core/src/func/dropout.rs new file mode 100644 index 00000000..00b24b13 --- /dev/null +++ b/core/src/func/dropout.rs @@ -0,0 +1,92 @@ +/* + Appellation: dropout + Contrib: FL03 +*/ +#![cfg(feature = "rand")] +use crate::Forward; +use nd::prelude::*; +use nd::{DataOwned, RemoveAxis, ScalarOperand}; +use ndrand::rand_distr::Bernoulli; +use ndrand::RandomExt; +use num::traits::Num; + +pub fn dropout(array: &ArrayBase, p: f64) -> Array +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + // Create a Bernoulli distribution for dropout + let distribution = Bernoulli::new(p).unwrap(); + + // Create a mask of the same shape as the input array + let mask: Array = Array::random(array.dim(), distribution); + let mask = mask.mapv(|x| if x { A::zero() } else { A::one() }); + + // Element-wise multiplication to apply dropout + array * mask +} + +pub fn dropout_axis(array: &ArrayBase, _axis: Axis, p: f64) -> Array +where + A: Num + ScalarOperand, + D: RemoveAxis, + S: DataOwned, +{ + // Create a Bernoulli distribution for dropout + let distribution = Bernoulli::new(p).unwrap(); + + // Create a mask of the same shape as the input array + let _mask: Array = Array::random(array.dim(), distribution); + + unimplemented!() +} + +/// The [Dropout] layer is randomly zeroizes inputs with a given probability (`p`). +/// This regularization technique is often used to prevent overfitting. +/// +/// +/// ### Config +/// +/// - (p) Probability of dropping an element +pub struct Dropout { + p: f64, +} + +impl Dropout { + pub fn new(p: f64) -> Self { + Self { p } + } + + pub fn dropout(&self, array: &ArrayBase) -> Array + where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, + { + dropout(array, self.p) + } + + pub fn scale(&self) -> f64 { + (1f64 - self.p).recip() + } +} + +impl Default for Dropout { + fn default() -> Self { + Self::new(0.5) + } +} + +impl Forward> for Dropout +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + type Output = Array; + + fn forward(&self, input: &ArrayBase) -> Self::Output { + dropout(input, self.p) + } +} diff --git a/core/src/func/loss.rs b/core/src/func/loss.rs deleted file mode 100644 index aed784ab..00000000 --- a/core/src/func/loss.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: loss - Contrib: FL03 -*/ - -pub(crate) mod prelude { - pub use super::Loss; -} - -pub trait Loss { - type Output; - - fn loss(&self, cmp: &T) -> Self::Output; -} diff --git a/core/src/func/loss/entropy.rs b/core/src/func/loss/entropy.rs new file mode 100644 index 00000000..5e966b72 --- /dev/null +++ b/core/src/func/loss/entropy.rs @@ -0,0 +1,12 @@ +/* + Appellation: entropy + Contrib: FL03 +*/ + +pub trait Entropy { + type Output; + + fn cross_entropy(&self, target: &T) -> Self::Output; +} + +pub struct CrossEntropy; diff --git a/core/src/func/loss/mod.rs b/core/src/func/loss/mod.rs new file mode 100644 index 00000000..71694cbb --- /dev/null +++ b/core/src/func/loss/mod.rs @@ -0,0 +1,23 @@ +/* + Appellation: loss + Contrib: FL03 +*/ +pub use self::reg::prelude::*; +pub use self::{entropy::*, utils::*}; + +pub(crate) mod utils; + +pub mod entropy; +pub mod reg; + +pub(crate) mod prelude { + pub use super::reg::prelude::*; + pub use super::utils::*; + pub use super::Loss; +} + +pub trait Loss { + type Output; + + fn loss(&self, a: &A, cmp: &B) -> Self::Output; +} diff --git a/core/src/func/loss/reg.rs b/core/src/func/loss/reg.rs new file mode 100644 index 00000000..679d8a6d --- /dev/null +++ b/core/src/func/loss/reg.rs @@ -0,0 +1,13 @@ +/* + Appellation: reg + Contrib: FL03 +*/ +//! # Regressive Loss Functions +//! +//! + +pub mod avg; + +pub(crate) mod prelude { + pub use super::avg::*; +} diff --git a/core/src/func/loss/reg/avg.rs b/core/src/func/loss/reg/avg.rs new file mode 100644 index 00000000..25d80beb --- /dev/null +++ b/core/src/func/loss/reg/avg.rs @@ -0,0 +1,65 @@ +/* + Appellation: avg + Contrib: FL03 +*/ +use crate::math::{Abs, Squared}; +use nd::prelude::*; +use nd::{Data, ScalarOperand}; +use num::traits::{FromPrimitive, Num, Pow, Signed}; + +pub trait MeanAbsoluteError { + type Output; + + fn mae(&self, target: &Rhs) -> Self::Output; +} + +pub trait MeanSquaredError { + type Output; + + fn mse(&self, target: &Rhs) -> Self::Output; +} + +losses! { + impl MSE::, ArrayBase, Output = Option>(MeanSquaredError::mse) + where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +} + +losses! { + impl MAE::, ArrayBase, Output = Option>(MeanAbsoluteError::mae) + where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +} + +/* + ************* Implementations ************* +*/ +impl MeanAbsoluteError> for ArrayBase +where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +{ + type Output = Option; + + fn mae(&self, target: &ArrayBase) -> Self::Output { + (target - self).abs().mean() + } +} + +impl MeanSquaredError> for ArrayBase +where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Option; + + fn mse(&self, target: &ArrayBase) -> Self::Output { + (target - self).sqrd().mean() + } +} diff --git a/core/src/func/loss/utils.rs b/core/src/func/loss/utils.rs new file mode 100644 index 00000000..9f61779b --- /dev/null +++ b/core/src/func/loss/utils.rs @@ -0,0 +1,29 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use crate::math::{Abs, Squared}; +use nd::prelude::*; +use nd::{Data, ScalarOperand}; +use num::traits::{FromPrimitive, Num, Pow, Signed}; + +/// A functional implementation of the mean absolute error loss function which compares two similar +/// [arrays](ndarray::ArrayBase) +pub fn mae(pred: &ArrayBase, target: &ArrayBase) -> Option +where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +{ + (pred - target).abs().mean() +} +/// A functional implementation of the mean squared error loss function that compares two similar +/// [arrays](ndarray::ArrayBase) +pub fn mse(pred: &ArrayBase, target: &ArrayBase) -> Option +where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +{ + (pred - target).sqrd().mean() +} diff --git a/core/src/func/mod.rs b/core/src/func/mod.rs index 266ce461..bb99ccba 100644 --- a/core/src/func/mod.rs +++ b/core/src/func/mod.rs @@ -5,10 +5,14 @@ //! Functional pub use self::prelude::*; +#[macro_use] pub mod activate; +pub mod dropout; pub mod loss; pub(crate) mod prelude { pub use super::activate::prelude::*; + #[cfg(feature = "rand")] + pub use super::dropout::*; pub use super::loss::prelude::*; } diff --git a/core/src/init/gen/lecun.rs b/core/src/init/gen/lecun.rs new file mode 100644 index 00000000..b8cae16c --- /dev/null +++ b/core/src/init/gen/lecun.rs @@ -0,0 +1,53 @@ +/* + Appellation: lecun + Contrib: FL03 +*/ +use num::Float; +use rand::Rng; +use rand_distr::{Distribution, Normal, NormalError, StandardNormal}; + +/// [LecunNormal] is a truncated [normal](rand_distr::Normal) distribution centered at 0 +/// with a standard deviation that is calculated as `σ = sqrt(1/n_in)` +/// where `n_in` is the number of input units. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LecunNormal { + n: usize, +} + +impl LecunNormal { + pub fn new(n: usize) -> Self { + Self { n } + } + /// Create a [normal](rand_distr::Normal) [distribution](Distribution) centered at 0; + /// See [Self::std_dev] for the standard deviation calculations. + pub fn distr(&self) -> Result, NormalError> + where + F: Float, + StandardNormal: Distribution, + { + Normal::new(F::zero(), self.std_dev()) + } + /// Calculate the standard deviation (`σ`) of the distribution. + /// This is done by computing the root of the reciprocal of the number of inputs + /// + /// Symbolically: `σ = sqrt(1/n)` + pub fn std_dev(&self) -> F + where + F: Float, + { + F::from(self.n).unwrap().recip().sqrt() + } +} + +impl Distribution for LecunNormal +where + F: Float, + StandardNormal: Distribution, +{ + fn sample(&self, rng: &mut R) -> F + where + R: Rng + ?Sized, + { + self.distr().unwrap().sample(rng) + } +} diff --git a/core/src/init/initialize.rs b/core/src/init/initialize.rs new file mode 100644 index 00000000..91b41b13 --- /dev/null +++ b/core/src/init/initialize.rs @@ -0,0 +1,212 @@ +/* + Appellation: initialize + Contrib: FL03 +*/ +use core::ops::Neg; +use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; +use ndrand::RandomExt; +use num::complex::ComplexDistribution; +use num::traits::Float; +use rand::{rngs, Rng, SeedableRng}; +use rand_distr::uniform::{SampleUniform, Uniform}; +use rand_distr::{Bernoulli, BernoulliError, Distribution, Normal, StandardNormal}; + +use super::LecunNormal; + +/// This trait provides the base methods required for initializing an [ndarray](ndarray::ArrayBase) with random values. +/// [Initialize] is similar to [RandomExt](ndarray_rand::RandomExt), however, it focuses on flexibility while implementing additional +/// features geared towards machine-learning models; such as lecun_normal initialization. +pub trait Initialize +where + D: Dimension, +{ + type Data: RawData; + /// Generate a random array using the given distribution + fn rand(shape: Sh, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Sh: ShapeBuilder, + Self::Data: DataOwned; + /// Generate a random array using the given distribution and random number generator + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + Self::Data: DataOwned; + /// Initialize an array with random values using the given distribution and current shape + fn init_rand(self, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Self: Sized, + Self::Data: DataOwned; + /// Initialize an array with random values from the current shape using the given distribution and random number generator + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Self::Data: DataOwned; +} + +/// This trait extends the [Initialize] trait with methods for generating random arrays from various distributions. +pub trait InitializeExt: Initialize + Sized +where + A: Clone, + D: Dimension, + S: RawData, +{ + fn bernoulli(shape: Sh, p: f64) -> Result + where + S: DataOwned, + Sh: ShapeBuilder, + Bernoulli: Distribution, + { + let dist = Bernoulli::new(p)?; + Ok(Self::rand(shape, dist)) + } + /// Initialize the object according to the Lecun Initialization scheme. + /// LecunNormal distributions are truncated [Normal](rand_distr::Normal) + /// distributions centered at 0 with a standard deviation equal to the + /// square root of the reciprocal of the number of inputs. + fn lecun_normal(shape: Sh, n: usize) -> Self + where + A: Float, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + let distr = LecunNormal::new(n); + Self::rand(shape, distr) + } + /// Given a shape, mean, and standard deviation generate a new object using the [Normal](rand_distr::Normal) distribution + fn normal(shape: Sh, mean: A, std: A) -> Result + where + A: Float, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + let distr = Normal::new(mean, std)?; + Ok(Self::rand(shape, distr)) + } + + fn randc(shape: Sh, re: A, im: A) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + ComplexDistribution: Distribution, + { + let distr = ComplexDistribution::new(re, im); + Self::rand(shape, distr) + } + /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution + fn stdnorm(shape: Sh) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + Self::rand(shape, StandardNormal) + } + /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution with a given seed + fn stdnorm_from_seed(shape: Sh, seed: u64) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + Self::rand_with( + shape, + StandardNormal, + &mut rngs::StdRng::seed_from_u64(seed), + ) + } + /// A [uniform](rand_distr::uniform::Uniform) generator with values between u(-dk, dk) + fn uniform(shape: Sh, dk: A) -> Self + where + A: Neg + SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + Self::rand(shape, Uniform::new(dk.clone().neg(), dk)) + } + /// Generate a random array with values between u(-a, a) where a is the reciprocal of the value at the given axis + fn uniform_along(shape: Sh, axis: usize) -> Self + where + A: Copy + Float + SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + let dim = shape.into_shape().raw_dim().clone(); + let dk = A::from(dim[axis]).unwrap().recip(); + Self::uniform(dim, dk) + } + /// A [uniform](rand_distr::uniform::Uniform) generator with values between u(-dk, dk) + fn uniform_between(shape: Sh, a: A, b: A) -> Self + where + A: SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + Self::rand(shape, Uniform::new(a, b)) + } +} +/* + ************ Implementations ************ +*/ +impl Initialize for ArrayBase +where + D: Dimension, + S: RawData, + ArrayBase: RandomExt, +{ + type Data = S; + + fn rand(shape: Sh, distr: Ds) -> ArrayBase + where + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::random(shape, distr) + } + + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> ArrayBase + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::random_using(shape, distr, rng) + } + + fn init_rand(self, distr: Ds) -> ArrayBase + where + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand(self.dim(), distr) + } + + fn init_rand_with(self, distr: Ds, rng: &mut R) -> ArrayBase + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } +} + +impl InitializeExt for U +where + A: Clone, + D: Dimension, + S: RawData, + U: Initialize, +{ +} diff --git a/core/src/init/mod.rs b/core/src/init/mod.rs new file mode 100644 index 00000000..22ee1bac --- /dev/null +++ b/core/src/init/mod.rs @@ -0,0 +1,40 @@ +/* + Appellation: init + Contrib: FL03 +*/ +//! # Initialization +//! +//! This module implements several initialization primitives for generating tensors using +//! various distributions and strategies. The module is designed to be used in conjuction with +//! the `rand` and `rand_distr` libraries. While `ndarray_rand` provides a `RandomExt` trait, +//! we provide an alternative [Initialize] trait which is designed to be more flexible and +//! better suited for machine-learning workloads. +#![cfg(feature = "rand")] + +pub use self::prelude::*; + +pub(crate) mod initialize; +pub(crate) mod utils; + +pub mod gen { + pub use self::prelude::*; + + pub mod lecun; + + pub(crate) mod prelude { + pub use super::lecun::*; + } +} + +#[doc(no_inline)] +pub use ndarray_rand as ndrand; +#[doc(no_inline)] +pub use rand; +#[doc(no_inline)] +pub use rand_distr; + +pub(crate) mod prelude { + pub use super::gen::prelude::*; + pub use super::initialize::{Initialize, InitializeExt}; + pub use super::utils::*; +} diff --git a/core/src/init/utils.rs b/core/src/init/utils.rs new file mode 100644 index 00000000..3994589c --- /dev/null +++ b/core/src/init/utils.rs @@ -0,0 +1,61 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use ndarray::*; +use ndrand::RandomExt; +use num::complex::{Complex, ComplexDistribution}; +use num::Num; +use rand::distributions::uniform::{SampleUniform, Uniform}; +use rand::rngs::StdRng; +use rand::{rngs, SeedableRng}; +use rand_distr::{Distribution, StandardNormal}; + +/// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) +pub fn randc(shape: impl IntoDimension) -> ArrayBase +where + A: Clone + Num, + D: Dimension, + S: DataOwned>, + ComplexDistribution: Distribution>, +{ + let distr = ComplexDistribution::::new(A::one(), A::one()); + ArrayBase::random(shape, distr) +} +/// Creates a random array from a uniform distribution using a given key +pub fn seeded_uniform( + key: u64, + start: T, + stop: T, + shape: impl IntoDimension, +) -> Array +where + D: Dimension, + T: SampleUniform, +{ + Array::random_using( + shape, + Uniform::new(start, stop), + &mut rngs::StdRng::seed_from_u64(key), + ) +} +/// Given a shape, generate a random array using the StandardNormal distribution +pub fn stdnorm(shape: Sh) -> ArrayBase +where + D: Dimension, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, +{ + ArrayBase::random(shape, StandardNormal) +} + +pub fn stdnorm_from_seed(shape: Sh, seed: u64) -> ArrayBase +where + D: Dimension, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, +{ + ArrayBase::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(seed)) +} diff --git a/core/src/lib.rs b/core/src/lib.rs index b7ee82c7..5906aa6f 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -16,7 +16,7 @@ pub use self::nn::Module; pub use self::{primitives::*, traits::prelude::*, types::prelude::*, utils::prelude::*}; #[cfg(feature = "rand")] -pub use self::rand::{GenerateRandom, RandomExt}; +pub use self::init::{Initialize, InitializeExt}; #[macro_use] pub(crate) mod macros; @@ -24,11 +24,11 @@ pub(crate) mod primitives; pub mod error; pub mod func; +pub mod init; +pub mod math; pub mod nn; pub mod ops; -pub mod params; -#[cfg(feature = "rand")] -pub mod rand; + pub mod traits; pub mod types; pub mod utils; @@ -38,12 +38,12 @@ pub mod prelude { pub use super::error::prelude::*; pub use super::func::prelude::*; + #[cfg(feature = "rand")] + pub use super::init::prelude::*; + pub use super::math::prelude::*; pub use super::nn::prelude::*; pub use super::ops::prelude::*; - pub use super::params::prelude::*; pub use super::primitives::*; - #[cfg(feature = "rand")] - pub use super::rand::prelude::*; pub use super::traits::prelude::*; pub use super::types::prelude::*; pub use super::utils::prelude::*; diff --git a/core/src/macros.rs b/core/src/macros.rs index b447975f..aaf6ceb6 100644 --- a/core/src/macros.rs +++ b/core/src/macros.rs @@ -2,115 +2,78 @@ Appellation: macros Contrib: FL03 */ -#![allow(unused_macros)] +#[macro_use] +mod activate; +#[macro_use] +mod builder; +#[macro_use] +mod enums; +#[macro_use] +mod getters; +#[macro_use] +mod ops; +#[macro_use] +mod toggle; -macro_rules! error_from { - ($base:ident::$variant:ident<$($err:ty),* $(,)?>) => { - error_from!(@loop $base::$variant<$($err),*>); - }; - ($base:ident::$variant:ident<$err:ty>$($rest:tt)*) => { - error_from!(@loop $base::$variant<$($err),*>$($rest)*); - }; - (@loop $base:ident::$variant:ident<$($err:ty),* $(,)?>) => { - $( - error_from!(@impl $base::$variant<$err>); - )* - }; - (@impl $base:ident::$variant:ident<$err:ty>) => { - impl From<$err> for $base { - fn from(err: $err) -> Self { - Self::$variant(err.to_string()) - } - } - }; - (@impl $base:ident::$variant:ident<$err:ty>.$method:ident) => { - impl From<$err> for $base { - fn from(err: $err) -> Self { - Self::$variant(err.$method()) - } - } - }; -} +/// AS +#[macro_export] +macro_rules! dimensional { -macro_rules! nested_constructor { - ($variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { - nested_constructor!(@loop $variant<$inner>, $method, [$($call),*]); - }; - (@loop $variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { - pub fn $method(inner:$inner) -> Self { - Self::$variant(inner) + (dim: $name:ident$(())?) => { + /// Returns a reference to the current dimension, as a slice. + pub fn as_slice(&self) -> &[usize] { + self.$name$(())?.shape() } - $( - pub fn $call() -> Self { - Self::$method($inner::$call()) - } - )* - - }; -} - -macro_rules! variant_constructor { - ($($rest:tt),* $(,)?) => { - $( - variant_constructor!(@loop $($rest),*); - )* - }; - ($($variant:ident.$method:ident$(($call:expr))?),* $(,)?) => { - $( - variant_constructor!(@loop $variant.$method$(($call))?); - )* - }; - - (@loop $variant:ident.$method:ident$(($call:expr))?) => { - pub fn $method() -> Self { - Self::$variant$(($call))? + pub fn into_pattern(self) -> D::Pattern { + self.$name$(())?.into_pattern() } - }; -} -macro_rules! impl_unary { - ($name:ident.$call:ident<$T:ty>($f:expr) $($rest:tt)*) => { - impl_unary!(@impl $name.$call<$T>($f) $($rest)*); - }; - (@impl $name:ident.$call:ident<$T:ty>($f:expr)) => { - impl $name for $T { - type Output = $T; + pub fn ndim(&self) -> usize { + self.$name$(())?.ndim() + } - fn $call(&self) -> Self::Output { - $f(self) - } + pub fn raw_dim(&self) -> D { + self.$name$(())?.dim().clone() } }; -} -macro_rules! build_unary_trait { - ($($name:ident.$call:ident),* $(,)?) => { - $( - build_unary_trait!(@impl $name.$call); - )* - }; - (@impl $name:ident.$call:ident) => { - pub trait $name { - type Output; - fn $call(&self) -> Self::Output; + ($name:ident) => { + /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension + pub fn dim(&self) -> D::Pattern { + self.$name.dim() + } + /// Returns rank (ndim) of the dimension + pub fn ndim(&self) -> usize { + self.$name.ndim() + } + /// Returns the raw dimension [D](ndarray::Dimension) + pub fn raw_dim(&self) -> D { + self.$name.dim() + } + /// Returns a reference to the current dimension, as a slice. + pub fn shape(&self) -> &[usize] { + self.$name.shape() } }; -} -macro_rules! linspace { - (start: $start:expr, end: $end:expr, n: $n:expr, dtype: $T:ty) => { - ndarray::Array1::<$T>::linspace($start, $end, $n) - }; - (end: $end:expr, dtype: $T:ty) => { - let n = ($end - $T::one()).to_usize().unwrap(); - ndarray::Array1::<$T>::linspace($T::zero(), $end, $end.to_usize().unwrap()) + ($name:ident()) => { + /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension + pub fn dim(&self) -> D::Pattern { + self.$name().dim() + } + /// Returns rank (ndim) of the dimension + pub fn ndim(&self) -> usize { + self.$name().ndim() + } + /// Returns the raw dimension [D](ndarray::Dimension) + pub fn raw_dim(&self) -> D { + self.$name().raw_dim() + } + /// Returns a reference to the current dimension, as a slice. + pub fn shape(&self) -> &[usize] { + self.$name().shape() + } }; - (dim: $dim:expr, dtype: $T:ty) => {{ - let dim = $dim.into_dimension(); - let n = dim.size(); - ndarray::Array1::<$T>::linspace(<$T>::zero(), <$T>::from(n - 1).unwrap(), n) - .into_shape($dim) - }}; } diff --git a/core/src/macros/activate.rs b/core/src/macros/activate.rs new file mode 100644 index 00000000..44e6eeb3 --- /dev/null +++ b/core/src/macros/activate.rs @@ -0,0 +1,36 @@ +/* + Appellation: activate + Contrib: FL03 +*/ + +macro_rules! activator { + ($name:ident::<$out:ty>($rho:expr) $($rest:tt)*) => { + #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct $name; + + impl $crate::func::activate::Activate for $name $($rest)* { + type Output = $out; + + fn activate(&self, args: &T) -> Self::Output { + $rho(args) + } + } + }; +} + +macro_rules! losses { + (impl<$($T:ident),* $(,)?> $name:ident::<$lhs:ty, $rhs:ty, Output = $out:ty>($loss:expr) $($rest:tt)*) => { + #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct $name; + + impl<$($T),*> $crate::func::Loss<$lhs, $rhs> for $name $($rest)* { + type Output = $out; + + fn loss(&self, a: &$lhs, b: &$rhs) -> Self::Output { + $loss(a, b) + } + } + }; +} diff --git a/core/src/macros/builder.rs b/core/src/macros/builder.rs new file mode 100644 index 00000000..8fba06d2 --- /dev/null +++ b/core/src/macros/builder.rs @@ -0,0 +1,47 @@ +/* + Appellation: builder + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! builder { + ($(#[derive($($d:ident),+)])?$name:ident::<$inner:ty> {$($k:ident: $v:ty),* $(,)?}) => { + builder!(@loop builder: $name, derive: [$($($d),+)?], inner: $inner {$($k: $v),*}); + }; + ($(#[derive($($d:ident),+)])? $name:ident($inner:ty) {$($k:ident: $v:ty),* $(,)?}) => { + builder!(@loop builder: $name, derive: [$($($d),+)?], inner: $inner {$($k: $v),*}); + }; + (@loop builder: $name:ident, derive: [$($d:ident),* $(,)?], inner: $inner:ty {$($k:ident: $v:ty),* $(,)?}) => { + + #[derive(Default, $($d),*)] + pub struct $name { + inner: $inner, + } + + builder!(@impl builder: $name, inner: $inner {$($k: $v),*}); + }; + (@impl builder: $name:ident, inner: $inner:ty {$($k:ident: $v:ty),* $(,)?}) => { + impl $name { + pub fn new() -> Self { + Self { + inner: Default::default() + } + } + + pub fn from_inner(inner: $inner) -> Self { + Self { inner } + } + + pub fn build(self) -> $inner { + self.inner + } + + $( + pub fn $k(mut self, $k: $v) -> Self { + self.inner.$k = $k; + self + } + )* + } + }; +} diff --git a/core/src/macros/enums.rs b/core/src/macros/enums.rs new file mode 100644 index 00000000..ce08a85a --- /dev/null +++ b/core/src/macros/enums.rs @@ -0,0 +1,44 @@ +/* + Appellation: enums + Contrib: FL03 +*/ + +macro_rules! from_variant { + ($base:ident::$variant:ident $($rest:tt)*) => { + from_variant!(@branch $base::$variant $($rest)*); + }; + (@branch $base:ident::$variant:ident($from:ty)$(.$method:ident())*) => { + from_variant!(@impl $base::$variant($from)$(.$method())*); + }; + (@branch $base:ident::$variant:ident{$(<$err:ty>$(.$method:ident())*),* $(,)?}) => { + $( + from_variant!(@impl $base::$variant($err)$(.$method())*); + )* + }; + (@impl $base:ident::$variant:ident($from:ty)$(.$method:ident())*) => { + impl From<$from> for $base { + fn from(val: $from) -> Self { + Self::$variant(val$(.$method())*) + } + } + }; +} + +#[allow(unused_macros)] +macro_rules! nested_enum_constructor { + ($variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { + nested_enum_constructor!(@loop $variant<$inner>, $method, [$($call),*]); + }; + (@loop $variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { + pub fn $method(inner:$inner) -> Self { + Self::$variant(inner) + } + + $( + pub fn $call() -> Self { + Self::$method($inner::$call()) + } + )* + + }; +} diff --git a/core/src/macros/getters.rs b/core/src/macros/getters.rs new file mode 100644 index 00000000..86d232b4 --- /dev/null +++ b/core/src/macros/getters.rs @@ -0,0 +1,47 @@ +/* + Appellation: getters + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! getters { + ($($call:ident$(.$field:ident)?<$out:ty>),* $(,)?) => { + $($crate::getters!(@impl $call$(.$field)?<$out>);)* + }; + ($via:ident::<[$($call:ident$(.$field:ident)?<$out:ty>),* $(,)?]>) => { + $($crate::getters!(@impl $via::$call$(.$field)?<$out>);)* + }; + ($($call:ident$(.$field:ident)?),* $(,)? => $out:ty) => { + $($crate::getters!(@impl $call$(.$field)?<$out>);)* + }; + ($via:ident::<[$($call:ident$(.$field:ident)?),* $(,)?]> => $out:ty) => { + $crate::getters!($via::<[$($call$(.$field)?<$out>),*]>); + }; + + (@impl $call:ident<$out:ty>) => { + $crate::getters!(@impl $call.$call<$out>); + }; + (@impl $via:ident::$call:ident<$out:ty>) => { + $crate::getters!(@impl $via::$call.$call<$out>); + }; + (@impl $call:ident.$field:ident<$out:ty>) => { + pub fn $call(&self) -> &$out { + &self.$field + } + paste::paste! { + pub fn [< $call _mut>](&mut self) -> &mut $out { + &mut self.$field + } + } + }; + (@impl $via:ident::$call:ident.$field:ident<$out:ty>) => { + pub fn $call(&self) -> &$out { + &self.$via.$field + } + paste::paste! { + pub fn [< $call _mut>](&mut self) -> &mut $out { + &mut self.$via.$field + } + } + }; +} diff --git a/core/src/macros/ops.rs b/core/src/macros/ops.rs new file mode 100644 index 00000000..f37089b0 --- /dev/null +++ b/core/src/macros/ops.rs @@ -0,0 +1,36 @@ +/* + Appellation: ops + Contrib: FL03 +*/ + +macro_rules! unary { + ($($name:ident::$call:ident),* $(,)?) => { + $( + unary!(@impl $name::$call(self)); + )* + }; + ($($name:ident::$call:ident(self)),* $(,)?) => { + $( + unary!(@impl $name::$call(self)); + )* + }; + ($($name:ident::$call:ident(&self)),* $(,)?) => { + $( + unary!(@impl $name::$call(&self)); + )* + }; + (@impl $name:ident::$call:ident(self)) => { + pub trait $name { + type Output; + + fn $call(self) -> Self::Output; + } + }; + (@impl $name:ident::$call:ident(&self)) => { + pub trait $name { + type Output; + + fn $call(&self) -> Self::Output; + } + }; +} diff --git a/core/src/macros/toggle.rs b/core/src/macros/toggle.rs new file mode 100644 index 00000000..908f6abf --- /dev/null +++ b/core/src/macros/toggle.rs @@ -0,0 +1,19 @@ +/* + Appellation: toggle + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! toggle { + (enum $($name:ident),* $(,)?) => { + $(toggle!(@enum $name);)* + }; + + (@enum $name:ident) => { + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub enum $name {} + + impl $crate::traits::misc::toggle::Toggle for $name {} + }; +} diff --git a/core/src/math/mod.rs b/core/src/math/mod.rs new file mode 100644 index 00000000..bc9dc16f --- /dev/null +++ b/core/src/math/mod.rs @@ -0,0 +1,15 @@ +/* + Appellation: math + Contrib: FL03 +*/ +//! # Mathematics +//! +//! This module focuses on implementing various mathematical objects and operations that are +//! critical to the development of machine learning algorithms. +pub use self::traits::*; + +pub mod traits; + +pub(crate) mod prelude { + pub use super::traits::*; +} diff --git a/core/src/math/traits.rs b/core/src/math/traits.rs new file mode 100644 index 00000000..d71d433d --- /dev/null +++ b/core/src/math/traits.rs @@ -0,0 +1,152 @@ +/* + Appellation: traits + Contrib: FL03 +*/ +use nd::{Array, ArrayBase, Data, Dimension}; +use num::complex::{Complex, ComplexFloat}; +use num::traits::Signed; + +unary!( + Abs::abs(self), + Cos::cos(self), + Cosh::cosh(self), + Exp::exp(self), + Sine::sin(self), + Sinh::sinh(self), + Squared::sqrd(self), + SquareRoot::sqrt(self) +); + +/* + ********* Implementations ********* +*/ + +macro_rules! unary_impl { + ($name:ident::$method:ident<[$($T:ty),* $(,)?]>) => { + unary_impl!(@loop $name::$method<[$($T),*]>); + }; + ($($name:ident::$method:ident<$T:ty$(, Output = $O:ty)?>),* $(,)?) => { + $(unary_impl!(@impl $name::$method<$T$(, Output = $O>)?);)* + }; + ($($name:ident::$method:ident<$T:ty, Output = $O:ty>),* $(,)?) => { + $(unary_impl!(@impl $name::$method<$T, Output = $O>);)* + }; + (@loop $name:ident::$method:ident<[$($T:ty),* $(,)?]>) => { + $(unary_impl!(@impl $name::$method<$T>);)* + }; + (@impl $name:ident::$method:ident<$T:ty>) => { + unary_impl!(@impl $name::$method<$T, Output = $T>); + }; + (@impl $name:ident::$method:ident<$T:ty, Output = $O:ty>) => { + impl $name for $T { + type Output = $O; + + fn $method(self) -> Self::Output { + <$T>::$method(self) + } + } + }; +} + +macro_rules! unary_impls { + ($($name:ident::$method:ident<[$($T:ty),* $(,)?]>),* $(,)?) => { + $(unary_impl!(@loop $name::$method<[$($T),*]>);)* + }; +} + +unary_impls!( + Abs::abs<[f32, f64]>, + Cosh::cosh<[f32, f64, Complex, Complex]>, + Cos::cos<[f32, f64, Complex, Complex]>, + Exp::exp<[f32, f64, Complex, Complex]>, + Sinh::sinh<[f32, f64, Complex, Complex]>, + Sine::sin<[f32, f64, Complex, Complex]>, + SquareRoot::sqrt<[f32, f64]> +); + +impl Abs for ArrayBase +where + A: Clone + Signed, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn abs(self) -> Self::Output { + self.mapv(|x| x.abs()) + } +} + +impl<'a, A, S, D> Abs for &'a ArrayBase +where + A: Clone + Signed, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn abs(self) -> Self::Output { + self.mapv(|x| x.abs()) + } +} + +impl Squared for A +where + A: Clone + core::ops::Mul, +{ + type Output = A; + + fn sqrd(self) -> Self::Output { + self.clone() * self + } +} + +impl SquareRoot for Complex +where + Complex: ComplexFloat, +{ + type Output = Self; + + fn sqrt(self) -> Self::Output { + ComplexFloat::sqrt(self) + } +} + +impl SquareRoot for ArrayBase +where + A: Clone + SquareRoot, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn sqrt(self) -> Self::Output { + self.mapv(|x| x.sqrt()) + } +} + +impl Exp for ArrayBase +where + A: Clone + Exp, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn exp(self) -> Self::Output { + self.mapv(|x| x.exp()) + } +} + +impl<'a, A, S, D> Exp for &'a ArrayBase +where + A: Clone + ComplexFloat, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn exp(self) -> Self::Output { + self.mapv(|x| x.exp()) + } +} diff --git a/core/src/nn/mod.rs b/core/src/nn/mod.rs index b82d7727..c0eb1f81 100644 --- a/core/src/nn/mod.rs +++ b/core/src/nn/mod.rs @@ -2,15 +2,19 @@ Appellation: nn Contrib: FL03 */ -pub use self::{error::ModelError, models::Module}; +pub use self::{error::ModelError, model::prelude::*}; pub mod error; -pub mod models; +pub mod model; pub(crate) mod prelude { pub use super::error::ModelError; - pub use super::models::prelude::*; + pub use super::model::prelude::*; } +#[cfg(any(feature = "alloc", feature = "std"))] +pub type ForwardDyn, O = T> = + crate::rust::Box>; + #[cfg(test)] mod tests {} diff --git a/core/src/nn/model.rs b/core/src/nn/model.rs new file mode 100644 index 00000000..8991d08e --- /dev/null +++ b/core/src/nn/model.rs @@ -0,0 +1,28 @@ +/* + Appellation: model + Contrib: FL03 +*/ +pub use self::module::*; + +pub mod config; +pub mod module; + +pub(crate) mod prelude { + pub use super::config::*; + pub use super::module::*; + pub use super::Model; +} + +use crate::traits::Forward; + +pub trait Model: Module +where + Self: Forward, +{ + type Ctx; + type Data; + + fn children(&self) -> Vec>; + + fn context(&self) -> Self::Ctx; +} diff --git a/core/src/nn/model/config.rs b/core/src/nn/model/config.rs new file mode 100644 index 00000000..7596c36b --- /dev/null +++ b/core/src/nn/model/config.rs @@ -0,0 +1,18 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use crate::traits::Config; + +pub struct ModelConfig { + pub name: String, + _children: Vec>, +} + +impl Config for ModelConfig {} + +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] +pub struct ConfigBase { + pub id: usize, + pub name: &'static str, +} diff --git a/core/src/nn/models/module.rs b/core/src/nn/model/module.rs similarity index 72% rename from core/src/nn/models/module.rs rename to core/src/nn/model/module.rs index af182f25..8d8ee23b 100644 --- a/core/src/nn/models/module.rs +++ b/core/src/nn/model/module.rs @@ -2,13 +2,17 @@ Appellation: modules Contrib: FL03 */ -use crate::Predict; +use crate::{Config, Predict}; + +pub type ModuleDyn = Box>; +pub type DynModuleExt = Box>; +pub type Stack = Vec>>; /// A `Module` defines any object that may be used as a layer in a neural network. /// [Config](Module::Config) is a type that defines the configuration of the module; including any and all hyperparameters. /// [Params](Module::Params) is a type that defines the parameters of the module; typically references a Linear set of parameters { weights, bias } pub trait Module { - type Config; + type Config: Config; type Params; fn config(&self) -> &Self::Config; @@ -20,4 +24,4 @@ pub trait Module { pub trait ModuleExt: Module + Predict {} -pub type Stack = Vec>>; +impl ModuleExt for M where M: Module + Predict {} diff --git a/core/src/nn/models/mod.rs b/core/src/nn/models/mod.rs deleted file mode 100644 index 37e9ebe7..00000000 --- a/core/src/nn/models/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: models - Contrib: FL03 -*/ -pub use self::{model::*, module::*}; - -pub mod model; -pub mod module; - -pub(crate) mod prelude { - pub use super::model::*; - pub use super::module::*; -} diff --git a/core/src/nn/models/model.rs b/core/src/nn/models/model.rs deleted file mode 100644 index 16687d4b..00000000 --- a/core/src/nn/models/model.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ - -pub trait Model {} diff --git a/core/src/ops/fft/cmp.rs b/core/src/ops/fft/cmp/direction.rs similarity index 92% rename from core/src/ops/fft/cmp.rs rename to core/src/ops/fft/cmp/direction.rs index b22eb815..5daff00a 100644 --- a/core/src/ops/fft/cmp.rs +++ b/core/src/ops/fft/cmp/direction.rs @@ -1,7 +1,8 @@ /* - Appellation: cmp - Contrib: FL03 + Appellation: direction + Contrib: FL03 */ +use scsys::VariantConstructors; use strum::{ AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames, }; @@ -81,6 +82,7 @@ impl From for usize { PartialEq, PartialOrd, VariantArray, + VariantConstructors, VariantNames, )] #[cfg_attr( diff --git a/core/src/ops/fft/cmp/mode.rs b/core/src/ops/fft/cmp/mode.rs new file mode 100644 index 00000000..7d538eae --- /dev/null +++ b/core/src/ops/fft/cmp/mode.rs @@ -0,0 +1,44 @@ +/* + Appellation: mode + Contrib: FL03 +*/ +use scsys::VariantConstructors; +use strum::{ + AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames, +}; + +toggle!(enum C, R); + +/// +#[derive( + AsRefStr, + Clone, + Copy, + Debug, + Default, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + VariantArray, + VariantConstructors, + VariantNames, +)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase", untagged) +)] +#[repr(usize)] +#[strum(serialize_all = "lowercase")] +pub enum FftMode { + #[default] + Complex, + Real, +} diff --git a/core/src/ops/fft/plan.rs b/core/src/ops/fft/cmp/plan.rs similarity index 61% rename from core/src/ops/fft/plan.rs rename to core/src/ops/fft/cmp/plan.rs index 10a82ed7..d6809acb 100644 --- a/core/src/ops/fft/plan.rs +++ b/core/src/ops/fft/cmp/plan.rs @@ -2,56 +2,62 @@ Appellation: plan Contrib: FL03 */ -#[cfg(all(feature = "alloc", no_std))] -use alloc::vec::{self, Vec}; use core::slice; -#[cfg(feature = "std")] -use std::vec; + +use crate::ops::prelude::fft_permutation; #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct FftPlan { - n: usize, + len: usize, plan: Vec, } impl FftPlan { - pub fn new(n: usize) -> Self { - let plan = Vec::with_capacity(n); - Self { n, plan } + pub fn new(len: usize) -> Self { + Self { + len, + plan: Vec::with_capacity(len), + } } pub fn build(self) -> Self { - let mut plan = Vec::with_capacity(self.n); - plan.extend(0..self.n); - - let mut rev = 0; // reverse - let mut pos = 1; // position - while pos < self.n { - let mut bit = self.n >> 1; - while bit & rev != 0 { - rev ^= bit; - bit >>= 1; - } - rev ^= bit; - // This is equivalent to adding 1 to a reversed number - if pos < rev { - // Only swap each element once - plan.swap(pos, rev); - } - pos += 1; - } + let plan = fft_permutation(self.len); Self { plan, ..self } } pub fn clear(&mut self) { - self.n = 0; + self.len = 0; self.plan.clear(); } + pub fn get(&self, index: usize) -> Option<&usize> { + self.plan().get(index) + } + + pub fn iter(&self) -> slice::Iter { + self.plan().iter() + } + + pub fn len(&self) -> usize { + self.len + } + pub fn plan(&self) -> &[usize] { &self.plan } + + pub fn set(&mut self, len: usize) { + self.len = len; + self.plan = Vec::with_capacity(len); + } + + pub fn with(self, len: usize) -> Self { + Self { + len, + plan: Vec::with_capacity(len), + } + } } impl AsRef<[usize]> for FftPlan { @@ -76,15 +82,16 @@ impl FromIterator for FftPlan { fn from_iter>(iter: T) -> Self { let plan = Vec::from_iter(iter); Self { - n: plan.len(), + len: plan.len(), plan, } } } +#[cfg(any(feature = "alloc", feature = "std"))] impl IntoIterator for FftPlan { type Item = usize; - type IntoIter = vec::IntoIter; + type IntoIter = crate::rust::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.plan.into_iter() diff --git a/core/src/ops/fft/fft.rs b/core/src/ops/fft/fft.rs index 80e3b337..d493c24d 100644 --- a/core/src/ops/fft/fft.rs +++ b/core/src/ops/fft/fft.rs @@ -4,21 +4,21 @@ */ use super::{FftDirection, FftPlan}; -pub struct FastFourierTransform { +pub struct Fft { direction: FftDirection, plan: FftPlan, } -impl FastFourierTransform { +impl Fft { pub fn new(direction: FftDirection, plan: FftPlan) -> Self { Self { direction, plan } } - pub fn direction(&self) -> FftDirection { + pub const fn direction(&self) -> FftDirection { self.direction } - pub fn plan(&self) -> &FftPlan { + pub const fn plan(&self) -> &FftPlan { &self.plan } } diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index 150558bc..3306c3b5 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -4,27 +4,39 @@ */ //! # Fast Fourier Transform //! -//! +//! The `fft` module provides an implementation of the Fast Fourier Transform (FFT) algorithm. +//! The Fast Fourier Transform is an efficient algorithm for computing the Discrete Fourier Transform (DFT). pub use self::prelude::*; pub(crate) mod fft; pub(crate) mod utils; -pub mod cmp; -pub mod plan; +pub mod cmp { + pub use self::prelude::*; + + pub mod direction; + pub mod mode; + pub mod plan; + + pub(crate) mod prelude { + pub use super::direction::FftDirection; + pub use super::mode::FftMode; + pub use super::plan::FftPlan; + } +} -pub trait Fft { - type Data; +/// Trait for computing the Discrete Fourier Transform (DFT) of a sequence. +pub trait DFT { + type Output; - fn rfft(&self) -> Self; + fn dft(&self) -> Self::Output; } pub(crate) mod prelude { - pub use super::cmp::*; + pub use super::cmp::prelude::*; pub use super::fft::*; - pub use super::plan::*; pub use super::utils::*; - pub use super::Fft; + pub use super::DFT; } #[cfg(test)] diff --git a/core/src/ops/fft/utils.rs b/core/src/ops/fft/utils.rs index fd1243f4..c3990202 100644 --- a/core/src/ops/fft/utils.rs +++ b/core/src/ops/fft/utils.rs @@ -169,3 +169,30 @@ where let scale = T::from(n).unwrap().recip(); result.iter().map(|x| x.re() * scale).collect() } + +#[doc(hidden)] +/// Generates a permutation for the Fast Fourier Transform. +pub fn fft_permutation(length: usize) -> Vec { + let mut result = Vec::new(); + result.reserve_exact(length); + for i in 0..length { + result.push(i); + } + let mut reverse = 0_usize; + let mut position = 1_usize; + while position < length { + let mut bit = length >> 1; + while bit & reverse != 0 { + reverse ^= bit; + bit >>= 1; + } + reverse ^= bit; + // This is equivalent to adding 1 to a reversed number + if position < reverse { + // Only swap each element once + result.swap(position, reverse); + } + position += 1; + } + result +} diff --git a/core/src/ops/pad.rs b/core/src/ops/pad.rs index 2b9c4367..e6962770 100644 --- a/core/src/ops/pad.rs +++ b/core/src/ops/pad.rs @@ -67,6 +67,7 @@ impl Padding { } mod utils { + #![cfg(any(feature = "std", feature = "alloc"))] use super::{PadAction, PadMode}; use crate::traits::ArrayLike; use nd::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; @@ -77,7 +78,7 @@ mod utils { #[cfg(feature = "std")] use std::borrow::Cow; - fn read_pad(nb_dim: usize, pad: &[[usize; 2]]) -> Cow<[[usize; 2]]> { + fn reader(nb_dim: usize, pad: &[[usize; 2]]) -> Cow<[[usize; 2]]> { if pad.len() == 1 && pad.len() < nb_dim { // The user provided a single padding for all dimensions Cow::from(vec![pad[0]; nb_dim]) @@ -94,7 +95,7 @@ mod utils { D: Dimension, S: DataOwned, { - let pad = read_pad(data.ndim(), pad); + let pad = reader(data.ndim(), pad); let mut new_dim = data.raw_dim(); for (ax, (&ax_len, pad)) in data.shape().iter().zip(pad.iter()).enumerate() { new_dim[ax] = ax_len + pad[0] + pad[1]; @@ -116,7 +117,7 @@ mod utils { D: Dimension, S: Data, { - let pad = read_pad(data.ndim(), pad); + let pad = reader(data.ndim(), pad); // Select portion of padded array that needs to be copied from the original array. output diff --git a/core/src/ops/pad/action.rs b/core/src/ops/pad/action.rs index c3a5c521..15325ddc 100644 --- a/core/src/ops/pad/action.rs +++ b/core/src/ops/pad/action.rs @@ -2,6 +2,7 @@ Appellation: action Contrib: FL03 */ +use scsys::VariantConstructors; use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; #[derive( @@ -20,6 +21,7 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN Ord, PartialEq, PartialOrd, + VariantConstructors, VariantNames, )] #[cfg_attr( diff --git a/core/src/params/impls/impl_rand.rs b/core/src/params/impls/impl_rand.rs deleted file mode 100644 index f8681da9..00000000 --- a/core/src/params/impls/impl_rand.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: impl_rand - Contrib: FL03 -*/ -use crate::params::Parameter; -use crate::rand::GenerateRandom; -use ndarray::{Array, Dimension}; -use ndrand::rand_distr::uniform::SampleUniform; -use ndrand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -impl Parameter -where - D: Dimension, - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init_uniform(mut self, dk: T) -> Self { - let dim = self.value.dim(); - self.value = Array::uniform_between(dk, dim); - self - } -} diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 7f881602..c2755083 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -5,21 +5,20 @@ pub use consts::*; pub mod consts { - - pub const DEFAULT_MODEL_SIZE: usize = 2048; - pub const EPSILON: f64 = 1e-8; + /// The default model size for any given model + pub const D_MODEL: usize = 512; + /// The default epsilon value for floating point operations + pub const EPSILON: f64 = 1e-5; } -#[allow(unused_imports)] +#[allow(unused)] pub(crate) mod rust { - pub(crate) use core::*; - - #[cfg(no_std)] + #[cfg(all(feature = "alloc", no_std))] pub(crate) use self::no_std::*; #[cfg(feature = "std")] pub(crate) use self::with_std::*; - - #[cfg(no_std)] + pub(crate) use core::*; + #[cfg(all(feature = "alloc", no_std))] mod no_std { pub use alloc::borrow::Cow; pub use alloc::boxed::{self, Box}; @@ -31,12 +30,12 @@ pub(crate) mod rust { pub use std::borrow::Cow; pub use std::boxed::{self, Box}; pub use std::collections::{self, BTreeMap, BTreeSet, BinaryHeap, VecDeque}; - pub(crate) use std::sync::Arc; + pub use std::sync::Arc; pub use std::vec::{self, Vec}; } - #[cfg(no_std)] - pub type Map = collections::BTreeMap; + #[cfg(all(feature = "alloc", no_std))] + pub type Map = alloc::collections::BTreeMap; #[cfg(feature = "std")] - pub type Map = collections::HashMap; + pub type Map = std::collections::HashMap; } diff --git a/core/src/rand/gen/lecun.rs b/core/src/rand/gen/lecun.rs deleted file mode 100644 index 33ee0cde..00000000 --- a/core/src/rand/gen/lecun.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: lecun - Contrib: FL03 -*/ diff --git a/core/src/rand/generate.rs b/core/src/rand/generate.rs deleted file mode 100644 index 7e9b27a3..00000000 --- a/core/src/rand/generate.rs +++ /dev/null @@ -1,99 +0,0 @@ -/* - Appellation: generate - Contrib: FL03 -*/ -use core::ops::Neg; -use ndarray::*; -use ndrand::rand::rngs::StdRng; -use ndrand::rand::{Rng, SeedableRng}; -use ndrand::rand_distr::uniform::{SampleUniform, Uniform}; -use ndrand::rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal}; -use ndrand::RandomExt; -use num::traits::real::Real; -use num::traits::Float; - -pub trait GenerateRandom: Sized -where - D: Dimension, -{ - fn rand(dim: Sh, distr: IdS) -> Self - where - IdS: Distribution, - Sh: ShapeBuilder; - - fn rand_using(dim: Sh, distr: IdS, rng: &mut R) -> Self - where - IdS: Distribution, - R: Rng, - Sh: ShapeBuilder; - - fn bernoulli(dim: impl IntoDimension, p: Option) -> Result - where - Bernoulli: Distribution, - { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Self::rand(dim.into_dimension(), dist)) - } - - fn stdnorm(dim: impl IntoDimension) -> Self - where - StandardNormal: Distribution, - { - Self::rand(dim, StandardNormal) - } - - fn normal_from_key(key: u64, dim: impl IntoDimension) -> Self - where - StandardNormal: Distribution, - R: Rng, - { - Self::rand_using( - dim.into_dimension(), - StandardNormal, - &mut StdRng::seed_from_u64(key), - ) - } - /// Generate a random array with values between u(-a, a) where a is the reciprocal of the value at the given axis - fn uniform(axis: usize, dim: impl IntoDimension) -> Self - where - T: Real + SampleUniform, - { - let dim = dim.into_dimension(); - let dk = T::from(dim[axis]).unwrap().recip(); - Self::uniform_between(dk, dim) - } - - fn uniform_between(dk: T, dim: impl IntoDimension) -> Self - where - T: Copy + Neg + SampleUniform, - { - Self::rand(dim, Uniform::new(-dk, dk)) - } -} - -/* - ************ Implementations ************ -*/ -impl GenerateRandom for Array -where - A: Float + SampleUniform, - D: Dimension, - StandardNormal: Distribution, -{ - fn rand(dim: Sh, distr: Dtr) -> Self - where - Dtr: Distribution, - Sh: ShapeBuilder, - { - Self::random(dim, distr) - } - - fn rand_using(dim: Sh, distr: Dtr, rng: &mut R) -> Self - where - Dtr: Distribution, - R: Rng + ?Sized, - Sh: ShapeBuilder, - { - Self::random_using(dim, distr, rng) - } -} diff --git a/core/src/rand/initialize.rs b/core/src/rand/initialize.rs deleted file mode 100644 index f625fac1..00000000 --- a/core/src/rand/initialize.rs +++ /dev/null @@ -1,128 +0,0 @@ -/* - Appellation: initialize - Contrib: FL03 -*/ -use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; -use ndrand::RandomExt; -use num::traits::Float; -use rand::{rngs, Rng, SeedableRng}; -use rand_distr::uniform::{SampleUniform, Uniform}; -use rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal}; - -/// This trait provides the base methods required for initializing an [ndarray](ndarray::ArrayBase) with random values. -/// [Initialize] is similar to [RandomExt](ndarray_rand::RandomExt), however, it focuses on flexibility while implementing additional -/// features geared towards machine-learning models; such as lecun_normal initialization. -pub trait Initialize -where - D: Dimension, - S: RawData, -{ - /// Generate a random array using the given distribution - fn genrand(shape: Sh, distr: Ds) -> ArrayBase - where - S: DataOwned, - Ds: Distribution, - Sh: ShapeBuilder; - /// Generate a random array using the given distribution and random number generator - fn genrand_with(shape: Sh, distr: Ds, rng: &mut R) -> ArrayBase - where - R: Rng + ?Sized, - S: DataOwned, - Ds: Distribution, - Sh: ShapeBuilder; -} - -/// This trait extends the [Initialize] trait with methods for generating random arrays from various distributions. -pub trait InitializeExt: Initialize -where - D: Dimension, - S: RawData, -{ - fn bernoulli(shape: Sh, p: Option) -> Result, BernoulliError> - where - S: DataOwned, - Sh: ShapeBuilder, - Bernoulli: Distribution, - { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Self::genrand(shape, dist)) - } - /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution - fn stdnorm(shape: Sh) -> ArrayBase - where - S: DataOwned, - Sh: ShapeBuilder, - StandardNormal: Distribution, - { - Self::genrand(shape, StandardNormal) - } - /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution with a given seed - fn stdnorm_from_seed(shape: Sh, seed: u64) -> ArrayBase - where - S: DataOwned, - Sh: ShapeBuilder, - StandardNormal: Distribution, - { - Self::genrand_with( - shape, - StandardNormal, - &mut rngs::StdRng::seed_from_u64(seed), - ) - } - /// Generate a random array with values between u(-a, a) where a is the reciprocal of the value at the given axis - fn uniform_along(shape: Sh, axis: usize) -> ArrayBase - where - A: Copy + Float + SampleUniform, - S: DataOwned, - Sh: ShapeBuilder, - { - let dim = shape.into_shape().raw_dim().clone(); - let dk = A::from(dim[axis]).unwrap().recip(); - Self::uniform(dim, -dk, dk) - } - /// A [uniform](rand_distr::uniform::Uniform) generator with values between u(-dk, dk) - fn uniform(shape: Sh, a: A, b: A) -> ArrayBase - where - A: SampleUniform, - S: DataOwned, - Sh: ShapeBuilder, - { - Self::genrand(shape, Uniform::new(a, b)) - } -} -/* - ************ Implementations ************ -*/ -impl Initialize for ArrayBase -where - D: Dimension, - S: RawData, - ArrayBase: RandomExt, -{ - fn genrand(shape: Sh, distr: Ds) -> ArrayBase - where - S: DataOwned, - Ds: Distribution, - Sh: ShapeBuilder, - { - Self::random(shape, distr) - } - - fn genrand_with(shape: Sh, distr: Ds, rng: &mut R) -> ArrayBase - where - R: Rng + ?Sized, - S: DataOwned, - Ds: Distribution, - Sh: ShapeBuilder, - { - Self::random_using(shape, distr, rng) - } -} - -impl InitializeExt for U -where - D: Dimension, - S: RawData, - U: Initialize, -{ -} diff --git a/core/src/rand/mod.rs b/core/src/rand/mod.rs deleted file mode 100644 index 2076b360..00000000 --- a/core/src/rand/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - Appellation: rand - Contrib: FL03 -*/ -#![cfg(feature = "rand")] - -pub use self::prelude::*; - -pub(crate) mod generate; -pub(crate) mod initialize; -pub(crate) mod utils; - -#[doc(hidden)] -pub mod gen { - pub mod lecun; -} - -#[doc(no_inline)] -pub use ndarray_rand as ndrand; -#[doc(no_inline)] -pub use ndrand::{RandomExt, SamplingStrategy}; -#[doc(no_inline)] -pub use rand; -#[doc(no_inline)] -pub use rand_distr; - -pub(crate) mod prelude { - #[doc(hidden)] - pub use super::generate::GenerateRandom; - pub use super::initialize::{Initialize, InitializeExt}; - pub use super::utils::*; -} diff --git a/core/src/rand/utils.rs b/core/src/rand/utils.rs deleted file mode 100644 index c4ab0470..00000000 --- a/core/src/rand/utils.rs +++ /dev/null @@ -1,125 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use ndarray::*; -use ndrand::rand::rngs::StdRng; -use ndrand::rand::{rngs, Rng, SeedableRng}; -use ndrand::rand_distr::{Distribution, StandardNormal}; -use ndrand::RandomExt; -use num::complex::{Complex, ComplexDistribution}; -use num::traits::real::Real; -use num::Num; -use rand::distributions::uniform::{SampleUniform, Uniform}; - -pub fn lecun_normal(shape: impl IntoDimension) -> Array -where - D: Dimension, - T: Real + ScalarOperand, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let n = dim.size(); - let scale = T::from(n).unwrap().recip().sqrt(); - Array::random(dim, StandardNormal) * scale -} - -pub fn lecun_normal_seeded(shape: impl IntoDimension, seed: u64) -> Array -where - D: Dimension, - T: Real + ScalarOperand, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let n = dim.size(); - let scale = T::from(n).unwrap().recip().sqrt(); - Array::random_using(dim, StandardNormal, &mut rngs::StdRng::seed_from_u64(seed)) * scale -} - -/// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) -pub fn randc(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Clone + Num, - ComplexDistribution: Distribution>, -{ - let distr = ComplexDistribution::::new(T::one(), T::one()); - Array::random(shape, distr) -} -/// -pub fn randcomplex(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = Array::random(dim.clone(), StandardNormal); - let im = Array::random(dim.clone(), StandardNormal); - let mut res = Array::zeros(dim); - ndarray::azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} -/// Creates a random array from a uniform distribution using a given key -pub fn seeded_uniform( - key: u64, - start: T, - stop: T, - shape: impl IntoDimension, -) -> Array -where - D: Dimension, - T: SampleUniform, -{ - Array::random_using( - shape, - Uniform::new(start, stop), - &mut rngs::StdRng::seed_from_u64(key), - ) -} -/// -pub fn seeded_stdnorm(shape: impl IntoDimension, key: u64) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random_using(shape, StandardNormal, &mut rngs::StdRng::seed_from_u64(key)) -} -/// -pub fn randc_normal(key: u64, shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = seeded_stdnorm(dim.clone(), key); - let im = seeded_stdnorm(dim.clone(), key); - let mut res = Array::zeros(dim); - azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} -/// Given a shape, generate a random array using the StandardNormal distribution -pub fn stdnorm(shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random(shape, StandardNormal) -} - -pub fn stdnorm_from_seed(shape: Sh, seed: u64) -> ArrayBase -where - D: Dimension, - R: Rng + ?Sized, - S: DataOwned, - Sh: ShapeBuilder, - StandardNormal: Distribution, - ArrayBase: RandomExt, -{ - ArrayBase::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(seed)) -} diff --git a/core/src/traits/arr/create.rs b/core/src/traits/arr/create.rs index 918cb6a3..b99a5eaa 100644 --- a/core/src/traits/arr/create.rs +++ b/core/src/traits/arr/create.rs @@ -2,10 +2,10 @@ Appellation: create Contrib: FL03 */ -use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; +use nd::{ArrayBase, DataOwned, Dimension, Ix2, ShapeBuilder}; use num::traits::Num; -pub trait TensorConstructor +pub trait NdLike where Self: DefaultLike + FillLike @@ -14,63 +14,38 @@ where { } -pub trait ArrayLike +pub trait ArrayLike where D: Dimension, - S: RawData, { - fn array_like(&self, shape: Sh, elem: A) -> ArrayBase - where - Sh: ShapeBuilder; -} - -impl ArrayLike for ArrayBase -where - A: Clone, - D: Dimension, - S: nd::DataOwned, -{ - fn array_like(&self, shape: Sh, elem: A) -> ArrayBase - where - Sh: ShapeBuilder, - { - if self.is_standard_layout() { - ArrayBase::from_elem(shape, elem) - } else { - ArrayBase::from_elem(shape.f(), elem) - } - } -} - -pub trait DefaultLike { type Output; - fn default_like(&self) -> Self::Output; + fn array_like(&self, shape: Sh, elem: A) -> Self::Output + where + Sh: ShapeBuilder; } -pub trait FillLike { - type Output; - - fn fill_like(&self, elem: T) -> Self::Output; -} +macro_rules! ndlike { + ($($name:ident::$(<$($T:ident),*>::)?$method:ident $(($($field:ident:$ft:ty),*))?),* $(,)?) => { + $(ndlike!(@impl $name::$(<$($T),*>::)?$method$(($($field:$ft),*))?);)* + }; + (@impl $name:ident::$(<$($T:ident),*>::)?$method:ident$(($($field:ident: $ft:ty),*))?) => { + pub trait $name$(<$($T),*>)? { + type Output; -pub trait OnesLike { - type Output; + fn $method(&self $(, $($field:$ft),*)?) -> Self::Output; + } + }; - fn ones_like(&self) -> Self::Output; } -pub trait ZerosLike { - type Output; - - fn zeros_like(&self) -> Self::Output; -} +ndlike!(DefaultLike::default_like, OnesLike::ones_like, ZerosLike::zeros_like, FillLike::::fill_like(elem: T)); /* ******** implementations ******** */ -impl TensorConstructor> for ArrayBase +impl NdLike> for ArrayBase where A: Clone + Default + Num, D: Dimension, @@ -78,6 +53,26 @@ where { } +impl ArrayLike for ArrayBase +where + A: Clone, + D: Dimension, + S: nd::DataOwned, +{ + type Output = ArrayBase; + + fn array_like(&self, shape: Sh, elem: A) -> Self::Output + where + Sh: ShapeBuilder, + { + if self.is_standard_layout() { + ArrayBase::from_elem(shape, elem) + } else { + ArrayBase::from_elem(shape.f(), elem) + } + } +} + impl FillLike for ArrayBase where A: Clone, diff --git a/core/src/traits/arr/misc.rs b/core/src/traits/arr/misc.rs index 1c5ac006..4cc76e4c 100644 --- a/core/src/traits/arr/misc.rs +++ b/core/src/traits/arr/misc.rs @@ -5,6 +5,16 @@ use nd::Axis; use nd::{ArrayBase, Dimension, RawData}; +pub trait Dimensional { + type Pattern; + + fn dim(&self) -> Self::Pattern; + + fn raw_dim(&self) -> D; + + fn shape(&self) -> &[usize]; +} + pub trait IntoAxis { fn into_axis(self) -> Axis; } @@ -16,6 +26,26 @@ pub trait IsSquare { /* ******** implementations ******** */ +impl Dimensional for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Pattern = D::Pattern; + + fn shape(&self) -> &[usize] { + ArrayBase::shape(self) + } + + fn dim(&self) -> Self::Pattern { + ArrayBase::dim(self) + } + + fn raw_dim(&self) -> D { + ArrayBase::raw_dim(self) + } +} + impl IntoAxis for S where S: AsRef, @@ -31,6 +61,7 @@ where S: RawData, { fn is_square(&self) -> bool { - self.shape().iter().all(|&x| x == self.shape()[0]) + let first = self.shape().first().unwrap(); + self.shape().iter().all(|x| x == first) } } diff --git a/core/src/traits/arr/ops.rs b/core/src/traits/arr/ops.rs index 60d829f7..5fc97c9e 100644 --- a/core/src/traits/arr/ops.rs +++ b/core/src/traits/arr/ops.rs @@ -1,6 +1,6 @@ /* - Appellation: arr - Contrib: FL03 + Appellation: ops + Contrib: FL03 */ use nd::linalg::Dot; use nd::*; diff --git a/core/src/traits/arr/shape.rs b/core/src/traits/arr/shape.rs deleted file mode 100644 index b72e993a..00000000 --- a/core/src/traits/arr/shape.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - Appellation: shape - Contrib: FL03 -*/ -use nd::{ArrayBase, Dimension, RawData}; - -pub trait Dimensional { - type Pattern; - - fn dim(&self) -> Self::Pattern; - - fn raw_dim(&self) -> D; - - fn shape(&self) -> &[usize]; -} - -/* - ********* Implementations ********* -*/ -impl Dimensional for ArrayBase -where - D: Dimension, - S: RawData, -{ - type Pattern = D::Pattern; - - fn shape(&self) -> &[usize] { - ArrayBase::shape(self) - } - - fn dim(&self) -> Self::Pattern { - ArrayBase::dim(self) - } - - fn raw_dim(&self) -> D { - ArrayBase::raw_dim(self) - } -} diff --git a/core/src/traits/arr/tensor.rs b/core/src/traits/arr/tensor.rs new file mode 100644 index 00000000..22a00f99 --- /dev/null +++ b/core/src/traits/arr/tensor.rs @@ -0,0 +1,264 @@ +/* + Appellation: generator + Contrib: FL03 +*/ +use nd::prelude::*; +use nd::{Data, DataMut, DataOwned, OwnedRepr, RawData}; +use num::{One, Zero}; + +/// [NdBuilder] describes common creation routines for [ArrayBase](ndarray::ArrayBase) +pub trait NdBuilder +where + D: Dimension, +{ + type Data: RawData; + type Store; + + /// Create a new array with the given shape whose elements are set to the default value of the element type. + fn default(shape: Sh) -> Self::Store + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn fill(shape: Sh, elem: A) -> Self::Store + where + A: Clone, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn ones(shape: Sh) -> Self::Store + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn zeros(shape: Sh) -> Self::Store + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned; +} + +pub trait NdBuilderExt: NdBuilder +where + D: Dimension, +{ + fn dim(&self) -> D::Pattern; + + fn default_like(&self) -> Self::Store + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::default(self.dim()) + } + + fn fill_like(&self, elem: A) -> Self::Store + where + A: Clone, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::fill(self.dim(), elem) + } + + fn ones_like(&self) -> Self::Store + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::ones(self.dim()) + } + + fn zeros_like(&self) -> Self::Store + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::zeros(self.dim()) + } +} + +pub trait AsOwned +where + D: Dimension, + S: RawData, +{ + type Output; + + fn into_owned(self) -> Self::Output + where + S: Data, + S::Elem: Clone; + + fn to_owned(&self) -> Self::Output + where + S: Data, + S::Elem: Clone; +} + +pub trait AsShared +where + D: Dimension, + S: RawData, +{ + type Output; + + fn into_shared(self) -> Self::Output + where + S: DataOwned, + S::Elem: Clone; + + fn to_shared(&self) -> Self::Output + where + S: DataOwned, + S::Elem: Clone; +} + +pub trait NdView, D = Ix2>: AsOwned + AsShared +where + D: Dimension, + S: RawData, +{ + fn view(&self) -> ArrayView<'_, A, D> + where + A: Clone, + S: Data; + + fn view_mut(&mut self) -> ArrayViewMut<'_, A, D> + where + A: Clone, + S: DataMut; +} + +/* + ************* Implementations ************* +*/ +impl NdBuilder for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Data = S; + type Store = ArrayBase; + + fn default(shape: Sh) -> Self + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::default(shape) + } + + fn fill(shape: Sh, elem: A) -> Self + where + A: Clone, + S: DataOwned, + Sh: ShapeBuilder, + { + ArrayBase::from_elem(shape, elem) + } + + fn ones(shape: Sh) -> Self + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::ones(shape) + } + + fn zeros(shape: Sh) -> Self + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::zeros(shape) + } +} + +impl NdBuilderExt for ArrayBase +where + D: Dimension, + S: RawData, +{ + fn dim(&self) -> D::Pattern { + ArrayBase::dim(self) + } +} + +impl AsOwned for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Output = Array; + + fn into_owned(self) -> Self::Output + where + A: Clone, + S: Data, + { + self.into_owned() + } + + fn to_owned(&self) -> Self::Output + where + A: Clone, + S: Data, + { + self.to_owned() + } +} + +impl AsShared for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Output = ArcArray; + + fn into_shared(self) -> Self::Output + where + A: Clone, + S: DataOwned, + { + self.into_shared() + } + + fn to_shared(&self) -> Self::Output + where + A: Clone, + S: DataOwned, + { + self.to_shared() + } +} + +impl NdView for ArrayBase +where + D: Dimension, + S: RawData, +{ + fn view(&self) -> ArrayView<'_, A, D> + where + A: Clone, + S: Data, + { + self.view() + } + + fn view_mut(&mut self) -> ArrayViewMut<'_, A, D> + where + A: Clone, + S: DataMut, + { + self.view_mut() + } +} diff --git a/core/src/traits/misc/setup.rs b/core/src/traits/misc/setup.rs index 1b7856f9..d6f2f611 100644 --- a/core/src/traits/misc/setup.rs +++ b/core/src/traits/misc/setup.rs @@ -2,6 +2,26 @@ Appellation: setup Contrib: FL03 */ +use core::borrow::{Borrow, BorrowMut}; + +/// A trait used to denote objects that may be used for configuring various items +pub trait Config {} + +/// [Configuration] describes composite configuration objects; +/// A configuration object is allowed to inherit from another configuration object +pub trait Configuration +where + C: Config, + Self::Config: Borrow, +{ + type Config: Config; + + fn root(&self) -> &C; + + fn set(&mut self, config: Self::Config); + + fn set_root(&mut self, config: C); +} pub trait Init { fn init(self) -> Self; @@ -16,3 +36,36 @@ pub trait Setup { fn setup(&mut self, config: Self::Config); } + +pub trait Context +where + C: Config, +{ + type Cnf: Configuration; + + fn config(&self) -> Self::Cnf; +} + +/* + ************* Implementations ************* +*/ + +impl Configuration for D +where + C: Config, + D: Config + BorrowMut, +{ + type Config = D; + + fn root(&self) -> &C { + self.borrow() + } + + fn set(&mut self, config: Self::Config) { + *self = config; + } + + fn set_root(&mut self, config: C) { + *self.borrow_mut() = config; + } +} diff --git a/core/src/traits/misc/toggle.rs b/core/src/traits/misc/toggle.rs new file mode 100644 index 00000000..4865d33f --- /dev/null +++ b/core/src/traits/misc/toggle.rs @@ -0,0 +1,48 @@ +/* + Appellation: toggle + Contrib: FL03 +*/ + +pub trait Toggle: 'static {} + +pub trait OfType { + fn of() -> bool + where + T: 'static, + Self: 'static, + { + core::any::TypeId::of::() == core::any::TypeId::of::() + } +} + +/* + ************* Implementations ************* +*/ +impl OfType for T {} + +macro_rules! impl_toggle { + ($($scope:ident$(<$T:ident>)?),* $(,)?) => { + $(impl_toggle!(@impl $scope$(<$T>)?);)* + }; + (@impl $scope:ident$(<$T:ident>)?) => { + impl$(<$T>)? Toggle for $scope$(<$T> where $T: 'static)? {} + }; +} + +impl_toggle!( + bool, + char, + i8, + i16, + i32, + i64, + i128, + isize, + u8, + u16, + u32, + u64, + u128, + usize, + Option +); diff --git a/core/src/traits/mod.rs b/core/src/traits/mod.rs index 8ee0a14a..b6aa6b21 100644 --- a/core/src/traits/mod.rs +++ b/core/src/traits/mod.rs @@ -4,7 +4,8 @@ */ pub use self::prelude::*; -pub mod math; +pub mod num; +pub mod ops; pub mod predict; pub mod train; @@ -14,47 +15,38 @@ pub mod arr { pub(crate) mod create; pub(crate) mod misc; pub(crate) mod ops; - pub(crate) mod shape; + pub(crate) mod tensor; pub(crate) mod prelude { pub use super::create::*; pub use super::misc::*; pub use super::ops::*; - pub use super::shape::*; + pub use super::tensor::*; } } -pub(crate) mod misc { +pub mod misc { pub mod adjust; #[doc(hidden)] pub mod container; pub mod setup; pub mod store; + pub mod toggle; pub(crate) mod prelude { pub use super::adjust::*; pub use super::container::*; pub use super::setup::*; pub use super::store::*; + pub use super::toggle::*; } } -pub trait Transform { - type Output; - - fn transform(&self, args: &T) -> Self::Output; -} - pub(crate) mod prelude { - pub use super::Transform; - - pub use super::math::*; - pub use super::predict::*; - pub use super::train::*; - pub use super::arr::prelude::*; pub use super::misc::prelude::*; + pub use super::num::*; + pub use super::ops::*; + pub use super::predict::*; + pub use super::train::*; } - -#[cfg(test)] -mod tests {} diff --git a/core/src/traits/math.rs b/core/src/traits/num.rs similarity index 85% rename from core/src/traits/math.rs rename to core/src/traits/num.rs index 1d499a72..d21e77f4 100644 --- a/core/src/traits/math.rs +++ b/core/src/traits/num.rs @@ -52,10 +52,6 @@ pub trait RoundTo { fn round_to(&self, places: usize) -> Self; } -pub trait SquareRoot { - fn sqrt(self) -> Self; -} - /* ********* Implementations ********* */ @@ -151,34 +147,3 @@ where crate::round_to(*self, places) } } - -impl SquareRoot for f32 { - fn sqrt(self) -> Self { - f32::sqrt(self) - } -} - -impl SquareRoot for f64 { - fn sqrt(self) -> Self { - f64::sqrt(self) - } -} - -impl SquareRoot for Complex -where - T: Float, -{ - fn sqrt(self) -> Self { - Complex::::sqrt(self) - } -} - -impl SquareRoot for Array -where - D: Dimension, - T: Float, -{ - fn sqrt(self) -> Self { - self.mapv(|x| x.sqrt()) - } -} diff --git a/core/src/traits/ops.rs b/core/src/traits/ops.rs new file mode 100644 index 00000000..48e1cf3a --- /dev/null +++ b/core/src/traits/ops.rs @@ -0,0 +1,47 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +/// A trait for applying a function to a type +pub trait Apply { + type Output; + + fn apply(&self, f: F) -> Self::Output + where + F: Fn(T) -> U; + + fn apply_mut(&mut self, f: F) -> Self::Output + where + F: FnMut(T) -> U; +} + +pub trait ApplyOn { + type Output; + + fn apply(self, f: F) -> Self::Output + where + F: FnMut(T) -> U; +} + +pub trait Transform { + type Output; + + fn transform(&self, args: &T) -> Self::Output; +} + +/* + ************* Implementations ************* +*/ +impl ApplyOn for S +where + S: Iterator, +{ + type Output = core::iter::Map; + + fn apply(self, f: F) -> Self::Output + where + F: FnMut(T) -> U, + { + self.map(f) + } +} diff --git a/core/src/traits/predict.rs b/core/src/traits/predict.rs index fd294757..632d05b1 100644 --- a/core/src/traits/predict.rs +++ b/core/src/traits/predict.rs @@ -4,16 +4,6 @@ */ use crate::error::PredictError; -#[doc(hidden)] -pub trait Activate: Forward -where - F: Fn(&Self::Output) -> Self::Output, -{ - fn activate(&self, args: &T, f: F) -> Self::Output { - f(&self.forward(args)) - } -} - /// [Forward] describes an object capable of forward propagation. pub trait Forward { type Output; @@ -39,17 +29,17 @@ pub trait Predict { /* ********* Implementations ********* */ -impl Forward for Option +impl Forward for S where - S: Forward, - T: Clone, + S: Predict, { - type Output = T; + type Output = Y; - fn forward(&self, args: &T) -> Self::Output { - match self { - Some(s) => s.forward(args), - None => args.clone(), + fn forward(&self, args: &X) -> Self::Output { + if let Ok(y) = self.predict(args) { + y + } else { + panic!("Error in forward propagation") } } } diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs index 655b3673..a639d5a4 100644 --- a/core/src/types/mod.rs +++ b/core/src/types/mod.rs @@ -6,25 +6,26 @@ pub use self::prelude::*; #[cfg(feature = "std")] pub use self::std_types::*; -pub mod direction; +pub mod propagate; +pub type NdResult = core::result::Result; /// A type alias for a [Result](core::result::Result) with the crate's [Error](crate::error::Error) type. /// Defaults to `Result<(), Error>` pub type Result = core::result::Result; #[cfg(feature = "std")] mod std_types { - /// + /// A type alias for a boxed [Error](std::error::Error) type that is `Send`, `Sync`, and `'static`. pub type BoxError = Box; - /// + /// A type alias for a boxed [Result](core::result::Result) which returns some object, `T`, and uses a [BoxError] as the error type. pub type BoxResult = core::result::Result; } pub(crate) mod prelude { - pub use super::direction::Direction; + pub use super::propagate::Propagate; #[cfg(feature = "std")] pub use super::std_types::*; - pub use super::Result; + pub use super::{NdResult, Result}; } #[cfg(test)] diff --git a/core/src/types/direction.rs b/core/src/types/propagate.rs similarity index 87% rename from core/src/types/direction.rs rename to core/src/types/propagate.rs index 2a86ada7..d765ccc5 100644 --- a/core/src/types/direction.rs +++ b/core/src/types/propagate.rs @@ -29,13 +29,13 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN serde(rename_all = "lowercase") )] #[strum(serialize_all = "lowercase")] -pub enum Direction { +pub enum Propagate { Backward = 0, #[default] Forward = 1, } -impl Direction { +impl Propagate { /// A functional alias for [Direction::Backward]. pub fn backward() -> Self { Self::Backward @@ -46,13 +46,13 @@ impl Direction { } } -impl From for usize { - fn from(direction: Direction) -> Self { +impl From for usize { + fn from(direction: Propagate) -> Self { direction as usize } } -impl From for Direction { +impl From for Propagate { fn from(index: usize) -> Self { match index % Self::COUNT { 0 => Self::Backward, diff --git a/core/tests/fft.rs b/core/tests/fft.rs index 279e28e7..61fff93b 100644 --- a/core/tests/fft.rs +++ b/core/tests/fft.rs @@ -12,31 +12,6 @@ use num::traits::Float; const EPSILON: f64 = 1e-6; -fn fft_permutation(length: usize) -> Vec { - let mut result = Vec::new(); - result.reserve_exact(length); - for i in 0..length { - result.push(i); - } - let mut reverse = 0_usize; - let mut position = 1_usize; - while position < length { - let mut bit = length >> 1; - while bit & reverse != 0 { - reverse ^= bit; - bit >>= 1; - } - reverse ^= bit; - // This is equivalent to adding 1 to a reversed number - if position < reverse { - // Only swap each element once - result.swap(position, reverse); - } - position += 1; - } - result -} - lazy_static! { static ref EXPECTED_RFFT: Vec> = vec![ Complex { re: 28.0, im: 0.0 }, diff --git a/core/tests/func.rs b/core/tests/func.rs new file mode 100644 index 00000000..e1a5ccef --- /dev/null +++ b/core/tests/func.rs @@ -0,0 +1,18 @@ +#![allow(unused_imports)] +extern crate concision_core as concision; + +use concision::func::Dropout; +use concision::Forward; +use ndarray::prelude::*; + +#[test] +#[cfg(feature = "rand")] +fn test_dropout() { + let shape = (512, 2048); + let arr = Array2::::ones(shape); + let dropout = Dropout::new(0.5); + let out = dropout.forward(&arr); + + assert!(arr.iter().all(|&x| x == 1.0)); + assert!(out.iter().any(|&x| x == 0.0)); +} diff --git a/core/tests/random.rs b/core/tests/random.rs index 1e7343a4..daa76435 100644 --- a/core/tests/random.rs +++ b/core/tests/random.rs @@ -4,7 +4,7 @@ */ extern crate concision_core as cnc; -use cnc::rand::InitializeExt; +use cnc::init::InitializeExt; use ndarray::prelude::*; #[test] diff --git a/data/Cargo.toml b/data/Cargo.toml index 0ca51862..e6a1c57f 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -80,6 +80,10 @@ crate-type = ["lib"] doctest = false test = true +[[test]] +name = "params" +required-features = ["std"] + [build-dependencies] [dependencies] diff --git a/data/src/lib.rs b/data/src/lib.rs index 31348676..0186ca18 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -11,17 +11,20 @@ extern crate alloc; extern crate concision_core as concision; +extern crate ndarray as nd; pub use self::dataset::Dataset; pub use self::traits::prelude::*; pub mod dataset; +pub mod params; #[doc(hidden)] pub mod preproc; pub mod tensor; pub mod traits; pub mod prelude { - pub use crate::dataset::*; - pub use crate::traits::prelude::*; + pub use super::dataset::*; + pub use super::params::prelude::*; + pub use super::traits::prelude::*; } diff --git a/data/src/params/impls/impl_rand.rs b/data/src/params/impls/impl_rand.rs new file mode 100644 index 00000000..d5f066cc --- /dev/null +++ b/data/src/params/impls/impl_rand.rs @@ -0,0 +1,27 @@ +/* + Appellation: impl_rand + Contrib: FL03 +*/ +use crate::params::Parameter; +use concision::init::rand_distr::uniform::SampleUniform; +use concision::init::rand_distr::{Distribution, StandardNormal}; +use concision::InitializeExt; +use ndarray::{Array, Dimension}; +use num::Float; + +impl Parameter +where + D: Dimension, + T: Float, + StandardNormal: Distribution, +{ + pub fn init_uniform(mut self, dk: T) -> Self + where + T: SampleUniform, + ::Sampler: Clone, + { + let dim = self.value.dim(); + self.value = Array::uniform(dim, dk); + self + } +} diff --git a/core/src/params/kinds.rs b/data/src/params/kinds.rs similarity index 86% rename from core/src/params/kinds.rs rename to data/src/params/kinds.rs index 70a71885..b9b92225 100644 --- a/core/src/params/kinds.rs +++ b/data/src/params/kinds.rs @@ -4,19 +4,6 @@ */ use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; -pub trait ParamType: ToString { - fn kind(&self) -> String; -} - -impl ParamType for T -where - T: ToString, -{ - fn kind(&self) -> String { - self.to_string() - } -} - #[derive( AsRefStr, Clone, diff --git a/core/src/params/mod.rs b/data/src/params/mod.rs similarity index 91% rename from core/src/params/mod.rs rename to data/src/params/mod.rs index 5ec083bf..8b35b030 100644 --- a/core/src/params/mod.rs +++ b/data/src/params/mod.rs @@ -37,9 +37,9 @@ pub(crate) mod prelude { #[cfg(test)] mod tests { use super::*; - use crate::linarr; - use ndarray::linalg::Dot; - use ndarray::prelude::{Ix1, Ix2}; + use concision::linarr; + use nd::linalg::Dot; + use nd::prelude::*; #[test] fn test_parameter() { diff --git a/core/src/params/parameter.rs b/data/src/params/parameter.rs similarity index 100% rename from core/src/params/parameter.rs rename to data/src/params/parameter.rs diff --git a/core/src/params/store.rs b/data/src/params/store.rs similarity index 93% rename from core/src/params/store.rs rename to data/src/params/store.rs index 6428c985..cf451e37 100644 --- a/core/src/params/store.rs +++ b/data/src/params/store.rs @@ -2,11 +2,16 @@ Appellation: store Contrib: FL03 */ +#![cfg(any(feature = "alloc", feature = "std"))] use super::{ParamKind, Parameter}; -use crate::prelude::Map; use ndarray::prelude::{Dimension, Ix2}; use num::Float; +#[cfg(all(feature = "alloc", no_std))] +use alloc::collections::BTreeMap as Map; +#[cfg(feature = "std")] +use std::collections::HashMap as Map; + #[derive(Clone, Debug, Default, Eq, PartialEq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct ParamStore diff --git a/core/tests/params.rs b/data/tests/params.rs similarity index 85% rename from core/tests/params.rs rename to data/tests/params.rs index e121f607..134ff343 100644 --- a/core/tests/params.rs +++ b/data/tests/params.rs @@ -2,9 +2,11 @@ Appellation: params Contrib: FL03 */ -extern crate concision_core as cnc; +extern crate concision_core as concision; +extern crate concision_data as data; -use cnc::prelude::{linarr, ParamKind, Parameter}; +use concision::linarr; +use data::params::{ParamKind, Parameter}; use ndarray::linalg::Dot; use ndarray::prelude::*; diff --git a/models/linear/Cargo.toml b/models/linear/Cargo.toml index 50ee7411..ab7770d1 100644 --- a/models/linear/Cargo.toml +++ b/models/linear/Cargo.toml @@ -87,12 +87,16 @@ doctest = true test = true [[test]] -name = "model" -required-features = ["rand"] +name = "linear" +required-features = ["std"] + +[[test]] +name = "norm" +required-features = ["approx", "std"] [[test]] name = "params" -required-features = ["rand"] +required-features = ["std"] [build-dependencies] @@ -106,6 +110,11 @@ strum.workspace = true optional = true version = "0.5" +[dependencies.concision-core] +default-features = false +path = "../../core" +version = "0.1.14" + [dependencies.serde] default-features = false features = ["derive"] @@ -116,12 +125,6 @@ version = "1" optional = true version = "0.1" - -[dependencies.concision-core] -default-features = false -path = "../../core" -version = "0.1.14" - [dev-dependencies] lazy_static.workspace = true diff --git a/models/linear/src/impls/impl_rand.rs b/models/linear/src/impls/impl_rand.rs index 8b441fcc..f2e602e1 100644 --- a/models/linear/src/impls/impl_rand.rs +++ b/models/linear/src/impls/impl_rand.rs @@ -6,69 +6,186 @@ use crate::params::{ParamMode, ParamsBase}; use crate::{bias_dim, Linear}; -use concision::prelude::GenerateRandom; -use concision::rand::rand_distr::{uniform, Distribution, StandardNormal}; +use concision::init::rand::Rng; +use concision::init::rand_distr::{uniform::SampleUniform, Distribution, StandardNormal}; +use concision::{Initialize, InitializeExt}; use nd::*; use num::Float; impl Linear where - A: Float + uniform::SampleUniform, + A: Clone + Float, D: RemoveAxis, K: ParamMode, StandardNormal: Distribution, { - pub fn uniform(self) -> Self { - let biased = self.is_biased(); + pub fn uniform(self) -> Self + where + A: SampleUniform, + ::Sampler: Clone, + { Self { - params: self.params.init_uniform(biased), + params: self.params.uniform(), ..self } } } -impl ParamsBase, D, K> +impl crate::LinearParams where - A: Float + uniform::SampleUniform, + A: Clone + Float + SampleUniform, D: RemoveAxis, K: ParamMode, StandardNormal: Distribution, + ::Sampler: Clone, { + /// Computes the reciprocal of the input features. pub(crate) fn dk(&self) -> A { - A::from(self.in_features()).unwrap().recip().sqrt() + A::from(self.in_features()).unwrap().recip() + } + /// Computes the square root of the reciprical of the input features. + pub(crate) fn dk_sqrt(&self) -> A { + self.dk().sqrt() + } + + pub fn uniform(self) -> Self { + let dk = self.dk_sqrt(); + self.uniform_between(-dk, dk) } - pub fn init_uniform(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); + pub fn uniform_between(self, low: A, high: A) -> Self { + if self.is_biased() && !self.bias.is_some() { + let b_dim = bias_dim(self.raw_dim()); + Self { + bias: Some(Array::uniform_between(b_dim, low, high)), + weight: Array::uniform_between(self.raw_dim(), low, high), + _mode: self._mode, + } + } else if !self.is_biased() && self.bias.is_some() { + Self { + bias: None, + weight: Array::uniform_between(self.raw_dim(), low, high), + _mode: self._mode, + } + } else { + Self { + bias: self + .bias + .as_ref() + .map(|b| Array::uniform_between(b.raw_dim(), low, high)), + weight: Array::uniform_between(self.raw_dim(), low, high), + _mode: self._mode, + } } - self.init_weight() + } +} + +impl Initialize for Linear +where + D: RemoveAxis, + K: ParamMode, + StandardNormal: Distribution, +{ + type Data = OwnedRepr; + fn rand(shape: Sh, distr: Ds) -> Self + where + Sh: ShapeBuilder, + Ds: Clone + Distribution, + { + Self::from_params(ParamsBase::rand(shape, distr)) } - pub fn init_bias(mut self) -> Self { - let dim = bias_dim(self.raw_dim()); - self.bias = Some(Array::uniform_between(self.dk(), dim)); - self + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::from_params(ParamsBase::rand_with(shape, distr, rng)) } - pub fn init_weight(mut self) -> Self { - self.weights = Array::uniform_between(self.dk(), self.raw_dim()); - self + fn init_rand(self, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Self: Sized, + { + Self::rand(self.dim(), distr) } - pub fn uniform(self) -> Self { - let dk = self.dk(); - let bias = if self.is_biased() { - let dim = bias_dim(self.raw_dim()); - Some(Array::uniform_between(dk, dim)) + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } +} + +impl Initialize for ParamsBase +where + D: RemoveAxis, + K: ParamMode, + S: DataOwned, + StandardNormal: Distribution, +{ + type Data = S; + fn rand(shape: Sh, distr: Dstr) -> Self + where + Sh: ShapeBuilder, + Dstr: Clone + Distribution, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::rand(bias_dim(dim.clone()), distr.clone())) } else { None }; - let weights = Array::uniform_between(dk, self.raw_dim()); Self { + weight: ArrayBase::rand(dim, distr), bias, - weights, - _mode: self._mode, + _mode: core::marker::PhantomData::, } } + + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::rand_with( + bias_dim(dim.clone()), + distr.clone(), + rng, + )) + } else { + None + }; + Self { + weight: ArrayBase::rand_with(dim, distr, rng), + bias, + _mode: core::marker::PhantomData::, + } + } + + fn init_rand(self, distr: Ds) -> Self + where + S: DataOwned, + Ds: Clone + Distribution, + Self: Sized, + { + Self::rand(self.dim(), distr) + } + + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } } diff --git a/models/linear/src/impls/model/impl_linear.rs b/models/linear/src/impls/model/impl_linear.rs index e182436b..03c97f82 100644 --- a/models/linear/src/impls/model/impl_linear.rs +++ b/models/linear/src/impls/model/impl_linear.rs @@ -15,7 +15,7 @@ where A: Clone + Default, { let config = Config::std(inputs, outputs); - let params = LinearParams::default(config.dim()); + let params = LinearParams::new(config.dim()); Self { config, params } } } @@ -23,29 +23,26 @@ where impl Borrow> for Linear where D: RemoveAxis, - K: ParamMode, { fn borrow(&self) -> &Config { &self.config } } -impl Borrow> for Linear +impl Borrow> for Linear where D: RemoveAxis, - K: ParamMode, { - fn borrow(&self) -> &LinearParams { + fn borrow(&self) -> &LinearParams { &self.params } } -impl BorrowMut> for Linear +impl BorrowMut> for Linear where D: RemoveAxis, - K: ParamMode, { - fn borrow_mut(&mut self) -> &mut LinearParams { + fn borrow_mut(&mut self) -> &mut LinearParams { &mut self.params } } diff --git a/models/linear/src/impls/model/impl_model.rs b/models/linear/src/impls/model/impl_model.rs index d61633f1..475c66bd 100644 --- a/models/linear/src/impls/model/impl_model.rs +++ b/models/linear/src/impls/model/impl_model.rs @@ -2,17 +2,16 @@ Appellation: impl_model Contrib: FL03 */ -use crate::{Config, Linear, LinearParams, ParamMode}; +use crate::{Config, Linear, LinearParams}; use concision::prelude::{Module, Predict, PredictError}; use nd::RemoveAxis; impl Module for Linear where D: RemoveAxis, - K: ParamMode, { type Config = Config; - type Params = LinearParams; + type Params = LinearParams; fn config(&self) -> &Self::Config { &self.config @@ -30,8 +29,7 @@ where impl Predict for Linear where D: RemoveAxis, - K: ParamMode, - LinearParams: Predict, + LinearParams: Predict, { type Output = V; diff --git a/models/linear/src/impls/params/impl_from.rs b/models/linear/src/impls/params/impl_from.rs index 2ea9df1e..108d7bd3 100644 --- a/models/linear/src/impls/params/impl_from.rs +++ b/models/linear/src/impls/params/impl_from.rs @@ -2,8 +2,7 @@ Appellation: impl_from Contrib: FL03 */ -use crate::params::{Biased, NodeBase, Pair, ParamMode, ParamsBase, Unbiased}; -use crate::Features; +use crate::{Biased, Features, NodeBase, Pair, ParamsBase, Unbiased}; #[cfg(all(feature = "alloc", no_std))] use alloc::vec; use core::marker::PhantomData; @@ -24,10 +23,9 @@ where fn into_iter(self) -> Self::IntoIter { let axis = Axis(0); - let bias = self.bias().unwrap(); self.weights() .axis_iter(axis) - .zip(bias.axis_iter(axis)) + .zip(self.bias().axis_iter(axis)) .map(|(w, b)| (w.to_owned(), b.to_owned())) .collect::>() .into_iter() @@ -63,7 +61,7 @@ where let mut iter = nodes.iter(); let node = iter.next().unwrap(); let shape = Features::new(node.0.len(), nodes.len()); - let mut params = ParamsBase::default(shape); + let mut params = ParamsBase::new(shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { params.set_node(i + 1, node.clone()); @@ -92,7 +90,7 @@ where let bias = ArrayBase::from_elem((), bias); Self { bias: Some(bias), - weights, + weight: weights, _mode: PhantomData, } } @@ -100,12 +98,11 @@ where impl From<(Array1, Option)> for ParamsBase, Ix1, K> where A: Clone, - K: ParamMode, { fn from((weights, bias): (Array1, Option)) -> Self { Self { bias: bias.map(|b| ArrayBase::from_elem((), b)), - weights, + weight: weights, _mode: PhantomData, } } @@ -114,13 +111,12 @@ where impl From> for ParamsBase where D: RemoveAxis, - K: ParamMode, S: RawData, { fn from((weights, bias): NodeBase) -> Self { Self { bias, - weights, + weight: weights, _mode: PhantomData::, } } @@ -134,7 +130,7 @@ where fn from((weights, bias): Pair, ArrayBase>) -> Self { Self { bias: Some(bias), - weights, + weight: weights, _mode: PhantomData::, } } diff --git a/models/linear/src/impls/params/impl_params.rs b/models/linear/src/impls/params/impl_params.rs index 1f29c8d3..17ae9863 100644 --- a/models/linear/src/impls/params/impl_params.rs +++ b/models/linear/src/impls/params/impl_params.rs @@ -2,7 +2,6 @@ Appellation: params Contrib: FL03 */ -use crate::params::mode::*; use crate::params::ParamsBase; use concision::prelude::{Predict, PredictError}; use core::ops::Add; @@ -13,7 +12,6 @@ use num::complex::ComplexFloat; impl ParamsBase where D: RemoveAxis, - K: ParamMode, S: RawData, { pub fn activate(&mut self, args: &X, f: F) -> Y @@ -26,27 +24,6 @@ where } } -impl<'a, A, B, T, S, D, K> Predict for &'a ParamsBase -where - A: Dot, Output = B>, - B: Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: NdFloat, -{ - type Output = B; - - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } -} - impl Clone for ParamsBase where A: Clone, @@ -55,7 +32,7 @@ where { fn clone(&self) -> Self { Self { - weights: self.weights.clone(), + weight: self.weight.clone(), bias: self.bias.clone(), _mode: self._mode, } @@ -78,84 +55,69 @@ where S: Data, { fn eq(&self, other: &Self) -> bool { - self.weights == other.weights && self.bias == other.bias + self.weights() == other.weight && self.bias == other.bias } } -impl PartialEq<(ArrayBase, Option>)> for ParamsBase +impl PartialEq<(ArrayBase, Option>)> + for ParamsBase where A: PartialEq, D: RemoveAxis, S: Data, { fn eq(&self, (weights, bias): &(ArrayBase, Option>)) -> bool { - self.weights == weights && self.bias == *bias + self.weights() == weights && self.bias.as_ref() == bias.as_ref() } } -impl PartialEq<(ArrayBase, ArrayBase)> for ParamsBase +impl PartialEq<(ArrayBase, ArrayBase)> for ParamsBase where A: PartialEq, D: RemoveAxis, S: Data, { fn eq(&self, (weights, bias): &(ArrayBase, ArrayBase)) -> bool { - let mut cmp = self.weights == weights; - if let Some(b) = &self.bias { - cmp &= b == bias; - } - cmp + self.weights() == weights && self.bias.as_ref() == Some(bias) } } -macro_rules! impl_predict { - ($( $($lt:lifetime)? $name:ident),* $(,)?) => { - $(impl_predict!(@impl $($lt)? $name);)* - }; - (@impl $name:ident) => { - impl Predict for $name - where - A: Dot, Output = B>, - B: for<'a> Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: ComplexFloat, - { - type Output = B; +impl Predict for ParamsBase +where + A: Dot, Output = B>, + B: for<'a> Add<&'a ArrayBase, Output = B>, + D: RemoveAxis, + S: Data, + T: ComplexFloat, +{ + type Output = B; - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } + fn predict(&self, input: &A) -> Result { + let wt = self.weights().t().to_owned(); + let mut res = input.dot(&wt); + if let Some(bias) = self.bias.as_ref() { + res = res + bias; } - }; - (@impl $lt:lifetime $name:ident) => { - impl<'a, A, B, T, S, D, K> Predict for $name - where - A: Dot, Output = B>, - B: for<'a> Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: ComplexFloat, - { - type Output = B; + Ok(res) + } +} + +impl<'a, A, B, T, S, D, K> Predict for &'a ParamsBase +where + A: Dot, Output = B>, + B: Add<&'a ArrayBase, Output = B>, + D: RemoveAxis, + S: Data, + T: ComplexFloat, +{ + type Output = B; - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } + fn predict(&self, input: &A) -> Result { + let wt = self.weights().t().to_owned(); + let mut res = input.dot(&wt); + if let Some(bias) = self.bias.as_ref() { + res = res + bias; } - }; + Ok(res) + } } - -impl_predict!(ParamsBase); diff --git a/models/linear/src/impls/params/impl_serde.rs b/models/linear/src/impls/params/impl_serde.rs index 9d0b77b5..407c0c69 100644 --- a/models/linear/src/impls/params/impl_serde.rs +++ b/models/linear/src/impls/params/impl_serde.rs @@ -4,7 +4,7 @@ */ #![cfg(feature = "serde")] -use crate::params::{Entry, ParamMode, ParamsBase}; +use crate::params::{Parameter, ParamsBase}; use core::marker::PhantomData; use nd::*; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -23,7 +23,7 @@ where let (bias, weights) = Deserialize::deserialize(deserializer)?; Ok(Self { bias, - weights, + weight: weights, _mode: PhantomData, }) } @@ -33,7 +33,6 @@ impl Serialize for ParamsBase where A: Serialize, D: RemoveAxis + Serialize, - K: ParamMode, S: Data, ::Smaller: Dimension + Serialize, { @@ -41,11 +40,11 @@ where where Ser: Serializer, { - (self.bias(), self.weights()).serialize(serializer) + (self.bias.as_ref(), self.weights()).serialize(serializer) } } -impl Serialize for Entry +impl Serialize for Parameter where A: Serialize, S: Data, diff --git a/models/linear/src/lib.rs b/models/linear/src/lib.rs index 81709daa..6f457ab7 100644 --- a/models/linear/src/lib.rs +++ b/models/linear/src/lib.rs @@ -14,14 +14,17 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; +// extern crate ndarray_stats as ndstats; pub use self::model::{Config, Features, Layout, Linear}; -pub use self::params::{mode::*, LinearParams}; +pub use self::norm::LayerNorm; +pub use self::params::{mode::*, ParamsBase}; #[allow(unused_imports)] -pub use self::{traits::*, utils::*}; +pub use self::{primitives::*, traits::*, utils::*}; #[macro_use] pub(crate) mod macros; +pub(crate) mod primitives; #[macro_use] pub(crate) mod seal; pub(crate) mod utils; @@ -33,6 +36,7 @@ pub mod dense; #[doc(hidden)] pub mod mlp; pub mod model; +pub mod norm; pub mod params; pub mod traits; @@ -53,6 +57,7 @@ mod impls { pub mod prelude { pub use crate::model::prelude::*; + pub use crate::norm::prelude::*; pub use crate::params::prelude::*; pub use crate::traits::*; } diff --git a/models/linear/src/macros.rs b/models/linear/src/macros.rs index 977fa5a3..b5672fb9 100644 --- a/models/linear/src/macros.rs +++ b/models/linear/src/macros.rs @@ -3,46 +3,7 @@ Contrib: FL03 */ -#[allow(unused_macros)] -macro_rules! params { - {$($k:ident: $v:expr),* $(,)?} => { - params!(@new $($k: $v),*); - }; - (@new bias: $b:expr, weights: $w:expr, mode: $mode:ty) => { - $crate::params::ParamsBase { - bias: $b, - weights: $w, - _mode: core::marker::PhantomData::<$mode>, - } - }; - (@new bias: $b:expr, weights: $w:expr) => { - params!(@new bias: $b, weights: $w, mode: $crate::params::mode::Biased); - }; - (@new bias: $b:expr, weights: $w:expr) => { - params!(@new bias: Some($b), weights: $w, mode: $crate::params::mode::Biased); - }; - (@new weights: $w:expr) => { - params!(@new bias: None, weights: $w, mode: $crate::params::mode::Unbiased); - }; -} - -macro_rules! impl_param_builder { - ($call:ident where $($rest:tt)*) => { - impl_param_builder!(@impl $call where $($rest)*); - }; - (@impl $call:ident where $($rest:tt)*) => { - pub fn $call(shape: Sh) -> Self - where - Sh: ndarray::ShapeBuilder, - $($rest)* - { - let shape = shape.into_shape(); - let dim = shape.raw_dim().clone(); - ParamsBase { - bias: build_bias(K::BIASED, dim.clone(), |dim| ndarray::ArrayBase::$call(dim)), - weights: ndarray::ArrayBase::$call(dim), - _mode: core::marker::PhantomData, - } - } - }; -} +#[macro_use] +mod model; +#[macro_use] +mod params; diff --git a/models/linear/src/macros/model.rs b/models/linear/src/macros/model.rs new file mode 100644 index 00000000..e0cba303 --- /dev/null +++ b/models/linear/src/macros/model.rs @@ -0,0 +1,22 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +macro_rules! mbuilder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + mbuilder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + mbuilder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + Sh: ndarray::ShapeBuilder, + $($rest)* + { + Linear::from_params($crate::params::ParamsBase::$call(shape)) + } + }; +} diff --git a/models/linear/src/macros/params.rs b/models/linear/src/macros/params.rs new file mode 100644 index 00000000..7da00db5 --- /dev/null +++ b/models/linear/src/macros/params.rs @@ -0,0 +1,92 @@ +/* + Appellation: params + Contrib: FL03 +*/ + +macro_rules! pbuilder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + pbuilder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + pbuilder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + K: $crate::params::mode::ParamMode, + Sh: ndarray::ShapeBuilder, + $($rest)* + { + let dim = shape.into_shape().raw_dim().clone(); + ParamsBase { + bias: build_bias(K::BIASED, dim.clone(), |dim| ndarray::ArrayBase::$call(dim)), + weight: ndarray::ArrayBase::$call(dim), + _mode: ::core::marker::PhantomData::, + } + } + }; +} + +macro_rules! wnbview { + ($method:ident::$($rest:tt)*) => { + wnbview!(@impl $method.$method::$($rest)*); + }; + ($method:ident.$call:ident::$($rest:tt)*) => { + wnbview!(@impl $method.$call::$($rest)*); + }; + (@impl $method:ident.$call:ident::<$view:ident>(self) where $($rest:tt)*) => { + pub fn $method(self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(mut self) where $($rest:tt)*) => { + pub fn $method(mut self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&self) where $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_ref()) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&self) where $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::ParamsBase<$view<&'_ A>, D, K> + where + $($rest)* + { + wnbview!(@apply $call(&self).as_ref()) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::ParamsBase<$view<&'_ mut A>, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@apply $call:ident($self:expr)$(.$as:ident())?) => { + $crate::params::ParamsBase { + bias: $self.bias$(.$as())?.map(|arr| arr.$call()), + weight: $self.weight.$call(), + _mode: $self._mode, + } + }; +} diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index 218ee165..c040a93a 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -18,6 +18,10 @@ pub trait MultiLayerPerceptron { type Output; } -pub trait Neuron { - type Rho; +pub trait Neuron {} + +pub trait Rho { + type Output; + + fn activate(&self, args: T) -> Self::Output; } diff --git a/models/linear/src/model/config.rs b/models/linear/src/model/config.rs index 568cfc25..ad6f9347 100644 --- a/models/linear/src/model/config.rs +++ b/models/linear/src/model/config.rs @@ -3,47 +3,62 @@ Contrib: FL03 */ use super::layout::{Features, Layout}; -use crate::params::mode::*; +use crate::params::{Biased, Unbiased}; use core::marker::PhantomData; -use nd::{Dimension, IntoDimension, Ix2, RemoveAxis}; +use nd::prelude::*; +use nd::{IntoDimension, RemoveAxis, ShapeError}; #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Config -where - D: Dimension, -{ +pub struct Config { pub layout: Layout, pub name: String, - _biased: PhantomData, + _biased: PhantomData, } impl Config where D: Dimension, - K: ParamMode, { pub fn new() -> Self { Self { layout: Layout::default(), name: String::new(), - _biased: PhantomData, + _biased: PhantomData::, } } + pub fn from_dim(dim: D) -> Result + where + D: Dimension, + { + let layout = Layout::from_dim(dim)?; + let res = Self::new().with_layout(layout); + Ok(res) + } + + pub fn from_shape(shape: Sh) -> Self + where + D: RemoveAxis, + Sh: ShapeBuilder, + { + let layout = Layout::from_shape(shape); + Self::new().with_layout(layout) + } + pub fn into_biased(self) -> Config { Config { - _biased: PhantomData, layout: self.layout, name: self.name, + _biased: PhantomData::, } } pub fn into_unbiased(self) -> Config { Config { - _biased: PhantomData, layout: self.layout, name: self.name, + _biased: PhantomData::, } } @@ -65,14 +80,19 @@ where } } - pub fn dim(&self) -> D { - self.layout.dim().clone() - } - - pub fn into_pattern(self) -> D::Pattern { - self.dim().into_pattern() + pub fn with_shape(self, shape: Sh) -> Config + where + E: RemoveAxis, + Sh: ShapeBuilder, + { + Config { + layout: self.layout.with_shape(shape), + name: self.name, + _biased: self._biased, + } } + /// This function attempts to convert the [layout](Layout) of the [Config] into a new [dimension](ndarray::Dimension) pub fn into_dimensionality(self, dim: E) -> Result, nd::ShapeError> where E: Dimension, @@ -84,7 +104,8 @@ where }; Ok(tmp) } - + /// Determine whether the [Config] is [Biased]; + /// Returns true by comparing the [TypeId](core::any::TypeId) of `K` against the [TypeId](core::any::TypeId) of the [Biased] type pub fn is_biased(&self) -> bool where K: 'static, @@ -93,21 +114,30 @@ where TypeId::of::() == TypeId::of::() } - + /// Returns an instance to the [Features] of the [Layout] pub fn features(&self) -> Features { - self.layout.features() + self.layout().features() } - - pub fn layout(&self) -> &Layout { + /// Returns an owned reference to the [Layout] + pub const fn layout(&self) -> &Layout { &self.layout } - + /// Returns an immutable reference to the `name` of the model. pub fn name(&self) -> &str { &self.name } + /// Returns a cloned reference to the [dimension](ndarray::Dimension) of the [layout](Layout) + pub fn dim(&self) -> D { + self.layout().dim() + } + + pub fn into_pattern(self) -> D::Pattern { + self.dim().into_pattern() + } + pub fn ndim(&self) -> usize { - self.layout.ndim() + self.layout().ndim() } } @@ -116,7 +146,7 @@ impl Config { Self { layout: Layout::new((outputs, inputs).into_dimension()), name: String::new(), - _biased: PhantomData, + _biased: PhantomData::, } } } @@ -125,17 +155,10 @@ impl Config where D: Dimension, { + /// The default constructor method for building [Biased] configurations. pub fn biased() -> Self { Self::new() } - - pub fn from_dim_biased(dim: D) -> Self - where - D: RemoveAxis, - { - let layout = Layout::from_dim(dim).unwrap(); - Self::new().with_layout(layout) - } } impl Config @@ -145,15 +168,10 @@ where pub fn unbiased() -> Self { Self::new() } - - pub fn from_dim(dim: D) -> Config - where - D: RemoveAxis, - { - Config::::new().with_layout(Layout::from_dim(dim).unwrap()) - } } +impl concision::Config for Config where D: Dimension {} + impl Default for Config where D: Dimension, diff --git a/models/linear/src/model/linear.rs b/models/linear/src/model/layer.rs similarity index 51% rename from models/linear/src/model/linear.rs rename to models/linear/src/model/layer.rs index b10bca02..486bb43d 100644 --- a/models/linear/src/model/linear.rs +++ b/models/linear/src/model/layer.rs @@ -3,21 +3,22 @@ Contrib: FL03 */ use super::{Config, Layout}; -use crate::{Biased, LinearParams, ParamMode}; +use crate::{Biased, LinearParams, ParamMode, Unbiased}; use concision::prelude::{Predict, Result}; -use nd::{Array, Dimension, Ix2, RemoveAxis}; +use nd::prelude::*; +use nd::RemoveAxis; /// An implementation of a linear model. /// /// In an effort to streamline the api, the [Linear] model relies upon a [ParamMode] type ([Biased] or [Unbiased](crate::params::mode::Unbiased)) -/// which enables the model to automatically determine whether or not to include a bias term. Doing so enables us to forward many methods +/// which enables the model to automatically determine whether or not to include a bias term. Doing so allows the model to inherit several methods /// familar to the underlying [ndarray](https://docs.rs/ndarray) crate. pub struct Linear where D: Dimension, { pub(crate) config: Config, - pub(crate) params: LinearParams, + pub(crate) params: LinearParams, } impl Linear @@ -25,38 +26,41 @@ where D: RemoveAxis, K: ParamMode, { + mbuilder!(new where A: Default); + mbuilder!(ones where A: Clone + num::One); + mbuilder!(zeros where A: Clone + num::Zero); + pub fn from_config(config: Config) -> Self where A: Clone + Default, - K: 'static, + K: ParamMode, { - let params = LinearParams::default(config.dim()); + let params = LinearParams::new(config.dim()); Self { config, params } } pub fn from_layout(layout: Layout) -> Self where A: Clone + Default, + K: ParamMode, { let config = Config::::new().with_layout(layout); - let params = LinearParams::default(config.dim()); + let params = LinearParams::new(config.dim()); Self { config, params } } - pub fn with_params(self, params: LinearParams) -> Linear - where - E: RemoveAxis, - { - let config = self.config.into_dimensionality(params.raw_dim()).unwrap(); - Linear { config, params } + pub fn from_params(params: LinearParams) -> Self { + let config = Config::::new().with_shape(params.raw_dim()); + Self { config, params } } + /// Applies an activcation function onto the prediction of the model. pub fn activate(&self, args: &X, func: F) -> Result where - F: for<'a> Fn(&'a Y) -> Y, + F: Fn(Y) -> Y, Self: Predict, { - Ok(func(&self.predict(args)?)) + Ok(func(self.predict(args)?)) } pub const fn config(&self) -> &Config { @@ -71,17 +75,18 @@ where self.params.weights_mut() } - pub const fn params(&self) -> &LinearParams { + pub const fn params(&self) -> &LinearParams { &self.params } - pub fn params_mut(&mut self) -> &mut LinearParams { + pub fn params_mut(&mut self) -> &mut LinearParams { &mut self.params } pub fn into_biased(self) -> Linear where A: Default, + K: 'static, { Linear { config: self.config.into_biased(), @@ -89,8 +94,30 @@ where } } - pub fn is_biased(&self) -> bool { - K::BIASED || self.config().is_biased() + pub fn into_unbiased(self) -> Linear + where + A: Default, + K: 'static, + { + Linear { + config: self.config.into_unbiased(), + params: self.params.into_unbiased(), + } + } + + pub fn is_biased(&self) -> bool + where + K: 'static, + { + self.config().is_biased() + } + + pub fn with_params(self, params: LinearParams) -> Linear + where + E: RemoveAxis, + { + let config = self.config.into_dimensionality(params.raw_dim()).unwrap(); + Linear { config, params } } pub fn with_name(self, name: impl ToString) -> Self { @@ -99,17 +126,44 @@ where ..self } } + + concision::dimensional!(params()); } impl Linear where D: RemoveAxis, { + pub fn biased(shape: Sh) -> Self + where + A: Default, + Sh: ShapeBuilder, + { + let config = Config::::new().with_shape(shape); + let params = LinearParams::biased(config.dim()); + Linear { config, params } + } + pub fn bias(&self) -> &Array { - self.params.bias().unwrap() + self.params().bias() } pub fn bias_mut(&mut self) -> &mut Array { - self.params.bias_mut().unwrap() + self.params_mut().bias_mut() + } +} + +impl Linear +where + D: RemoveAxis, +{ + pub fn unbiased(shape: Sh) -> Self + where + A: Default, + Sh: ShapeBuilder, + { + let config = Config::::new().with_shape(shape); + let params = LinearParams::unbiased(config.dim()); + Linear { config, params } } } diff --git a/models/linear/src/model/layout/layout.rs b/models/linear/src/model/layout/layout.rs index d4ce877a..0cf5df2f 100644 --- a/models/linear/src/model/layout/layout.rs +++ b/models/linear/src/model/layout/layout.rs @@ -9,10 +9,7 @@ use nd::{Dimension, RemoveAxis, ShapeBuilder, ShapeError}; #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Layout -where - D: Dimension, -{ +pub struct Layout { pub(crate) dim: D, pub(crate) features: Features, } @@ -44,6 +41,17 @@ where Self { dim, features } } + pub fn with_shape(self, shape: Sh) -> Layout + where + E: RemoveAxis, + Sh: ShapeBuilder, + { + let shape = shape.into_shape(); + let dim = shape.raw_dim().clone(); + let features = Features::from_shape(dim.clone()); + Layout { dim, features } + } + pub fn as_slice(&self) -> &[usize] { self.dim.slice() } @@ -52,8 +60,8 @@ where self.dim.slice_mut() } - pub fn dim(&self) -> &D { - &self.dim + pub fn dim(&self) -> D { + self.dim.clone() } pub fn features(&self) -> Features { diff --git a/models/linear/src/model/mod.rs b/models/linear/src/model/mod.rs index f511cb77..66fdce93 100644 --- a/models/linear/src/model/mod.rs +++ b/models/linear/src/model/mod.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ pub use self::layout::prelude::*; -pub use self::{config::Config, linear::Linear}; +pub use self::{config::Config, layer::Linear}; -mod linear; +mod layer; pub mod config; @@ -22,5 +22,5 @@ pub mod layout { } pub(crate) mod prelude { - pub use super::linear::Linear; + pub use super::layer::Linear; } diff --git a/models/linear/src/norm/batch/mod.rs b/models/linear/src/norm/batch/mod.rs new file mode 100644 index 00000000..35c46ebd --- /dev/null +++ b/models/linear/src/norm/batch/mod.rs @@ -0,0 +1,14 @@ +/* + Appellation: batch + Contrib: FL03 +*/ +//! # Batch Normalization +//! +//! +pub use self::model::*; + +mod model; + +pub(crate) mod prelude { + pub use super::BatchNorm; +} diff --git a/models/linear/src/norm/batch/model.rs b/models/linear/src/norm/batch/model.rs new file mode 100644 index 00000000..ab33e536 --- /dev/null +++ b/models/linear/src/norm/batch/model.rs @@ -0,0 +1,6 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +pub struct BatchNorm; diff --git a/models/linear/src/norm/layer/config.rs b/models/linear/src/norm/layer/config.rs new file mode 100644 index 00000000..ef9e7469 --- /dev/null +++ b/models/linear/src/norm/layer/config.rs @@ -0,0 +1,116 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use super::EPSILON; +use nd::prelude::{Axis, Dimension, Ix2}; + +pub struct Config { + pub axis: Option, + pub dim: D, + pub eps: f64, +} + +impl Config +where + D: Dimension, +{ + pub fn new() -> ConfigBuilder { + ConfigBuilder::new() + } + + pub fn axis(&self) -> Option<&Axis> { + self.axis.as_ref() + } + + pub fn axis_mut(&mut self) -> &mut Option { + &mut self.axis + } + + pub fn eps(&self) -> f64 { + self.eps + } + + pub fn eps_mut(&mut self) -> &mut f64 { + &mut self.eps + } + + pub fn dim(&self) -> D::Pattern { + self.raw_dim().into_pattern() + } + + pub fn dim_mut(&mut self) -> &mut D { + &mut self.dim + } + + pub fn ndim(&self) -> usize { + self.dim.ndim() + } + + pub fn raw_dim(&self) -> D { + self.dim.clone() + } + + pub fn shape(&self) -> &[usize] { + self.dim.slice() + } + + pub fn shape_mut(&mut self) -> &mut [usize] { + self.dim.slice_mut() + } +} + +impl Default for Config +where + D: Default, +{ + fn default() -> Self { + Self { + axis: None, + dim: D::default(), + eps: EPSILON, + } + } +} + +pub struct ConfigBuilder { + axis: Option, + dim: D, + eps: f64, +} + +impl ConfigBuilder +where + D: Dimension, +{ + pub fn new() -> Self { + Self { + axis: None, + dim: D::default(), + eps: 1e-5, + } + } + + pub fn axis(mut self, axis: Axis) -> Self { + self.axis = Some(axis); + self + } + + pub fn dim(mut self, dim: D) -> Self { + self.dim = dim; + self + } + + pub fn eps(mut self, eps: f64) -> Self { + self.eps = eps; + self + } + + pub fn build(self) -> Config { + Config { + axis: self.axis, + dim: self.dim, + eps: self.eps, + } + } +} diff --git a/models/linear/src/norm/layer/mod.rs b/models/linear/src/norm/layer/mod.rs new file mode 100644 index 00000000..6b54d6e8 --- /dev/null +++ b/models/linear/src/norm/layer/mod.rs @@ -0,0 +1,51 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +//! # Layer Normalization +//! +//! This module provides the necessary tools for creating and training layer normalization layers. +pub(crate) use self::utils::*; +pub use self::{config::*, model::*}; + +pub(crate) mod config; +pub(crate) mod model; + +pub const EPSILON: f64 = 1e-5; + +pub(crate) mod prelude { + pub use super::config::Config as LayerNormConfig; + pub use super::model::LayerNorm; +} + +pub(crate) mod utils { + use nd::{Array, Axis, Dimension, RemoveAxis}; + use num::traits::{Float, FromPrimitive}; + + pub(crate) fn layer_norm(x: &Array, eps: f64) -> Array + where + A: Float + FromPrimitive, + D: Dimension, + { + let mean = x.mean().unwrap(); + let denom = { + let eps = A::from(eps).unwrap(); + let var = x.var(A::zero()); + (var + eps).sqrt() + }; + x.mapv(|xi| (xi - mean) / denom) + } + + pub(crate) fn layer_norm_axis(x: &Array, axis: Axis, eps: f64) -> Array + where + A: Float + FromPrimitive, + D: RemoveAxis, + { + let eps = A::from(eps).unwrap(); + let mean = x.mean_axis(axis).unwrap(); + let var = x.var_axis(axis, A::zero()); + let inv_std = var.mapv(|v| (v + eps).recip().sqrt()); + let x_norm = (x - &mean) * &inv_std; + x_norm + } +} diff --git a/models/linear/src/norm/layer/model.rs b/models/linear/src/norm/layer/model.rs new file mode 100644 index 00000000..e5dc6b67 --- /dev/null +++ b/models/linear/src/norm/layer/model.rs @@ -0,0 +1,174 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use super::Config; +use crate::{Biased, LinearParams, ParamMode, Unbiased}; +use concision::Forward; +use nd::prelude::*; +use nd::RemoveAxis; +use num::traits::{Float, FromPrimitive, One, Zero}; + +// #62 +/// +/// Layer Normalization directly estimates the normalization statistics from the summed inputs +/// to the neurons within a _hidden_ layer, eliminating the need to introduce any additional dependencies. +/// +/// [LayerNorm] follows the [Layer Normalization](https://arxiv.org/abs/1607.06450) paper. +/// +/// ### Resources +pub struct LayerNorm +where + D: Dimension, +{ + config: Config, + params: LinearParams, +} + +macro_rules! impl_norm_builder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + impl_norm_builder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + impl_norm_builder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + Sh: ShapeBuilder, + $($rest)* + { + Self::from_params(LinearParams::::$call(shape)) + } + }; +} + +impl LayerNorm +where + D: RemoveAxis, + K: ParamMode, +{ + pub fn from_config(config: Config) -> Self + where + A: Default, + { + let params = LinearParams::::new(config.dim()); + Self { config, params } + } + + pub fn from_elem(shape: Sh, elem: A) -> Self + where + A: Clone, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let config = Config::new().dim(dim.clone()).build(); + let params = LinearParams::::from_elem(dim, elem); + Self { config, params } + } + + pub fn from_params(params: LinearParams) -> Self { + let config = Config::new().dim(params.raw_dim()).build(); + Self { config, params } + } + + impl_norm_builder!(new where A: Default); + impl_norm_builder!(ones where A: Clone + One); + impl_norm_builder!(zeros where A: Clone + Zero); + + pub const fn config(&self) -> &Config { + &self.config + } + + pub fn is_biased(&self) -> bool { + self.params().is_biased() + } + /// Returns an immutable reference to the layer's parameters. + pub const fn params(&self) -> &LinearParams { + &self.params + } + /// Returns a mutable reference to the layer's parameters. + pub fn params_mut(&mut self) -> &mut LinearParams { + &mut self.params + } + + pub fn dim(&self) -> D::Pattern { + self.config().dim() + } + + pub fn eps(&self) -> f64 { + self.config().eps() + } + + pub fn ndim(&self) -> usize { + self.config().ndim() + } + + pub fn raw_dim(&self) -> D { + self.config().raw_dim() + } + + pub fn shape(&self) -> &[usize] { + self.config().shape() + } +} + +impl Default for LayerNorm +where + A: Default, + D: RemoveAxis, +{ + fn default() -> Self { + Self { + config: Config::default(), + params: Default::default(), + } + } +} + +impl Default for LayerNorm +where + A: Default, + D: RemoveAxis, +{ + fn default() -> Self { + Self { + config: Config::default(), + params: Default::default(), + } + } +} + +impl Forward> for LayerNorm +where + A: Float + FromPrimitive, + D: RemoveAxis, +{ + type Output = Array; + + fn forward(&self, x: &Array) -> Self::Output { + let norm = if let Some(axis) = self.config().axis() { + super::layer_norm_axis(x, *axis, self.eps()) + } else { + super::layer_norm(x, self.eps()) + }; + norm * self.params().weights() + self.params().bias() + } +} + +impl Forward> for LayerNorm +where + A: Float + FromPrimitive, + D: RemoveAxis, +{ + type Output = Array; + + fn forward(&self, x: &Array) -> Self::Output { + let norm = if let Some(axis) = self.config().axis() { + super::layer_norm_axis(x, *axis, self.eps()) + } else { + super::layer_norm(x, self.eps()) + }; + norm * self.params().weights() + } +} diff --git a/models/linear/src/norm/mod.rs b/models/linear/src/norm/mod.rs new file mode 100644 index 00000000..5fa9972d --- /dev/null +++ b/models/linear/src/norm/mod.rs @@ -0,0 +1,16 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +//! # Normalization +//! +//! +pub use self::layer::LayerNorm; + +pub mod batch; +pub mod layer; + +pub(crate) mod prelude { + pub use super::batch::prelude::*; + pub use super::layer::prelude::*; +} diff --git a/models/linear/src/params/entry.rs b/models/linear/src/params/item.rs similarity index 88% rename from models/linear/src/params/entry.rs rename to models/linear/src/params/item.rs index 2a193361..9ddbf28b 100644 --- a/models/linear/src/params/entry.rs +++ b/models/linear/src/params/item.rs @@ -34,7 +34,7 @@ use strum::{AsRefStr, EnumDiscriminants, EnumIs, VariantNames}; ), strum(serialize_all = "lowercase") )] -pub enum Entry +pub enum Parameter where S: RawData, D: RemoveAxis, @@ -43,16 +43,16 @@ where Weight(ArrayBase), } -impl Entry +impl Parameter where D: RemoveAxis, S: RawData, { - pub fn bias(data: ArrayBase) -> Self { + pub fn from_bias(data: ArrayBase) -> Self { Self::Bias(data) } - pub fn weight(data: ArrayBase) -> Self { + pub fn from_weight(data: ArrayBase) -> Self { Self::Weight(data) } } diff --git a/models/linear/src/params/mod.rs b/models/linear/src/params/mod.rs index b5d72583..571bdecd 100644 --- a/models/linear/src/params/mod.rs +++ b/models/linear/src/params/mod.rs @@ -3,53 +3,18 @@ Contrib: FL03 */ #[doc(inline)] -pub use self::entry::{Entry, Param}; -pub use self::mode::{Biased, ParamMode, Unbiased}; -pub use self::params::ParamsBase; +pub use self::item::{Param, Parameter}; +pub use self::mode::*; +pub use self::store::*; -mod params; +mod store; -pub mod entry; +pub mod item; pub mod mode; -use nd::{ArrayBase, Ix0, Ix1}; - -pub(crate) type Pair = (A, B); -pub(crate) type MaybePair = Pair>; -pub(crate) type NodeBase = MaybePair, ArrayBase>; -pub(crate) type Node = NodeBase, D, E>; - -macro_rules! params_ty { - ($($name:ident<$repr:ident>),* $(,)?) => { - $(params_ty!(@impl $name<$repr>);)* - }; - (@impl $name:ident<$repr:ident>) => { - pub type $name = $crate::params::ParamsBase, D, K>; - }; -} - -params_ty!(LinearParams, LinearParamsShared,); +#[doc(inline)] +pub use crate::primitives::params::*; pub(crate) mod prelude { - pub use super::{LinearParams, LinearParamsShared}; -} - -#[cfg(test)] -mod tests { - use super::*; - use core::str::FromStr; - - #[test] - fn test_param_kind() { - for i in [(Param::Bias, "bias"), (Param::Weight, "weight")].iter() { - let kind = Param::from_str(i.1).unwrap(); - assert_eq!(i.0, kind); - } - } - - #[test] - fn test_ones() { - let a = LinearParams::::ones((1, 300)); - assert!(a.is_biased()); - } + pub use super::mode::*; } diff --git a/models/linear/src/params/mode.rs b/models/linear/src/params/mode.rs index 53e16726..ff153868 100644 --- a/models/linear/src/params/mode.rs +++ b/models/linear/src/params/mode.rs @@ -2,21 +2,23 @@ Appellation: mode Contrib: FL03 */ +use concision::Toggle; +use core::option::Option; -pub trait State { - type Mode: ParamMode; -} - -pub trait ParamMode: 'static { +pub trait ParamMode: Toggle { const BIASED: bool = false; fn is_biased(&self) -> bool { - Self::BIASED + core::any::TypeId::of::() == core::any::TypeId::of::() } private!(); } +/* + ************* Implementations ************* +*/ + impl ParamMode for Option where T: 'static, @@ -30,15 +32,17 @@ where seal!(); } -macro_rules! param_mode { +macro_rules! mode { {$($T:ident: $opt:expr),* $(,)?} => { - $(param_mode!(@impl $T: $opt);)* + $(mode!(@impl $T: $opt);)* }; (@impl $T:ident: $opt:expr) => { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] pub enum $T {} + impl Toggle for $T {} + impl ParamMode for $T { const BIASED: bool = $opt; @@ -52,7 +56,7 @@ macro_rules! param_mode { } -param_mode! { +mode! { Biased: true, Unbiased: false, } diff --git a/models/linear/src/params/params.rs b/models/linear/src/params/params.rs deleted file mode 100644 index fab0a7b6..00000000 --- a/models/linear/src/params/params.rs +++ /dev/null @@ -1,195 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use super::mode::*; -use super::Node; -use crate::{build_bias, Features}; -use core::marker::PhantomData; -use nd::*; -use num::{One, Zero}; - -/// -pub struct ParamsBase, D = Ix2, K = Unbiased> -where - D: Dimension, - S: RawData, -{ - pub(crate) bias: Option>, - pub(crate) weights: ArrayBase, - pub(crate) _mode: PhantomData, -} - -impl ParamsBase -where - D: RemoveAxis, - K: ParamMode, - S: RawData, -{ - impl_param_builder!(default where A: Default, S: DataOwned); - impl_param_builder!(ones where A: Clone + One, S: DataOwned); - impl_param_builder!(zeros where A: Clone + Zero, S: DataOwned); - - #[doc(hidden)] - pub fn build(shape: Sh, builder: F) -> Self - where - F: Fn(Sh) -> ArrayBase, - Sh: ShapeBuilder, - { - let _weights = builder(shape); - unimplemented!() - } - - pub fn into_biased(self) -> ParamsBase - where - A: Default, - S: DataOwned, - { - let sm = crate::bias_dim(self.raw_dim()); - ParamsBase { - bias: Some(ArrayBase::default(sm)), - weights: self.weights, - _mode: PhantomData, - } - } - - pub fn into_unbiased(self) -> ParamsBase { - ParamsBase { - bias: None, - weights: self.weights, - _mode: PhantomData, - } - } - - pub fn bias(&self) -> Option<&ArrayBase> { - self.bias.as_ref() - } - - pub fn bias_mut(&mut self) -> Option<&mut ArrayBase> { - self.bias.as_mut() - } - - pub const fn weights(&self) -> &ArrayBase { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut ArrayBase { - &mut self.weights - } - - pub fn features(&self) -> Features { - Features::from_shape(self.shape()) - } - - pub fn in_features(&self) -> usize { - self.features().dmodel() - } - - pub fn is_biased(&self) -> bool { - self.bias().is_some() - } - - pub fn ndim(&self) -> usize { - self.weights().ndim() - } - - pub fn out_features(&self) -> usize { - if self.ndim() == 1 { - return 1; - } - self.shape()[1] - } - /// Returns the raw dimension of the weights. - pub fn raw_dim(&self) -> D { - self.weights().raw_dim() - } - /// Returns the shape of the weights. - pub fn shape(&self) -> &[usize] { - self.weights().shape() - } -} - -impl ParamsBase -where - D: RemoveAxis, - S: RawData, -{ - /// Create a new biased parameter store from the given shape. - pub fn biased(shape: Sh) -> Self - where - A: Default, - S: DataOwned, - Sh: ShapeBuilder, - { - let dim = shape.into_shape().raw_dim().clone(); - Self { - bias: build_bias(true, dim.clone(), ArrayBase::default), - weights: ArrayBase::default(dim), - _mode: PhantomData::, - } - } -} - -impl ParamsBase -where - D: Dimension, - S: RawData, -{ - /// Create a new unbiased parameter store from the given shape. - pub fn unbiased(shape: Sh) -> Self - where - A: Default, - S: DataOwned, - Sh: ShapeBuilder, - { - Self { - bias: None, - weights: ArrayBase::default(shape), - _mode: PhantomData::, - } - } -} -impl ParamsBase -where - K: ParamMode, - S: RawData, -{ - pub fn set_node(&mut self, idx: usize, node: Node) - where - A: Clone + Default, - S: DataMut + DataOwned, - { - let (weight, bias) = node; - if let Some(bias) = bias { - if !self.is_biased() { - let mut tmp = ArrayBase::default(self.out_features()); - tmp.index_axis_mut(Axis(0), idx).assign(&bias); - self.bias = Some(tmp); - } - self.bias - .as_mut() - .unwrap() - .index_axis_mut(Axis(0), idx) - .assign(&bias); - } - - self.weights_mut() - .index_axis_mut(Axis(0), idx) - .assign(&weight); - } -} - -impl Default for ParamsBase -where - A: Default, - D: Dimension, - S: DataOwned, -{ - fn default() -> Self { - Self { - bias: None, - weights: Default::default(), - _mode: PhantomData, - } - } -} diff --git a/models/linear/src/params/store.rs b/models/linear/src/params/store.rs new file mode 100644 index 00000000..b8afc752 --- /dev/null +++ b/models/linear/src/params/store.rs @@ -0,0 +1,240 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use crate::{build_bias, Biased, Features, Node, ParamMode, Unbiased}; +use concision::dimensional; +use core::marker::PhantomData; +use nd::*; +use num::{One, Zero}; + +/// The [ParamsBase] struct is a generic store for linear parameters. The store mimics +/// the underlying [ArrayBase](ndarray::ArrayBase), enabling developers to specify +/// the data repr and dimension. Additionally, the store is parameterized to +/// accept a `K` type, used to designate the store as either [Biased](crate::Biased) or [Unbiased](crate::Unbiased). +pub struct ParamsBase, D = Ix2, K = Biased> +where + D: Dimension, + S: RawData, +{ + pub(crate) bias: Option>, + pub(crate) weight: ArrayBase, + pub(crate) _mode: PhantomData, +} + +impl ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + pub fn from_elem(shape: Sh, elem: A) -> Self + where + A: Clone, + K: ParamMode, + S: DataOwned, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::from_elem( + crate::bias_dim(dim.clone()), + elem.clone(), + )) + } else { + None + }; + Self { + bias, + weight: ArrayBase::from_elem(dim, elem), + _mode: PhantomData::, + } + } + + pub fn into_biased(self) -> ParamsBase + where + A: Default, + K: 'static, + S: DataOwned, + { + if self.is_biased() { + return ParamsBase { + bias: self.bias, + weight: self.weight, + _mode: PhantomData::, + }; + } + let sm = crate::bias_dim(self.raw_dim()); + ParamsBase { + bias: Some(ArrayBase::default(sm)), + weight: self.weight, + _mode: PhantomData::, + } + } + + pub fn into_unbiased(self) -> ParamsBase { + ParamsBase { + bias: None, + weight: self.weight, + _mode: PhantomData::, + } + } + + pub const fn weights(&self) -> &ArrayBase { + &self.weight + } + + pub fn weights_mut(&mut self) -> &mut ArrayBase { + &mut self.weight + } + + pub fn features(&self) -> Features { + Features::from_shape(self.shape()) + } + + pub fn in_features(&self) -> usize { + self.features().dmodel() + } + + pub fn out_features(&self) -> usize { + if self.ndim() == 1 { + return 1; + } + self.shape()[1] + } + /// Returns true if the parameter store is biased; + /// Compares the [TypeId](core::any::TypeId) of the store with the [Biased](crate::Biased) type. + pub fn is_biased(&self) -> bool + where + K: 'static, + { + crate::is_biased::() + } + + pbuilder!(new.default where A: Default, S: DataOwned); + + pbuilder!(ones where A: Clone + One, S: DataOwned); + + pbuilder!(zeros where A: Clone + Zero, S: DataOwned); + + dimensional!(weights()); + + wnbview!(into_owned::(self) where A: Clone, S: Data); + + wnbview!(into_shared::(self) where A: Clone, S: DataOwned); + + wnbview!(to_owned::(&self) where A: Clone, S: Data); + + wnbview!(to_shared::(&self) where A: Clone, S: DataOwned); + + wnbview!(view::<'a, ViewRepr>(&self) where A: Clone, S: Data); + + wnbview!(view_mut::<'a, ViewRepr>(&mut self) where A: Clone, S: DataMut); +} + +impl ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + /// Create a new biased parameter store from the given shape. + pub fn biased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + bias: build_bias(true, dim.clone(), ArrayBase::default), + weight: ArrayBase::default(dim), + _mode: PhantomData::, + } + } + /// Return an unwraped, immutable reference to the bias array. + pub fn bias(&self) -> &ArrayBase { + self.bias.as_ref().unwrap() + } + /// Return an unwraped, mutable reference to the bias array. + pub fn bias_mut(&mut self) -> &mut ArrayBase { + self.bias.as_mut().unwrap() + } +} + +impl ParamsBase +where + D: Dimension, + S: RawData, +{ + /// Create a new unbiased parameter store from the given shape. + pub fn unbiased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + Self { + bias: None, + weight: ArrayBase::default(shape), + _mode: PhantomData::, + } + } +} +impl ParamsBase +where + K: 'static, + S: RawData, +{ + pub fn set_node(&mut self, idx: usize, node: Node) + where + A: Clone + Default, + S: DataMut + DataOwned, + { + let (weight, bias) = node; + if let Some(bias) = bias { + if !self.is_biased() { + let mut tmp = ArrayBase::default(self.out_features()); + tmp.index_axis_mut(Axis(0), idx).assign(&bias); + self.bias = Some(tmp); + } + self.bias + .as_mut() + .unwrap() + .index_axis_mut(Axis(0), idx) + .assign(&bias); + } + + self.weights_mut() + .index_axis_mut(Axis(0), idx) + .assign(&weight); + } +} + +impl Default for ParamsBase +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + bias: Some(Default::default()), + weight: Default::default(), + _mode: PhantomData::, + } + } +} + +impl Default for ParamsBase +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + bias: None, + weight: Default::default(), + _mode: PhantomData::, + } + } +} diff --git a/models/linear/src/primitives.rs b/models/linear/src/primitives.rs new file mode 100644 index 00000000..6cda38fd --- /dev/null +++ b/models/linear/src/primitives.rs @@ -0,0 +1,28 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::params::*; +use nd::{ArrayBase, Ix0, Ix1}; + +pub(crate) type Pair = (A, B); + +pub(crate) type MaybePair = Pair>; + +pub(crate) type NodeBase = MaybePair, ArrayBase>; + +pub(crate) type Node = NodeBase, D, E>; + +pub(crate) mod params { + + macro_rules! params_ty { + ($($name:ident<$repr:ident>),* $(,)?) => { + $(params_ty!(@impl $name<$repr>);)* + }; + (@impl $name:ident<$repr:ident>) => { + pub type $name = $crate::params::ParamsBase, D, K>; + }; + } + + params_ty!(LinearParams, LinearParamsShared,); +} diff --git a/models/linear/src/traits.rs b/models/linear/src/traits.rs index a1b7aef1..5d609a02 100644 --- a/models/linear/src/traits.rs +++ b/models/linear/src/traits.rs @@ -2,6 +2,17 @@ Appellation: traits Contrib: FL03 */ +use crate::Biased; + pub trait IsBiased { fn is_biased(&self) -> bool; } + +impl IsBiased for T +where + T: 'static, +{ + fn is_biased(&self) -> bool { + core::any::TypeId::of::() == core::any::TypeId::of::() + } +} diff --git a/models/linear/src/utils.rs b/models/linear/src/utils.rs index abd8fb77..ca6142dc 100644 --- a/models/linear/src/utils.rs +++ b/models/linear/src/utils.rs @@ -2,7 +2,8 @@ Appellation: utils Contrib: FL03 */ -use nd::*; +use crate::params::Biased; +use nd::{ArrayBase, Axis, Dimension, RawData, RemoveAxis}; /// A utilitarian funciton for building bias tensors. pub(crate) fn build_bias(biased: bool, dim: D, builder: F) -> Option> @@ -30,3 +31,10 @@ where dim.remove_axis(Axis(1)) } } + +/// A utilitarian function for checking if a type is [Biased]; returns false otherwise. +/// Compares the [TypeId](core::any::TypeId) of `K` to the [TypeId](core::any::TypeId) of [Biased]. +pub fn is_biased() -> bool { + use core::any::TypeId; + TypeId::of::() == TypeId::of::() +} diff --git a/models/linear/tests/model.rs b/models/linear/tests/linear.rs similarity index 78% rename from models/linear/tests/model.rs rename to models/linear/tests/linear.rs index 70c6f8b4..a941510d 100644 --- a/models/linear/tests/model.rs +++ b/models/linear/tests/linear.rs @@ -23,13 +23,28 @@ lazy_static! { #[test] fn test_config() { let dim = FEATURES.clone().into_dimension(); - let config = Config::from_dim_biased(dim); + let config = Config::::from_shape(dim); assert!(config.is_biased()); - let config = Config::from_dim(dim); + let config = Config::::from_shape(dim); assert!(!config.is_biased()); } #[test] +fn test_model_toggle() { + let (_samples, (outputs, inputs)) = SHAPE; + + let model = Linear::::from_features(inputs, outputs); + assert!(model.is_biased()); + + let model = Linear::::from_features(inputs, outputs); + assert!(!model.is_biased()); + + let model = Linear::::from_features(inputs, outputs).into_unbiased(); + assert!(!model.is_biased()); +} + +#[test] +#[cfg(feature = "rand")] fn test_linear() { let (samples, (outputs, inputs)) = SHAPE; @@ -40,14 +55,3 @@ fn test_linear() { assert_eq!(y.shape(), &[samples, outputs]); } - -#[test] -fn test_model_modes() { - let (_samples, (outputs, inputs)) = SHAPE; - - let model = Linear::::from_features(inputs, outputs).uniform(); - assert!(model.is_biased()); - - let model = Linear::::from_features(inputs, outputs).uniform(); - assert!(!model.is_biased()); -} diff --git a/models/linear/tests/norm.rs b/models/linear/tests/norm.rs new file mode 100644 index 00000000..119b67d4 --- /dev/null +++ b/models/linear/tests/norm.rs @@ -0,0 +1,35 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_linear as linear; + +use concision::{linarr, Forward}; +use linear::{Biased, LayerNorm}; + +use approx::assert_abs_diff_eq; +use lazy_static::lazy_static; +use ndarray::prelude::*; + +const SHAPE: (usize, usize) = (3, 3); + +lazy_static! { + static ref NORM: Array2 = array![ + [-0.5492, -0.1619, 0.2254], + [0.6127, 1.0000, 1.3873], + [1.7746, 2.1619, 2.5492], + ]; +} + +#[test] +fn test_layer_norm() { + let shape = SHAPE; + let x = linarr::(shape).unwrap(); + + let ln = LayerNorm::::ones(shape); + let y = ln.forward(&x); + + assert_eq!(y.dim(), shape); + assert_abs_diff_eq!(y, *NORM, epsilon = 1e-4); +} diff --git a/models/linear/tests/params.rs b/models/linear/tests/params.rs index acfaa093..fed65fb7 100644 --- a/models/linear/tests/params.rs +++ b/models/linear/tests/params.rs @@ -2,22 +2,46 @@ Appellation: params Contrib: FL03 */ +#![allow(unused_imports)] extern crate concision_core as concision; extern crate concision_linear as linear; use concision::Predict; -use linear::{Features, LinearParams}; +use core::str::FromStr; +use linear::params::{LinearParams, Param, Unbiased}; +use linear::Features; use ndarray::prelude::*; const SAMPLES: usize = 20; -const INPUTS: usize = 5; -const DMODEL: usize = 3; +const D_MODEL: usize = 5; +const FEATURES: usize = 3; #[test] +fn test_keys() { + for i in [(Param::Bias, "bias"), (Param::Weight, "weight")].iter() { + let kind = Param::from_str(i.1).unwrap(); + assert_eq!(i.0, kind); + } +} + +#[test] +fn test_builders() { + let shape = (D_MODEL, FEATURES); + let params = LinearParams::::ones(shape); + assert!(params.is_biased()); + assert_eq!(params.weights(), &Array2::ones(shape)); + assert_eq!(params.bias(), &Array1::ones(D_MODEL)); + let params = LinearParams::::zeros(shape); + assert!(!params.is_biased()); + assert_eq!(params.weights(), &Array2::zeros(shape)); +} + +#[test] +#[cfg(feature = "rand")] fn test_linear_params() { - let (samples, inputs, outputs) = (SAMPLES, INPUTS, DMODEL); + let (samples, inputs, outputs) = (SAMPLES, D_MODEL, FEATURES); let features = Features::new(outputs, inputs); - let data = Array2::::zeros((samples, inputs)); + let data = Array2::::ones((samples, inputs)); let params = LinearParams::biased(features).uniform(); let y: Array2 = params.predict(&data).unwrap(); assert_eq!(y.dim(), (samples, outputs)); diff --git a/models/transformers/Cargo.toml b/models/transformers/Cargo.toml index 34782d1c..00bf9fb7 100644 --- a/models/transformers/Cargo.toml +++ b/models/transformers/Cargo.toml @@ -28,27 +28,33 @@ full = [ alloc = [ "concision-core/alloc", + "concision-linear/alloc", + "serde?/alloc", ] approx = [ "dep:approx", "concision-core/approx", + "concision-linear/approx", "ndarray/approx-0_5", ] blas = [ "concision-core/blas", + "concision-linear/blas", "ndarray/blas", ] rand = [ "concision-core/rand", + "concision-linear/rand", "num/rand" ] serde = [ "serde-1", "concision-core/serde", + "concision-linear/serde", "ndarray/serde-1", "num/serde" ] @@ -59,11 +65,14 @@ serde-1 = [ tracing = [ "dep:tracing", + "concision-core/tracing", + "concision-linear/tracing", ] # ********* [FF] Environments ********* std = [ "concision-core/std", + "concision-linear/std", "ndarray/std", "num/std", "serde?/std", @@ -72,10 +81,12 @@ std = [ wasm = [ "concision-core/wasm", + "concision-linear/wasm", ] wasi = [ "concision-core/wasi", + "concision-linear/wasi", ] [lib] @@ -102,6 +113,11 @@ default-features = false path = "../../core" version = "0.1.14" +[dependencies.concision-linear] +default-features = false +path = "../linear" +version = "0.1.14" + [dependencies.serde] default-features = false features = ["derive"] @@ -112,8 +128,8 @@ version = "1" optional = true version = "0.1" -[dev-dependencies] -lazy_static.workspace = true +[dev-dependencies.lazy_static] +workspace = true [package.metadata.docs.rs] all-features = true diff --git a/models/transformers/src/attention/head.rs b/models/transformers/src/attention/head.rs index c85523bb..c5146a34 100644 --- a/models/transformers/src/attention/head.rs +++ b/models/transformers/src/attention/head.rs @@ -2,24 +2,30 @@ Appellation: head Contrib: FL03 */ -use crate::params::QKVBase; +use crate::params::QkvBase; +use concision::getters; +use core::borrow::{Borrow, BorrowMut}; +use nd::linalg::Dot; use nd::*; +use num::complex::ComplexFloat; -pub struct AttentionHead, D = Ix2> +// #68 +pub struct AttentionHead> where D: Dimension, S: RawData, { - params: QKVBase, + pub(crate) mask: Option>, + pub(crate) params: QkvBase, } -impl AttentionHead +impl AttentionHead where D: Dimension, S: RawData, { - pub fn from_params(params: QKVBase) -> Self { - Self { params } + pub fn from_params(params: QkvBase) -> Self { + Self { mask: None, params } } pub fn builder(shape: Sh, builder: F) -> Self @@ -27,19 +33,67 @@ where F: Fn(D) -> ArrayBase, Sh: ShapeBuilder, { - Self::from_params(QKVBase::builder(shape, builder)) + Self::from_params(QkvBase::builder(shape, builder)) } - pub fn params(&self) -> &QKVBase { - &self.params + pub fn from_elem(shape: Sh, value: A) -> Self + where + Sh: ShapeBuilder, + A: Clone, + S: DataOwned, + { + Self::from_params(QkvBase::from_elem(shape, value)) } - pub fn params_mut(&mut self) -> &mut QKVBase { + pub fn attention(&self) -> Array + where + A: ComplexFloat + ScalarOperand, + S: Data, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + let (q, k, v) = self.qkv(); + crate::attention::scaled_dot_product_attention(q, k, v) + } + /// Returns an immuable reference to the underlying parameters. + pub const fn params(&self) -> &QkvBase { + &self.params + } + /// Returns a mutable reference to the underlying parameters. + pub fn params_mut(&mut self) -> &mut QkvBase { &mut self.params } + /// Returns a three-tuple consisting of immputable references to the query, key, and value matrices respectively. + pub fn qkv(&self) -> (&ArrayBase, &ArrayBase, &ArrayBase) { + self.params().qkv() + } + /// Consumes the head, returning a three-tuple consisting of mutable references to the query, key, and value matrices respectively. + pub fn into_qkv(self) -> (ArrayBase, ArrayBase, ArrayBase) { + self.params.into_qkv() + } + + getters!(params::<[q, k, v]> => ArrayBase); + ndbuilder!(new::default() where A: Default, S: DataOwned); + ndbuilder!(ones() where A: Clone + num::One, S: DataOwned); + ndbuilder!(zeros() where A: Clone + num::Zero, S: DataOwned); +} - access!(params::); - fwd_builder!(new.default where A: Default, S: DataOwned); - fwd_builder!(ones.ones where A: Clone + num::One, S: DataOwned); - fwd_builder!(zeros.zeros where A: Clone + num::Zero, S: DataOwned); +impl Borrow> for AttentionHead +where + D: Dimension, + S: RawData, +{ + fn borrow(&self) -> &QkvBase { + self.params() + } +} + +impl BorrowMut> for AttentionHead +where + D: Dimension, + S: RawData, +{ + fn borrow_mut(&mut self) -> &mut QkvBase { + self.params_mut() + } } diff --git a/models/transformers/src/attention/mod.rs b/models/transformers/src/attention/mod.rs index f449f39e..80a264c7 100644 --- a/models/transformers/src/attention/mod.rs +++ b/models/transformers/src/attention/mod.rs @@ -2,12 +2,53 @@ Appellation: attention Contrib: FL03 */ +//! # Attention +//! +//! Attention allows a model to focus on specific parts of the input sequence. +//! Today, these mechanisms are found in several state-of-the-art models, such as +//! the Transformer model, primarily due to its capabilities in natural language +//! processing (NLP) domains pub use self::head::AttentionHead; +pub use self::utils::*; pub(crate) mod head; +// #69: Multi-Head Attention implementation pub mod multi; pub(crate) mod prelude { pub use super::head::AttentionHead; + pub use super::utils::*; +} + +pub(crate) mod utils { + use concision::func::activate::Softmax; + use nd::linalg::Dot; + use nd::prelude::{Array, ArrayBase, ArrayView, Axis, Dimension}; + use nd::{Data, ScalarOperand}; + use num::complex::ComplexFloat; + + pub(crate) fn scale(dk: usize) -> A + where + A: ComplexFloat, + { + A::from(dk).unwrap().sqrt().recip() + } + + /// A functional implementation of the scaled dot-product attention mechanism; + pub fn scaled_dot_product_attention( + q: &ArrayBase, + k: &ArrayBase, + v: &ArrayBase, + ) -> Array + where + A: ComplexFloat + ScalarOperand, + S: Data, + D: Dimension, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + let dk = scale::(k.len_of(Axis(1))); + (q.dot(&k.t()) * dk).softmax().dot(&v) + } } diff --git a/models/transformers/src/codec/decoder.rs b/models/transformers/src/codec/decoder.rs new file mode 100644 index 00000000..3d00af1c --- /dev/null +++ b/models/transformers/src/codec/decoder.rs @@ -0,0 +1,31 @@ +/* + Appellation: decoder + Contrib: FL03 +*/ +pub use self::{config::DecoderConfig, layer::DecoderLayer}; + +pub mod config; +pub mod layer; + +#[derive(Default)] +pub struct Decoder { + config: DecoderConfig, + layers: Vec, +} + +impl Decoder { + pub fn new() -> Self { + Self { + config: DecoderConfig::default(), + layers: Vec::new(), + } + } + + pub const fn config(&self) -> &DecoderConfig { + &self.config + } + + pub fn layers(&self) -> &[DecoderLayer] { + &self.layers + } +} diff --git a/models/transformers/src/codec/decoder/config.rs b/models/transformers/src/codec/decoder/config.rs new file mode 100644 index 00000000..8056b7cd --- /dev/null +++ b/models/transformers/src/codec/decoder/config.rs @@ -0,0 +1,14 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub struct DecoderConfig { + pub layers: usize, +} + +impl Default for DecoderConfig { + fn default() -> Self { + Self { layers: crate::N } + } +} diff --git a/models/transformers/src/codec/decoder/layer.rs b/models/transformers/src/codec/decoder/layer.rs new file mode 100644 index 00000000..90514acc --- /dev/null +++ b/models/transformers/src/codec/decoder/layer.rs @@ -0,0 +1,13 @@ +/* + Appellation: layer + Contrib: FL03 +*/ + +#[derive(Default)] +pub struct DecoderLayer {} + +impl DecoderLayer { + pub fn new() -> Self { + Self {} + } +} diff --git a/models/transformers/src/codec/encoder.rs b/models/transformers/src/codec/encoder.rs new file mode 100644 index 00000000..e9fdb490 --- /dev/null +++ b/models/transformers/src/codec/encoder.rs @@ -0,0 +1,39 @@ +/* + Appellation: encoder + Contrib: FL03 +*/ +pub use self::{config::EncoderConfig, layer::EncoderLayer}; + +pub mod config; +pub mod layer; + +use linear::norm::LayerNorm; + +#[derive(Default)] +pub struct Encoder { + config: EncoderConfig, + layers: Vec, + norm: LayerNorm, +} + +impl Encoder { + pub fn new() -> Self { + Self { + config: EncoderConfig::default(), + layers: Vec::new(), + norm: LayerNorm::default(), + } + } + + pub const fn config(&self) -> &EncoderConfig { + &self.config + } + + pub fn layers(&self) -> &[EncoderLayer] { + &self.layers + } + + pub fn norm(&self) -> &LayerNorm { + &self.norm + } +} diff --git a/models/transformers/src/codec/encoder/config.rs b/models/transformers/src/codec/encoder/config.rs new file mode 100644 index 00000000..2c5dcf93 --- /dev/null +++ b/models/transformers/src/codec/encoder/config.rs @@ -0,0 +1,14 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub struct EncoderConfig { + pub layers: usize, +} + +impl Default for EncoderConfig { + fn default() -> Self { + Self { layers: crate::N } + } +} diff --git a/models/transformers/src/codec/encoder/layer.rs b/models/transformers/src/codec/encoder/layer.rs new file mode 100644 index 00000000..10821bd3 --- /dev/null +++ b/models/transformers/src/codec/encoder/layer.rs @@ -0,0 +1,13 @@ +/* + Appellation: layer + Contrib: FL03 +*/ + +#[derive(Default)] +pub struct EncoderLayer {} + +impl EncoderLayer { + pub fn new() -> Self { + Self {} + } +} diff --git a/models/transformers/src/codec/mod.rs b/models/transformers/src/codec/mod.rs new file mode 100644 index 00000000..3a7e3f77 --- /dev/null +++ b/models/transformers/src/codec/mod.rs @@ -0,0 +1,25 @@ +/* + Appellation: codec + Contrib: FL03 +*/ +pub use self::{decoder::Decoder, encoder::Encoder, model::*}; + +pub(crate) mod model; + +pub mod decoder; +pub mod encoder; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_codec_builder() { + let ctx = Context::new() + .with_src("src".to_string()) + .with_tgt("tgt".to_string()); + let codec = Codec::new().ctx(ctx).build(); + assert_eq!(codec.context().src, "src"); + assert_eq!(codec.context().tgt, "tgt"); + } +} diff --git a/models/transformers/src/codec/model.rs b/models/transformers/src/codec/model.rs new file mode 100644 index 00000000..494c0a0e --- /dev/null +++ b/models/transformers/src/codec/model.rs @@ -0,0 +1,60 @@ +/* + Appellation: codec + Contrib: FL03 +*/ +use super::{Decoder, Encoder}; +use concision::{builder, getters}; + +#[derive(Default)] +pub struct Codec { + ctx: Context, + decoder: Decoder, + encoder: Encoder, +} + +impl Codec { + pub fn new() -> CodecBuilder { + CodecBuilder::new() + } + + getters!( + context.ctx, + decoder, + encoder, + ); +} + +builder!(CodecBuilder:: { + ctx: Context, + decoder: Decoder, + encoder: Encoder, +}); + +#[derive(Default)] +pub struct Generator { + pub dmodel: usize, + pub vocab: Vec, +} + +#[derive(Default)] +pub struct Context { + pub src: String, // source embedding + pub tgt: String, // target embedding +} + +impl Context { + pub fn new() -> Self { + Self { + src: String::new(), + tgt: String::new(), + } + } + + pub fn with_src(self, src: String) -> Self { + Self { src, ..self } + } + + pub fn with_tgt(self, tgt: String) -> Self { + Self { tgt, ..self } + } +} diff --git a/models/transformers/src/impls/impl_head.rs b/models/transformers/src/impls/impl_head.rs index ee5eaf10..fa22f80a 100644 --- a/models/transformers/src/impls/impl_head.rs +++ b/models/transformers/src/impls/impl_head.rs @@ -3,17 +3,39 @@ Contrib: FL03 */ use crate::attention::AttentionHead; -use crate::params::QKVBase; +use crate::params::QkvBase; use nd::prelude::*; -use nd::DataOwned; +use nd::{DataOwned, RawDataClone}; -impl Default for AttentionHead +impl Clone for AttentionHead +where + A: Copy, + D: Dimension, + S: RawDataClone, +{ + fn clone(&self) -> Self { + Self { + mask: self.mask.clone(), + params: self.params.clone(), + } + } +} + +impl Copy for AttentionHead +where + A: Copy, + D: Copy + Dimension, + S: Copy + RawDataClone, +{ +} + +impl Default for AttentionHead where A: Default, D: Dimension, S: DataOwned, { fn default() -> Self { - Self::from_params(QKVBase::default()) + Self::from_params(QkvBase::default()) } } diff --git a/models/transformers/src/impls/impl_linalg.rs b/models/transformers/src/impls/impl_linalg.rs index 4440ec34..ce069afe 100644 --- a/models/transformers/src/impls/impl_linalg.rs +++ b/models/transformers/src/impls/impl_linalg.rs @@ -2,12 +2,12 @@ Appellation: impl_linalg Contrib: FL03 */ -use crate::params::{QKVBase, QKV}; +use crate::params::{Params, QkvBase}; use concision::Matmul; use nd::linalg::Dot; use nd::*; -impl Matmul> for QKVBase +impl Matmul> for QkvBase where A: LinalgScalar, D: Dimension, @@ -17,10 +17,10 @@ where T: Data, ArrayBase: Dot, Output = Array>, { - type Output = QKV; + type Output = Params; - fn matmul(&self, rhs: &QKVBase) -> Self::Output { - QKVBase { + fn matmul(&self, rhs: &QkvBase) -> Self::Output { + QkvBase { q: self.q().dot(rhs.q()), k: self.k().dot(rhs.k()), v: self.v().dot(rhs.v()), @@ -28,7 +28,7 @@ where } } -impl Matmul> for QKVBase +impl Matmul> for QkvBase where A: LinalgScalar, D: Dimension, @@ -38,10 +38,10 @@ where T: Data, ArrayBase: Dot, Output = Array>, { - type Output = QKV; + type Output = Params; fn matmul(&self, rhs: &ArrayBase) -> Self::Output { - QKVBase { + QkvBase { q: self.q().dot(rhs), k: self.k().dot(rhs), v: self.v().dot(rhs), diff --git a/models/transformers/src/impls/impl_params.rs b/models/transformers/src/impls/impl_params.rs index ad79e30d..9736c1b0 100644 --- a/models/transformers/src/impls/impl_params.rs +++ b/models/transformers/src/impls/impl_params.rs @@ -2,11 +2,11 @@ Appellation: impl_params Contrib: FL03 */ -use crate::params::QKVBase; +use crate::params::QkvBase; use nd::prelude::*; use nd::{Data, DataOwned, RawDataClone}; -impl Clone for QKVBase +impl Clone for QkvBase where D: Dimension, S: RawDataClone, @@ -20,14 +20,14 @@ where } } -impl Copy for QKVBase +impl Copy for QkvBase where D: Copy + Dimension, S: Copy + RawDataClone, { } -impl Default for QKVBase +impl Default for QkvBase where D: Dimension, S: DataOwned, @@ -42,7 +42,7 @@ where } } -impl PartialEq for QKVBase +impl PartialEq for QkvBase where A: PartialEq, D: Dimension, @@ -53,7 +53,7 @@ where } } -impl PartialEq> for QKVBase +impl PartialEq> for QkvBase where A: PartialEq, B: PartialEq, diff --git a/models/transformers/src/lib.rs b/models/transformers/src/lib.rs index e39da023..ed9cf63e 100644 --- a/models/transformers/src/lib.rs +++ b/models/transformers/src/lib.rs @@ -14,17 +14,22 @@ extern crate alloc; extern crate concision_core as concision; +extern crate concision_linear as linear; extern crate ndarray as nd; pub use self::attention::AttentionHead; -pub use self::params::QKV; +pub use self::params::*; +pub use self::primitives::*; pub use self::transformer::Transformer; #[macro_use] pub(crate) mod macros; +pub(crate) mod primitives; pub(crate) mod transformer; pub mod attention; +pub mod codec; +pub mod ops; pub mod params; pub(crate) mod impls { @@ -35,6 +40,6 @@ pub(crate) mod impls { pub mod prelude { pub use super::attention::prelude::*; - pub use super::params::prelude::*; + pub use super::primitives::*; pub use super::Transformer; } diff --git a/models/transformers/src/macros.rs b/models/transformers/src/macros.rs index d7b352d0..fd05142e 100644 --- a/models/transformers/src/macros.rs +++ b/models/transformers/src/macros.rs @@ -3,96 +3,81 @@ Contrib: FL03 */ -macro_rules! access { - ($($var:ident),* $(,)?) => { - $(access!(@impl $var);)* +macro_rules! ndbuilder { + ($method:ident$(::$call:ident)?() where $($rest:tt)*) => { + ndbuilder!(@impl $method$(::$call)?() where $($rest)*); }; - ($via:ident::<$($var:ident),* $(,)?>) => { - $(access!(@impl $via::$var);)* + (@impl $method:ident() where $($rest:tt)*) => { + ndbuilder!(@impl $method::$method() where $($rest)*); }; - (@impl $var:ident) => { - pub fn $var(&self) -> &ArrayBase { - &self.$var - } - paste::paste! { - pub fn [< $var _mut>](&mut self) -> &mut ArrayBase { - &mut self.$var - } - } - }; - (@impl $via:ident::$var:ident) => { - pub fn $var(&self) -> &ArrayBase { - &self.$via.$var - } - paste::paste! { - pub fn [< $var _mut>](&mut self) -> &mut ArrayBase { - &mut self.$via.$var - } + (@impl $method:ident::$call:ident() where $($rest:tt)*) => { + pub fn $method>(shape: Sh) -> Self where $($rest)* { + Self::builder(shape, ndarray::ArrayBase::$call) } }; } -macro_rules! fwd_builder { - ($method:ident.$call:ident where $($rest:tt)*) => { - pub fn $method(shape: Sh) -> Self - where - Sh: ndarray::ShapeBuilder, - $($rest)* - { - Self::builder(shape, ArrayBase::$call) - } - }; -} - -macro_rules! param_views { +// # TODO: +macro_rules! ndview { ($method:ident::$($rest:tt)*) => { - param_views!(@impl $method.$method::$($rest)*); + ndview!(@impl $method.$method::$($rest)*); }; ($method:ident.$call:ident::$($rest:tt)*) => { - param_views!(@impl $method.$call::$($rest)*); + ndview!(@impl $method.$call::$($rest)*); }; (@impl $method:ident.$call:ident::<$view:ident>(self) where $($rest:tt)*) => { - pub fn $method(self) -> QKVBase<$view, D> + pub fn $method(self) -> $crate::params::QkvBase<$view, D> + where + $($rest)* + { + ndview!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(mut self) where $($rest:tt)*) => { + pub fn $method(mut self) -> $crate::params::QkvBase<$view, D> where $($rest)* { - param_views!(@apply $call(self)) + ndview!(@apply $call(self)) } }; (@impl $method:ident.$call:ident::<$view:ident>(&self) where $($rest:tt)*) => { - pub fn $method(&self) -> QKVBase<$view, D> + pub fn $method(&self) -> $crate::params::QkvBase<$view, D> where $($rest)* { - param_views!(@apply $call(self)) + ndview!(@apply $call(self)) } }; - (@impl $method:ident.$call:ident::<'a, $view:ident>(&self) where $($rest:tt)*) => { - pub fn $method(&self) -> QKVBase<$view<&'_ A>, D> + (@impl $method:ident.$call:ident::<$view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::QkvBase<$view, D> where $($rest)* { - param_views!(@apply $call(self)) + ndview!(@apply $call(self)) } }; - (@apply $call:ident($self:expr)) => { - $crate::params::QKVBase { - q: $self.q.$call(), - k: $self.k.$call(), - v: $self.v.$call(), + (@impl $method:ident.$call:ident::<'a, $view:ident>(&self) where $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::QkvBase<$view<&'_ A>, D> + where + $($rest)* + { + ndview!(@apply $call(self)) } }; -} - -macro_rules! qkv_builder { - - ($method:ident.$call:ident where $($rest:tt)*) => { - pub fn $method(shape: Sh) -> Self + (@impl $method:ident.$call:ident::<'a, $view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::QkvBase<$view<&'_ mut A>, D> where - Sh: ndarray::ShapeBuilder, $($rest)* { - Self::builder(shape, ArrayBase::$call) + ndview!(@apply $call(self)) + } + }; + (@apply $call:ident($self:expr)) => { + $crate::params::QkvBase { + q: $self.q.$call(), + k: $self.k.$call(), + v: $self.v.$call(), } }; } diff --git a/models/transformers/src/ops/merge.rs b/models/transformers/src/ops/merge.rs new file mode 100644 index 00000000..747cae82 --- /dev/null +++ b/models/transformers/src/ops/merge.rs @@ -0,0 +1,42 @@ +/* + Appellation: merge + Contrib: FL03 +*/ +use concision::NdResult; +use nd::prelude::*; +use nd::{Data, RemoveAxis}; + +// #67: Optimize the Merge trait +pub trait Merge { + type Output; + + fn merge(&self) -> NdResult { + self.merge_along(0) + } + + fn merge_along(&self, axis: usize) -> NdResult; +} + +/* + ************* Implementations ************* +*/ +impl Merge for ArrayBase +where + A: Clone, + D: RemoveAxis, + E: Dimension, + S: Data, + ArrayBase: Clone, +{ + type Output = Array; + + fn merge(&self) -> NdResult { + let swap = if self.ndim() >= 3 { self.ndim() - 3 } else { 0 }; + self.merge_along(swap) + } + + fn merge_along(&self, swap: usize) -> NdResult { + use ndarray::Order; + super::merger(self, swap, swap + 1, Order::RowMajor) + } +} diff --git a/models/transformers/src/ops/mod.rs b/models/transformers/src/ops/mod.rs new file mode 100644 index 00000000..778e4abc --- /dev/null +++ b/models/transformers/src/ops/mod.rs @@ -0,0 +1,107 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +pub use self::{merge::*, split::*, utils::*}; + +pub(crate) mod merge; +pub(crate) mod split; + +pub(crate) mod utils { + use concision::NdResult; + use nd::prelude::*; + use nd::{Data, Order, RemoveAxis}; + + #[doc(hidden)] + pub fn merge( + arr: &ArrayBase, + src: usize, + tgt: usize, + ) -> NdResult> + where + A: Clone, + D: RemoveAxis, + S: Data, + D::Smaller: Dimension, + ArrayBase: Clone, + { + merger(arr, src, tgt, Order::RowMajor) + } + + pub(crate) fn merger( + arr: &ArrayBase, + src: usize, + tgt: usize, + order: Order, + ) -> NdResult> + where + A: Clone, + D: RemoveAxis, + S: Data, + D::Smaller: Dimension, + ArrayBase: Clone, + { + let shape = merge_dims(arr.raw_dim(), src); + let mut head = arr.clone(); + head.swap_axes(src, tgt); + head.to_shape((shape, order)).map(|x| x.to_owned()) + } + + #[doc(hidden)] + pub fn merge_dims(dim: D, src: usize) -> D::Smaller + where + D: RemoveAxis, + D::Smaller: Dimension, + { + // create a new dimension with one less axis; initialized with zeros + let mut new_dim = ::Smaller::zeros(dim.ndim() - 1); + // create a mutable vector from the slice + let mut shape = dim.slice().to_vec(); + // multiply the last axis by the target + shape[new_dim.ndim()] *= shape[src]; + // remove the last dimension + shape.remove(src); + + new_dim.slice_mut().copy_from_slice(&shape); + new_dim + } + + #[doc(hidden)] + pub fn merge_batch(heads: &Array4) -> NdResult> + where + T: Clone, + { + let (batch, n, seq, query) = heads.dim(); + let mut tmp = heads.clone(); + // swap the head and sequence axes + tmp.swap_axes(1, 2); + // reshape the qkv matrix into a 2d array + tmp.into_shape((batch, seq, n * query)) + } + + pub fn split_heads(param: &Array2, h: usize) -> NdResult> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / h; + // reshape the qkv matrix into a 3d array + let mut res = param.clone().into_shape((param.shape()[0], h, dim))?; + // swap the sequence and head axes + res.swap_axes(0, 1); + Ok(res) + } + + pub fn split_batch(param: &Array3, h: usize) -> NdResult> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / h; + // reshape the qkv matrix into a 3d array + let mut res = param + .clone() + .into_shape((param.shape()[0], param.shape()[1], h, dim))?; + // swap the sequence and head axes + res.swap_axes(1, 2); + Ok(res) + } +} diff --git a/models/transformers/src/ops/split.rs b/models/transformers/src/ops/split.rs new file mode 100644 index 00000000..3a182710 --- /dev/null +++ b/models/transformers/src/ops/split.rs @@ -0,0 +1,50 @@ +/* + Appellation: split + Contrib: FL03 +*/ +use ndarray::prelude::{Array2, Array3, Array4}; +use ndarray::ShapeError; + +// pub fn split(param: &Array, heads: usize) -> Result, ShapeError> { +// let mut dim = param.dim() +// let query = param.shape().last().unwrap() / heads; +// // reshape the qkv matrix into a 3d array +// let mut res = param.clone().into_shape((param.shape()[0], heads, query))?; +// // swap the sequence and head axes +// res.swap_axes(0, 1); +// Ok(res) +// } + +pub trait Split { + type Output; + + fn split(&self, heads: usize) -> Result; +} + +impl Split for Array2 { + type Output = Array3; + + fn split(&self, heads: usize) -> Result { + let (seq, model) = self.dim(); + let query = model / heads; + // reshape the qkv matrix into a 3d array + let mut res = self.clone().into_shape((seq, heads, query))?; + // swap the sequence and head axes + res.swap_axes(0, 1); + Ok(res) + } +} + +impl Split for Array3 { + type Output = Array4; + + fn split(&self, heads: usize) -> Result { + let (batch, seq, model) = self.dim(); + let query = model / heads; + // reshape the qkv matrix into a 3d array + let mut res = self.clone().into_shape((batch, seq, heads, query))?; + // swap the sequence and head axes + res.swap_axes(1, 2); + Ok(res) + } +} diff --git a/models/transformers/src/params/item.rs b/models/transformers/src/params/item.rs new file mode 100644 index 00000000..67528ab1 --- /dev/null +++ b/models/transformers/src/params/item.rs @@ -0,0 +1,79 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use nd::{ArrayBase, Dimension, Ix2, OwnedRepr, RawData}; +use strum::{AsRefStr, EnumCount, EnumDiscriminants, EnumIs, VariantNames}; + +#[derive(AsRefStr, EnumCount, EnumDiscriminants, EnumIs, VariantNames)] +#[strum_discriminants( + derive( + AsRefStr, + EnumCount, + EnumIs, + Hash, + Ord, + PartialOrd, + VariantNames, + strum::Display, + strum::EnumString, + ), + name(QKV), + strum(serialize_all = "lowercase") +)] +#[cfg_attr( + feature = "serde", + strum_discriminants( + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase", untagged) + ) +)] +#[strum(serialize_all = "lowercase")] +pub enum Entry, D = Ix2> +where + D: Dimension, + S: RawData, +{ + Q(ArrayBase), + K(ArrayBase), + V(ArrayBase), +} + +impl Entry +where + D: Dimension, + S: RawData, +{ + pub fn from_q(q: ArrayBase) -> Self { + Self::Q(q) + } + + pub fn from_k(k: ArrayBase) -> Self { + Self::K(k) + } + + pub fn from_v(v: ArrayBase) -> Self { + Self::V(v) + } + + pub fn q(&self) -> Option<&ArrayBase> { + match self { + Self::Q(q) => Some(q), + _ => None, + } + } + + pub fn k(&self) -> Option<&ArrayBase> { + match self { + Self::K(k) => Some(k), + _ => None, + } + } + + pub fn v(&self) -> Option<&ArrayBase> { + match self { + Self::V(v) => Some(v), + _ => None, + } + } +} diff --git a/models/transformers/src/params/mod.rs b/models/transformers/src/params/mod.rs index b4baf401..367f8b2a 100644 --- a/models/transformers/src/params/mod.rs +++ b/models/transformers/src/params/mod.rs @@ -2,9 +2,10 @@ Appellation: params Contrib: FL03 */ -pub use self::qkv::QKVBase; +pub use self::{item::*, store::QkvBase}; -mod qkv; +pub(crate) mod item; +pub(crate) mod store; macro_rules! params_ty { ($target:ident: [$($name:ident<$(&$lt:lifetime)?$repr:ident>),* $(,)?]) => { @@ -19,14 +20,16 @@ macro_rules! params_ty { } params_ty!( - QKVBase: [ - QKV, - ArcQKV, - ViewQKV<&'a ViewRepr>, + QkvBase: [ + Params, + ArcParams, + ParamsView<&'a ViewRepr>, ] ); +#[allow(unused_imports)] pub(crate) mod prelude { - pub use super::QKVBase; - pub use super::{ArcQKV, QKV}; + pub use super::item::{Entry, QKV}; + pub use super::store::QkvBase; + pub use super::{ArcParams, Params}; } diff --git a/models/transformers/src/params/qkv.rs b/models/transformers/src/params/qkv.rs deleted file mode 100644 index 231f48d9..00000000 --- a/models/transformers/src/params/qkv.rs +++ /dev/null @@ -1,70 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use nd::*; - -use num::traits::{One, Zero}; - -pub struct QKVBase, D = Ix2> -where - D: Dimension, - S: RawData, -{ - pub(crate) q: ArrayBase, - pub(crate) k: ArrayBase, - pub(crate) v: ArrayBase, -} - -impl QKVBase -where - D: Dimension, - S: RawData, -{ - pub fn builder(shape: Sh, builder: F) -> Self - where - F: Fn(D) -> ArrayBase, - Sh: ShapeBuilder, - { - let dim = shape.into_shape().raw_dim().clone(); - Self { - q: builder(dim.clone()), - k: builder(dim.clone()), - v: builder(dim), - } - } - - access!(q, k, v); - - qkv_builder!(new.default where A: Default, S: DataOwned); - qkv_builder!(ones.ones where A: Clone + One, S: DataOwned); - qkv_builder!(zeros.zeros where A: Clone + Zero, S: DataOwned); - - pub fn as_views(&self) -> (ArrayView, ArrayView, ArrayView) - where - S: Data, - { - (self.q.view(), self.k.view(), self.v.view()) - } - - /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension - pub fn dim(&self) -> D::Pattern { - self.q.dim() - } - /// Get the rank of the parameters; i.e. the number of dimensions. - pub fn rank(&self) -> usize { - self.q.ndim() - } - /// Returns the raw dimension ([D](ndarray::Dimension)) of the parameters. - pub fn raw_dim(&self) -> D { - self.q.raw_dim() - } - - pub fn shape(&self) -> &[usize] { - self.q.shape() - } - - param_views!(to_owned::(&self) where A: Clone, S: Data); - param_views!(to_shared::(&self) where A: Clone, S: DataShared); - param_views!(view::<'a, ViewRepr>(&self) where S: Data); -} diff --git a/models/transformers/src/params/store.rs b/models/transformers/src/params/store.rs new file mode 100644 index 00000000..90c13693 --- /dev/null +++ b/models/transformers/src/params/store.rs @@ -0,0 +1,83 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use concision::{dimensional, getters}; +use nd::*; +use num::traits::{One, Zero}; + +pub struct QkvBase, D = Ix2> +where + D: Dimension, + S: RawData, +{ + pub(crate) q: ArrayBase, + pub(crate) k: ArrayBase, + pub(crate) v: ArrayBase, +} + +impl QkvBase +where + D: Dimension, + S: RawData, +{ + pub fn builder(shape: Sh, builder: F) -> Self + where + F: Fn(D) -> ArrayBase, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: builder(dim.clone()), + k: builder(dim.clone()), + v: builder(dim), + } + } + + pub fn from_elem(shape: Sh, value: A) -> Self + where + Sh: ShapeBuilder, + A: Clone, + S: DataOwned, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: ArrayBase::from_elem(dim.clone(), value.clone()), + k: ArrayBase::from_elem(dim.clone(), value.clone()), + v: ArrayBase::from_elem(dim, value), + } + } + + pub fn as_qkv(&self) -> (ArrayView, ArrayView, ArrayView) + where + S: Data, + { + (self.q.view(), self.k.view(), self.v.view()) + } + + /// Consumes the store and returns a three-tuple consisting of the query, key, and value arrays respectively. + pub fn into_qkv(self) -> (ArrayBase, ArrayBase, ArrayBase) { + (self.q, self.k, self.v) + } + + pub fn qkv(&self) -> (&ArrayBase, &ArrayBase, &ArrayBase) { + (&self.q, &self.k, &self.v) + } + + ndbuilder!(new::default() where A: Default, S: DataOwned); + ndbuilder!(ones() where A: Clone + One, S: DataOwned); + ndbuilder!(zeros() where A: Clone + Zero, S: DataOwned); + + getters!(q, k, v => ArrayBase); + + dimensional!(q()); + + ndview!(into_owned::(self) where A: Clone, S: Data); + ndview!(to_owned::(&self) where A: Clone, S: Data); + + ndview!(into_shared::(self) where A: Clone, S: DataOwned); + ndview!(to_shared::(&self) where A: Clone, S: DataShared); + + ndview!(view::<'a, ViewRepr>(&self) where S: Data); + ndview!(view_mut::<'a, ViewRepr>(&mut self) where S: DataMut); +} diff --git a/models/transformers/src/primitives.rs b/models/transformers/src/primitives.rs new file mode 100644 index 00000000..96db829b --- /dev/null +++ b/models/transformers/src/primitives.rs @@ -0,0 +1,10 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::consts::*; + +pub mod consts { + /// The default number of layers used for the encoder / decoder. + pub const N: usize = 6; +} diff --git a/models/transformers/tests/attention.rs b/models/transformers/tests/attention.rs index 98aa4c0e..db1efe2a 100644 --- a/models/transformers/tests/attention.rs +++ b/models/transformers/tests/attention.rs @@ -6,14 +6,14 @@ extern crate concision_core as concision; extern crate concision_transformers as transformers; use concision::{linarr, Matmul}; -use transformers::{AttentionHead, QKV}; +use transformers::{AttentionHead, Params}; use ndarray::prelude::*; #[test] fn test_qkv() { let shape = (2048, 10); - let params = QKV::::new(shape); + let params = Params::::new(shape); assert_eq!(params.q(), &Array::default(shape)); } @@ -23,7 +23,7 @@ fn test_qkv_matmul() { // generate some sample data let data = linarr(shape).unwrap(); // initialize the parameters - let params = QKV::::ones(shape); + let params = Params::::ones(shape); // calculate the expected result let exp = Array2::::ones(shape).dot(&data.t()); // calculate the result diff --git a/models/transformers/tests/ops.rs b/models/transformers/tests/ops.rs new file mode 100644 index 00000000..c39b8efa --- /dev/null +++ b/models/transformers/tests/ops.rs @@ -0,0 +1,52 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_transformers as transformers; + +use concision::linarr; +use ndarray::prelude::*; +use transformers::ops::*; + +#[test] +fn test_merge() { + let shape = (3, 4, 5); + let dout = (4, 15); + let arr = linarr::(shape.clone()).unwrap(); + let a = arr.clone().merge().unwrap(); + let b = merge(&arr, 0, 1).unwrap(); + + assert_eq!(a.dim(), dout); + assert_eq!(a.dim(), b.dim()); + assert_eq!(a, b); +} + +#[test] +fn test_merge_batch() { + let shape = (2, 3, 4, 5); + let dout = (2, 4, 15); + let arr = linarr::(shape).unwrap(); + let a = arr.merge().unwrap(); + let b = merge(&arr, 1, 2).unwrap(); + + assert_eq!(a.dim(), dout); + assert_eq!(a, b); +} + +#[test] +fn reshape_ops() { + let dim_input: [usize; 3] = [2, 4, 6]; // (batch, seq, model) + let dim_split = [2, 2, 4, 3]; // (batch, heads, seq, model) + let data = linarr::(dim_input).unwrap(); + + let a = split_batch(&data, 2).unwrap(); + let b = a.merge().unwrap(); // merge_batch(&a).unwrap(); + + assert_eq!(a.shape(), &dim_split); + assert_eq!(b.shape(), &dim_input); + assert_eq!(a, data.split(2).unwrap()); + for (i, &j) in b.indexed_iter() { + assert_eq!(j, data[i]); + } +}