From bbd96f02e118caf8942a56398b46401d0879b2e4 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 12:46:45 -0500 Subject: [PATCH 01/12] v0.1.15-dev Signed-off-by: Joe McCain III --- core/src/func/activate/mod.rs | 12 +++- core/src/func/activate/nl.rs | 57 ++++++++++--------- core/src/macros/getters.rs | 2 +- models/linear/src/mlp/model.rs | 3 +- models/linear/src/mlp/perceptron.rs | 37 +++++++++--- models/transformers/src/attention/head.rs | 34 +++++------ models/transformers/src/attention/mod.rs | 2 +- .../src/attention/multi/multi_head.rs | 9 +-- .../transformers/src/codec/encoder/layer.rs | 35 ++++++++---- models/transformers/src/config/mod.rs | 20 +++---- models/transformers/src/lib.rs | 5 +- models/transformers/src/params/qkv.rs | 11 +++- 12 files changed, 144 insertions(+), 83 deletions(-) diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs index cd1c2f0b..2de12f71 100644 --- a/core/src/func/activate/mod.rs +++ b/core/src/func/activate/mod.rs @@ -15,7 +15,6 @@ pub(crate) mod prelude { pub use super::{Activate, Evaluate}; } -#[doc(hidden)] pub trait Activate { type Output; @@ -30,3 +29,14 @@ pub trait Evaluate { } activator!(LinearActor::(T::clone) where T: Clone); + +impl Activate for F +where + F: for<'a> Fn(&'a U) -> V, +{ + type Output = V; + + fn activate(&self, args: &U) -> Self::Output { + self(args) + } +} diff --git a/core/src/func/activate/nl.rs b/core/src/func/activate/nl.rs index 6901db0e..2d48b659 100644 --- a/core/src/func/activate/nl.rs +++ b/core/src/func/activate/nl.rs @@ -7,7 +7,7 @@ use nd::*; use num::complex::{Complex, ComplexFloat}; use num::traits::Zero; -fn _relu(args: T) -> T +pub fn relu(args: T) -> T where T: PartialOrd + Zero, { @@ -17,14 +17,14 @@ where T::zero() } -fn _sigmoid(args: T) -> T +pub fn sigmoid(args: T) -> T where T: ComplexFloat, { (T::one() + args.neg().exp()).recip() } -fn _softmax(args: &ArrayBase) -> Array +pub fn softmax(args: &ArrayBase) -> Array where A: ComplexFloat + ScalarOperand, D: Dimension, @@ -34,7 +34,7 @@ where &e / e.sum() } -fn _softmax_axis(args: &ArrayBase, axis: usize) -> Array +pub fn softmax_axis(args: &ArrayBase, axis: usize) -> Array where A: ComplexFloat + ScalarOperand, D: RemoveAxis, @@ -45,7 +45,7 @@ where &e / &e.sum_axis(axis) } -fn _tanh(args: T) -> T +pub fn tanh(args: T) -> T where T: ComplexFloat, { @@ -62,6 +62,8 @@ unary!( pub trait SoftmaxAxis { type Output; + fn softmax(self) -> Self::Output; + fn softmax_axis(self, axis: usize) -> Self::Output; } @@ -92,25 +94,24 @@ macro_rules! nonlinear { // nonlinear!(@arr $rho::$call); }; (@impl $rho:ident::$call:ident<$T:ty>) => { - paste::paste! { - impl $rho for $T { - type Output = $T; + impl $rho for $T { + type Output = $T; - fn $call(self) -> Self::Output { - [<_ $call>](self) - } + fn $call(self) -> Self::Output { + $call(self) } + } - impl<'a> $rho for &'a $T { - type Output = $T; + impl<'a> $rho for &'a $T { + type Output = $T; - fn $call(self) -> Self::Output { - [<_ $call>](*self) - } + fn $call(self) -> Self::Output { + $call(*self) } } + }; (@arr $name:ident::$call:ident) => { impl $name for ArrayBase @@ -143,8 +144,6 @@ macro_rules! nonlinear { macro_rules! nonlinear_rho { ($name:ident::$call:ident where A: $($rest:tt)* ) => { - - paste::paste! { impl $name for ArrayBase where D: Dimension, @@ -154,7 +153,7 @@ macro_rules! nonlinear_rho { type Output = Array; fn $call(self) -> Self::Output { - self.mapv([<_ $call>]) + self.mapv($call) } } @@ -167,11 +166,9 @@ macro_rules! nonlinear_rho { type Output = Array; fn $call(self) -> Self::Output { - self.mapv([<_ $call>]) + self.mapv($call) } } - } - }; } @@ -232,7 +229,7 @@ where type Output = Array; fn softmax(self) -> Self::Output { - _softmax(&self) + softmax(&self) } } @@ -245,7 +242,7 @@ where type Output = Array; fn softmax(self) -> Self::Output { - _softmax(self) + softmax(self) } } @@ -257,8 +254,12 @@ where { type Output = Array; + fn softmax(self) -> Self::Output { + softmax(&self) + } + fn softmax_axis(self, axis: usize) -> Self::Output { - _softmax_axis(&self, axis) + softmax_axis(&self, axis) } } @@ -270,7 +271,11 @@ where { type Output = Array; + fn softmax(self) -> Self::Output { + softmax(self) + } + fn softmax_axis(self, axis: usize) -> Self::Output { - _softmax_axis(&self, axis) + softmax_axis(&self, axis) } } diff --git a/core/src/macros/getters.rs b/core/src/macros/getters.rs index f8bbd3a1..921e2c3a 100644 --- a/core/src/macros/getters.rs +++ b/core/src/macros/getters.rs @@ -75,4 +75,4 @@ macro_rules! nested_getter { ($($field:ident).*::<[$($call:ident),* $(,)?]> => $out:ty) => { $crate::getter!($($($field).*::$call<$out>)*); }; -} \ No newline at end of file +} diff --git a/models/linear/src/mlp/model.rs b/models/linear/src/mlp/model.rs index 4128cb88..8bcc66f6 100644 --- a/models/linear/src/mlp/model.rs +++ b/models/linear/src/mlp/model.rs @@ -2,9 +2,10 @@ Appellation: model Contrib: FL03 */ +use super::Perceptron; pub struct Mlp { - input: I, + input: Perceptron, hidden: H, output: O, } \ No newline at end of file diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index ac62f0b9..da3f8dcb 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -2,21 +2,39 @@ Appellation: perceptron Contrib: FL03 */ -use concision::prelude::{Module, Predict, PredictError}; +use crate::{Biased, Linear}; +use concision::prelude::{Predict, PredictError}; +use nd::prelude::*; +use num::Zero; -pub struct Perceptron +pub struct ReLU; + +impl ReLU { + pub fn new() -> Self { + Self + } + + pub fn activate(&self, args: &T) -> T + where + T: Clone + PartialOrd + Zero, + { + concision::func::relu(args.clone()) + } +} + +pub struct Perceptron where - M: Module, + D: Dimension, { - module: M, + module: Linear, rho: F, } -impl Perceptron +impl Perceptron where - M: Module, + D: Dimension, { - pub fn new(module: M, rho: F) -> Self { + pub fn new(module: Linear, rho: F) -> Self { Self { module, rho } } @@ -28,10 +46,11 @@ where } } -impl Predict for Perceptron +impl Predict for Perceptron where + D: Dimension, F: for<'a> Fn(&'a Y) -> Y, - M: Predict + Module, + Linear: Predict, { type Output = Y; diff --git a/models/transformers/src/attention/head.rs b/models/transformers/src/attention/head.rs index 76e69f0c..afcd8e8e 100644 --- a/models/transformers/src/attention/head.rs +++ b/models/transformers/src/attention/head.rs @@ -4,7 +4,6 @@ */ use super::{Score, _attention}; use crate::params::QkvBase; -use concision::getters; use concision::nn::Dropout; use nd::linalg::Dot; use nd::*; @@ -12,9 +11,9 @@ use num::complex::ComplexFloat; // #68 /// [AttentionHead] implements the scaled dot-product attention mechanism formally defined in -/// [Attention is all you need](https://arxiv.org/abs/1706.03762). The structure is designed to -/// be flexible, relying upon the n-dimensional [QkvBase] to store the query, key, and value tensors. -/// More so, the head may be configured with an optional dropout and/or masking layers. +/// [Attention is all you need](https://arxiv.org/abs/1706.03762). The mechanism utilizes +/// [QkvBase] to store the query, key, and value tensors. More so, the attention head +/// has two optional features: a dropout and masking layer. /// /// ### `dropout` /// @@ -58,15 +57,7 @@ where D: Dimension, S: RawData, { - pub fn from_params(params: QkvBase) -> Self { - Self { - #[cfg(feature = "rand")] - dropout: None, - mask: None, - params, - } - } - + /// Create a new instance with the specified shape and builder function. pub fn builder(shape: Sh, builder: F) -> Self where F: Fn(D) -> ArrayBase, @@ -74,7 +65,7 @@ where { Self::from_params(QkvBase::builder(shape, builder)) } - + /// Create a new instance with the specified shape and value. pub fn from_elem(shape: Sh, value: A) -> Self where Sh: ShapeBuilder, @@ -83,6 +74,15 @@ where { Self::from_params(QkvBase::from_elem(shape, value)) } + /// Create a new instance from the [QkvBase] parameters. + pub fn from_params(params: QkvBase) -> Self { + Self { + #[cfg(feature = "rand")] + dropout: None, + mask: None, + params, + } + } /// Computes the [Score] using scaled dot-product attention. pub fn attention(&self) -> Score where @@ -139,10 +139,12 @@ where } } - getters!(params::<[q, k, v]> => ArrayBase); - ndbuilder!(new::default() where A: Default, S: DataOwned); + ndbuilder!(default() where A: Default, S: DataOwned); ndbuilder!(ones() where A: Clone + num::One, S: DataOwned); ndbuilder!(zeros() where A: Clone + num::Zero, S: DataOwned); + + concision::dimensional!(params()); + concision::getters!(params::<[q, k, v]> => ArrayBase); } #[cfg(feature = "rand")] diff --git a/models/transformers/src/attention/mod.rs b/models/transformers/src/attention/mod.rs index 2f4c7295..c8f2c176 100644 --- a/models/transformers/src/attention/mod.rs +++ b/models/transformers/src/attention/mod.rs @@ -9,8 +9,8 @@ //! the Transformer model, primarily due to its capabilities in natural language //! processing (NLP) domains pub(crate) use self::_impl_methods::*; -pub use self::{config::AttentionConfig, head::AttentionHead, score::Score}; pub use self::utils::*; +pub use self::{config::AttentionConfig, head::AttentionHead, score::Score}; pub(crate) mod config; pub(crate) mod head; diff --git a/models/transformers/src/attention/multi/multi_head.rs b/models/transformers/src/attention/multi/multi_head.rs index f6baa7cb..fc81cfc6 100644 --- a/models/transformers/src/attention/multi/multi_head.rs +++ b/models/transformers/src/attention/multi/multi_head.rs @@ -22,18 +22,19 @@ where D: Dimension, S: RawData, { + /// Returns an immutable reference to the [AttentionConfig] pub const fn config(&self) -> &AttentionConfig { &self.config } - + /// Returns an immutable reference to the [AttentionHead] pub const fn head(&self) -> &AttentionHead { &self.head } - + /// Returns a mutable reference to the [AttentionHead] pub fn head_mut(&mut self) -> &mut AttentionHead { &mut self.head } - + /// Returns an immutable slice containing the [Linear] layers pub fn linears(&self) -> &[Linear] { &self.linears } @@ -69,7 +70,7 @@ where fn default() -> Self { Self { config: AttentionConfig::default(), - head: AttentionHead::default(), + head: Default::default(), linears: Vec::new(), } } diff --git a/models/transformers/src/codec/encoder/layer.rs b/models/transformers/src/codec/encoder/layer.rs index 5c00ebcf..549dd173 100644 --- a/models/transformers/src/codec/encoder/layer.rs +++ b/models/transformers/src/codec/encoder/layer.rs @@ -3,24 +3,39 @@ Contrib: FL03 */ use crate::attention::multi::MultiHeadAttention; +use crate::model::ffn::FeedForwardNetwork; +use linear::Biased; +use nd::prelude::*; -#[derive(Default)] -pub struct EncoderLayer { - pub(crate) attention: MultiHeadAttention, +pub struct EncoderLayer +where + D: Dimension, +{ + pub(crate) attention: MultiHeadAttention, + pub(crate) ffn: FeedForwardNetwork, } -impl EncoderLayer { - pub fn new() -> Self { - let attention = MultiHeadAttention::default(); - - Self { attention } +impl EncoderLayer +where + D: Dimension, +{ + pub fn new(attention: MultiHeadAttention, ffn: FeedForwardNetwork) -> Self { + Self { attention, ffn } } /// Returns an immutable reference to the multi-head, self-attention layer. - pub fn attention(&self) -> &MultiHeadAttention { + pub fn attention(&self) -> &MultiHeadAttention { &self.attention } /// Returns a mutable reference to the multi-head, self-attention layer. - pub fn attention_mut(&mut self) -> &mut MultiHeadAttention { + pub fn attention_mut(&mut self) -> &mut MultiHeadAttention { &mut self.attention } + /// Returns an immutable reference to the feed-forward network layer. + pub fn ffn(&self) -> &FeedForwardNetwork { + &self.ffn + } + /// Returns a mutable reference to the feed-forward network layer. + pub fn ffn_mut(&mut self) -> &mut FeedForwardNetwork { + &mut self.ffn + } } diff --git a/models/transformers/src/config/mod.rs b/models/transformers/src/config/mod.rs index 8f57601c..f9400309 100644 --- a/models/transformers/src/config/mod.rs +++ b/models/transformers/src/config/mod.rs @@ -21,7 +21,12 @@ impl TransformerConfig { } } - getters!(dropout>, features, heads, layers); + getters!( + dropout>, + features, + heads, + layers + ); getters!(features::<[d_model, qkv]>); getters!(features::<[dk, dq, dv]> => usize); } @@ -33,10 +38,7 @@ pub struct Features { impl Features { pub fn new(d_model: usize, qkv: QkvShape) -> Self { - Self { - d_model, - qkv, - } + Self { d_model, qkv } } getters!(d_model, qkv); @@ -51,13 +53,9 @@ pub struct QkvShape { impl QkvShape { pub fn new(dq: usize, dk: usize, dv: usize) -> Self { - Self { - dq, - dk, - dv, - } + Self { dq, dk, dv } } - + pub fn std(dk: usize) -> Self { let (dq, dv) = (dk, dk); diff --git a/models/transformers/src/lib.rs b/models/transformers/src/lib.rs index a6439f8f..f85a4e5b 100644 --- a/models/transformers/src/lib.rs +++ b/models/transformers/src/lib.rs @@ -18,7 +18,10 @@ extern crate concision_linear as linear; extern crate ndarray as nd; #[doc(inline)] -pub use self::{attention::prelude::*, params::prelude::*, ops::prelude::*, primitives::*, transformer::Transformer}; +pub use self::{ + attention::prelude::*, ops::prelude::*, params::prelude::*, primitives::*, + transformer::Transformer, +}; #[macro_use] pub(crate) mod macros; diff --git a/models/transformers/src/params/qkv.rs b/models/transformers/src/params/qkv.rs index d35298e6..307b968c 100644 --- a/models/transformers/src/params/qkv.rs +++ b/models/transformers/src/params/qkv.rs @@ -88,8 +88,15 @@ where qkv_view!(view_mut::<'a, ViewRepr>(&mut self) where S: DataMut); } -impl QkvBase where S: RawData { - pub fn std(dk: usize, d_model: usize) -> Self where A: Default, S: DataOwned { +impl QkvBase +where + S: RawData, +{ + pub fn std(dk: usize, d_model: usize) -> Self + where + A: Default, + S: DataOwned, + { Self::new((dk, d_model)) } } From de13d9367b042105cf6bf0a2994ae139086ff3ef Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 13:34:31 -0500 Subject: [PATCH 02/12] update Signed-off-by: Joe McCain III --- core/src/func/activate/mod.rs | 19 +++++-- core/src/macros/activate.rs | 4 +- models/kan/Cargo.toml | 8 ++- models/kan/src/lib.rs | 6 +++ models/kan/src/model/mod.rs | 1 + models/linear/src/impls/model/impl_linear.rs | 18 ++++++- models/linear/src/lib.rs | 1 + models/linear/src/mlp/mod.rs | 22 +++++--- models/linear/src/mlp/perceptron.rs | 53 ++++++++++++-------- models/linear/tests/mlp.rs | 23 +++++++++ 10 files changed, 122 insertions(+), 33 deletions(-) create mode 100644 models/kan/src/model/mod.rs create mode 100644 models/linear/tests/mlp.rs diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs index 2de12f71..81488f57 100644 --- a/core/src/func/activate/mod.rs +++ b/core/src/func/activate/mod.rs @@ -15,10 +15,15 @@ pub(crate) mod prelude { pub use super::{Activate, Evaluate}; } +/// [Activate] designates a function or structure that can be used +/// as an activation function for a neural network. +/// +/// The trait enables implemented models to employ various activation +/// functions either as a pure function or as a structure. pub trait Activate { type Output; - fn activate(&self, args: &T) -> Self::Output; + fn activate(&self, args: T) -> Self::Output; } #[doc(hidden)] @@ -32,11 +37,19 @@ activator!(LinearActor::(T::clone) where T: Clone); impl Activate for F where - F: for<'a> Fn(&'a U) -> V, + F: Fn(U) -> V, { type Output = V; - fn activate(&self, args: &U) -> Self::Output { + fn activate(&self, args: U) -> Self::Output { self(args) } } + +impl Activate for Box> { + type Output = V; + + fn activate(&self, args: U) -> Self::Output { + self.as_ref().activate(args) + } +} diff --git a/core/src/macros/activate.rs b/core/src/macros/activate.rs index 44e6eeb3..74cc1395 100644 --- a/core/src/macros/activate.rs +++ b/core/src/macros/activate.rs @@ -9,10 +9,10 @@ macro_rules! activator { #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct $name; - impl $crate::func::activate::Activate for $name $($rest)* { + impl<'a, T> $crate::func::activate::Activate<&'a T> for $name $($rest)* { type Output = $out; - fn activate(&self, args: &T) -> Self::Output { + fn activate(&self, args: &'a T) -> Self::Output { $rho(args) } } diff --git a/models/kan/Cargo.toml b/models/kan/Cargo.toml index d135fb66..ab1ddb9d 100644 --- a/models/kan/Cargo.toml +++ b/models/kan/Cargo.toml @@ -50,7 +50,8 @@ serde = [ "serde-1", "concision-core/serde", "ndarray/serde-1", - "num/serde" + "num/serde", + "splines/serde", ] serde-1 = [ @@ -67,6 +68,7 @@ std = [ "ndarray/std", "num/std", "serde?/std", + "splines/std", "strum/std", ] @@ -107,6 +109,10 @@ features = ["derive"] optional = true version = "1" +[dependencies.splines] +default-features = false +version = "4" + [dependencies.tracing] optional = true version = "0.1" diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs index cd6bc790..0f618f66 100644 --- a/models/kan/src/lib.rs +++ b/models/kan/src/lib.rs @@ -5,6 +5,10 @@ //! # Kolmogorov-Arnold Networks (KAN) //! //! Kolmogorov-Arnold Networks (KAN) are a novel class of neural networks based on the Kolmogorov-Arnold Representation Theorem. +//! KANs propose a fundamental shift in network architecture, elegantly blending splines with Multi-Layer Perceptrons. +//! +//! While scaling these models has yet to be fully explored, KANs have demonstrated their ability to rival traditional neural networks in terms of performance +//! with far fewer parameters. //! These models have already demonstrated that they are viable alternatives to traditional multi-layer perceptrons (MLPs) and convolutional neural networks (CNNs). //! //! ### Resources @@ -18,4 +22,6 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; +pub mod model; + pub mod prelude {} diff --git a/models/kan/src/model/mod.rs b/models/kan/src/model/mod.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/models/kan/src/model/mod.rs @@ -0,0 +1 @@ + diff --git a/models/linear/src/impls/model/impl_linear.rs b/models/linear/src/impls/model/impl_linear.rs index 49ee85ba..e1c6d546 100644 --- a/models/linear/src/impls/model/impl_linear.rs +++ b/models/linear/src/impls/model/impl_linear.rs @@ -4,7 +4,7 @@ */ use crate::{Config, Linear, ParamMode, ParamsBase}; use core::borrow::{Borrow, BorrowMut}; -use nd::{DataOwned, Ix2, RawData, RemoveAxis}; +use nd::{DataOwned, Ix2, RawData, RawDataClone, RemoveAxis}; impl Linear where @@ -51,3 +51,19 @@ where &mut self.params } } + +impl Clone for Linear +where + A: Clone, + D: RemoveAxis, + K: Clone, + S: RawDataClone, + ParamsBase: Clone, +{ + fn clone(&self) -> Self { + Self { + config: self.config.clone(), + params: self.params.clone(), + } + } +} diff --git a/models/linear/src/lib.rs b/models/linear/src/lib.rs index 6f457ab7..cbfdc0e6 100644 --- a/models/linear/src/lib.rs +++ b/models/linear/src/lib.rs @@ -56,6 +56,7 @@ mod impls { } pub mod prelude { + pub use crate::mlp::prelude::*; pub use crate::model::prelude::*; pub use crate::norm::prelude::*; pub use crate::params::prelude::*; diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index c040a93a..00cc601e 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -8,9 +8,13 @@ //! //! #[doc(inline)] -pub use self::perceptron::Perceptron; +pub use self::perceptron::*; -pub mod perceptron; +pub(crate) mod perceptron; + +pub(crate) mod prelude { + pub use super::perceptron::Perceptron; +} pub trait MultiLayerPerceptron { type Input; @@ -18,10 +22,16 @@ pub trait MultiLayerPerceptron { type Output; } -pub trait Neuron {} +pub trait Container { + type Elem; +} -pub trait Rho { - type Output; +pub trait Params { + type Data: Container; + type Dim; + type Elem; +} - fn activate(&self, args: T) -> Self::Output; +pub trait Neuron { + type Rho; } diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index da3f8dcb..16042739 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -3,26 +3,39 @@ Contrib: FL03 */ use crate::{Biased, Linear}; -use concision::prelude::{Predict, PredictError}; +use concision::prelude::{relu, Activate, Predict, PredictError}; use nd::prelude::*; -use num::Zero; - +use nd::Data; +use num::traits::Zero; pub struct ReLU; -impl ReLU { - pub fn new() -> Self { - Self +impl Activate> for ReLU +where + A: Clone + PartialOrd + Zero, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn activate(&self, args: ArrayBase) -> Self::Output { + args.mapv(relu) } +} - pub fn activate(&self, args: &T) -> T - where - T: Clone + PartialOrd + Zero, - { - concision::func::relu(args.clone()) +impl<'a, A, S, D> Activate<&'a ArrayBase> for ReLU +where + A: Clone + PartialOrd + Zero, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn activate(&self, args: &'a ArrayBase) -> Self::Output { + args.mapv(relu) } } -pub struct Perceptron +pub struct Perceptron where D: Dimension, { @@ -30,7 +43,7 @@ where rho: F, } -impl Perceptron +impl Perceptron where D: Dimension, { @@ -38,24 +51,24 @@ where Self { module, rho } } - pub fn activate(&self, args: &T) -> T + pub fn activate(&self, args: T) -> F::Output where - F: Fn(&T) -> T, + F: Activate, { - (self.rho)(args) + self.rho.activate(args) } } -impl Predict for Perceptron +impl Predict for Perceptron where D: Dimension, - F: for<'a> Fn(&'a Y) -> Y, + F: for<'a> Activate<&'a Y, Output = Z>, Linear: Predict, { - type Output = Y; + type Output = Z; fn predict(&self, args: &X) -> Result { let res = self.module.predict(args)?; - Ok(self.activate(&res)) + Ok(self.rho.activate(&res)) } } diff --git a/models/linear/tests/mlp.rs b/models/linear/tests/mlp.rs new file mode 100644 index 00000000..558e8e7e --- /dev/null +++ b/models/linear/tests/mlp.rs @@ -0,0 +1,23 @@ +/* + Appellation: mlp + Contrib: FL03 +*/ +extern crate concision_core as cnc; +extern crate concision_linear as linear; + +use cnc::prelude::{linarr, Forward, ReLU}; +use linear::mlp::{self, Perceptron}; +use linear::{Biased, Features, Linear}; +use ndarray::prelude::*; + +#[test] +#[cfg(feature = "rand")] +fn test_perceptron() { + use cnc::InitializeExt; + let samples = 100; + let features = Features::new(1, 300); + let data = linarr::((samples, features.dmodel())).unwrap(); + let layer = Linear::::lecun_normal(features, 1); + let mlp = Perceptron::::new(layer.clone(), mlp::ReLU); + assert_eq!(mlp.forward(&data), layer.forward(&data).relu()); +} From ef074ba01aa5f4374d0e7da6e0bf5adafd1b2824 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 14:06:44 -0500 Subject: [PATCH 03/12] update Signed-off-by: Joe McCain III --- core/src/nn/model/module.rs | 20 +-- core/src/traits/mod.rs | 2 + core/src/traits/params.rs | 23 ++++ data/Cargo.toml | 5 +- data/src/lib.rs | 2 - data/src/params/impls/impl_rand.rs | 27 ---- data/src/params/kinds.rs | 61 ---------- data/src/params/mod.rs | 56 --------- data/src/params/parameter.rs | 115 ------------------ data/src/params/store.rs | 113 ----------------- data/tests/params.rs | 40 ------ models/linear/src/impls/model/impl_model.rs | 18 --- models/linear/src/impls/params/impl_params.rs | 10 +- models/linear/src/mlp/perceptron.rs | 50 +++++--- models/linear/tests/mlp.rs | 2 +- 15 files changed, 83 insertions(+), 461 deletions(-) create mode 100644 core/src/traits/params.rs delete mode 100644 data/src/params/impls/impl_rand.rs delete mode 100644 data/src/params/kinds.rs delete mode 100644 data/src/params/mod.rs delete mode 100644 data/src/params/parameter.rs delete mode 100644 data/src/params/store.rs delete mode 100644 data/tests/params.rs diff --git a/core/src/nn/model/module.rs b/core/src/nn/model/module.rs index 8d8ee23b..edb450fd 100644 --- a/core/src/nn/model/module.rs +++ b/core/src/nn/model/module.rs @@ -2,18 +2,18 @@ Appellation: modules Contrib: FL03 */ -use crate::{Config, Predict}; +use crate::{Config, Params, Predict}; pub type ModuleDyn = Box>; -pub type DynModuleExt = Box>; -pub type Stack = Vec>>; +pub type DynModuleExt = Box>; +pub type Stack = Vec>>; /// A `Module` defines any object that may be used as a layer in a neural network. -/// [Config](Module::Config) is a type that defines the configuration of the module; including any and all hyperparameters. -/// [Params](Module::Params) is a type that defines the parameters of the module; typically references a Linear set of parameters { weights, bias } +/// [Config](Module::Config) contains all of the hyperparameters for the model. +/// [Params](Module::Params) refers to an object used to store the various learnable parameters. pub trait Module { type Config: Config; - type Params; + type Params: Params; fn config(&self) -> &Self::Config; @@ -22,6 +22,10 @@ pub trait Module { fn params_mut(&mut self) -> &mut Self::Params; } -pub trait ModuleExt: Module + Predict {} +pub trait Layer: Module + Predict {} -impl ModuleExt for M where M: Module + Predict {} +/* + ************* Implementations ************* +*/ + +impl Layer for M where M: Module + Predict {} diff --git a/core/src/traits/mod.rs b/core/src/traits/mod.rs index 9dc12247..8a59195e 100644 --- a/core/src/traits/mod.rs +++ b/core/src/traits/mod.rs @@ -6,6 +6,7 @@ pub use self::prelude::*; pub mod num; pub mod ops; +pub mod params; pub mod predict; pub mod setup; pub mod train; @@ -49,6 +50,7 @@ pub(crate) mod prelude { pub use super::misc::prelude::*; pub use super::num::*; pub use super::ops::*; + pub use super::params::*; pub use super::predict::*; pub use super::setup::*; pub use super::train::*; diff --git a/core/src/traits/params.rs b/core/src/traits/params.rs new file mode 100644 index 00000000..1e3834b0 --- /dev/null +++ b/core/src/traits/params.rs @@ -0,0 +1,23 @@ +/* + Appellation: params + Contrib: FL03 +*/ + +/// A `Params` object is used to store the various learnable parameters of a model. +/// +/// ### Specifications +/// +/// - `Elem`: The type of the elements being stored +/// +pub trait Params { + type Elem; +} + +pub trait ParamFeatures { + type Dim: nd::Dimension; + +} + +pub trait Parameter { + type Kind: 'static; +} \ No newline at end of file diff --git a/data/Cargo.toml b/data/Cargo.toml index 6c184db9..460d512a 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -77,12 +77,11 @@ wasi = [ [lib] crate-type = ["lib"] -doctest = false +doctest = true test = true [[test]] -name = "params" -required-features = ["std"] +name = "default" [build-dependencies] diff --git a/data/src/lib.rs b/data/src/lib.rs index 5f1d6ead..621752cb 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -17,7 +17,6 @@ pub use self::dataset::Dataset; pub use self::traits::prelude::*; pub mod dataset; -pub mod params; #[doc(hidden)] pub mod preproc; pub mod tensor; @@ -26,7 +25,6 @@ pub mod types; pub mod prelude { pub use super::dataset::*; - pub use super::params::prelude::*; pub use super::traits::prelude::*; pub use super::types::prelude::*; } diff --git a/data/src/params/impls/impl_rand.rs b/data/src/params/impls/impl_rand.rs deleted file mode 100644 index d5f066cc..00000000 --- a/data/src/params/impls/impl_rand.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - Appellation: impl_rand - Contrib: FL03 -*/ -use crate::params::Parameter; -use concision::init::rand_distr::uniform::SampleUniform; -use concision::init::rand_distr::{Distribution, StandardNormal}; -use concision::InitializeExt; -use ndarray::{Array, Dimension}; -use num::Float; - -impl Parameter -where - D: Dimension, - T: Float, - StandardNormal: Distribution, -{ - pub fn init_uniform(mut self, dk: T) -> Self - where - T: SampleUniform, - ::Sampler: Clone, - { - let dim = self.value.dim(); - self.value = Array::uniform(dim, dk); - self - } -} diff --git a/data/src/params/kinds.rs b/data/src/params/kinds.rs deleted file mode 100644 index b9b92225..00000000 --- a/data/src/params/kinds.rs +++ /dev/null @@ -1,61 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - AsRefStr, - Clone, - Debug, - Default, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - VariantNames, -)] -#[non_exhaustive] -#[repr(usize)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "lowercase", tag = "kind") -)] -#[strum(serialize_all = "lowercase")] -pub enum ParamKind { - #[default] - Bias, - Weight, - Other(String), -} - -impl ParamKind { - pub fn bias() -> Self { - Self::Bias - } - - pub fn weight() -> Self { - Self::Weight - } - - pub fn other(name: impl ToString) -> Self { - Self::Other(name.to_string()) - } -} - -impl core::fmt::Display for ParamKind { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let content = match self { - ParamKind::Bias => "bias", - ParamKind::Weight => "weight", - ParamKind::Other(name) => name, - }; - write!(f, "{}", content) - } -} diff --git a/data/src/params/mod.rs b/data/src/params/mod.rs deleted file mode 100644 index 8b35b030..00000000 --- a/data/src/params/mod.rs +++ /dev/null @@ -1,56 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -//! # Parameters -//! -//! ## Overview -//! -pub use self::{kinds::*, parameter::*}; - -pub(crate) mod kinds; -pub(crate) mod parameter; - -pub mod store; - -mod impls { - #[cfg(feature = "rand")] - mod impl_rand; -} - -pub trait Param { - type Key; - type Value; -} - -pub trait Params { - type Store; -} - -pub(crate) mod prelude { - pub use super::kinds::ParamKind; - pub use super::parameter::Parameter; - pub use super::store::ParamStore; - pub use super::{Param, Params}; -} - -#[cfg(test)] -mod tests { - use super::*; - use concision::linarr; - use nd::linalg::Dot; - use nd::prelude::*; - - #[test] - fn test_parameter() { - let value = linarr::((3, 3)).unwrap(); - let mut param = Parameter::::new((10, 1), ParamKind::Bias, "bias"); - param.set_params(value.clone()); - - assert_eq!(param.kind(), &ParamKind::Bias); - assert_eq!(param.name(), "bias"); - - let x = linarr::((3,)).unwrap(); - assert_eq!(param.dot(&x), value.dot(&x)); - } -} diff --git a/data/src/params/parameter.rs b/data/src/params/parameter.rs deleted file mode 100644 index 46dbe68b..00000000 --- a/data/src/params/parameter.rs +++ /dev/null @@ -1,115 +0,0 @@ -/* - Appellation: param - Contrib: FL03 -*/ -use super::{Param, ParamKind}; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Dimension, Ix2}; -use ndarray::IntoDimension; -use num::Float; - -#[derive(Clone, Debug, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Parameter -where - D: Dimension, -{ - pub(crate) features: D, - pub(crate) kind: ParamKind, - pub(crate) name: String, - pub(crate) value: Array, -} - -impl Parameter -where - D: Dimension, -{ - pub fn new(features: impl IntoDimension, kind: ParamKind, name: impl ToString) -> Self - where - T: Clone + Default, - { - let features = features.into_dimension(); - Self { - features: features.clone(), - kind, - name: name.to_string(), - value: Array::default(features), - } - } - - pub fn kind(&self) -> &ParamKind { - &self.kind - } - - pub fn kind_mut(&mut self) -> &mut ParamKind { - &mut self.kind - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn name_mut(&mut self) -> &mut String { - &mut self.name - } - - pub fn data(&self) -> &Array { - &self.value - } - - pub fn params_mut(&mut self) -> &mut Array { - &mut self.value - } - - pub fn set_kind(&mut self, kind: ParamKind) { - self.kind = kind; - } - - pub fn set_name(&mut self, name: String) { - self.name = name; - } - - pub fn set_params(&mut self, params: Array) { - self.value = params; - } - - pub fn with_kind(self, kind: ParamKind) -> Self { - Self { kind, ..self } - } - - pub fn with_name(self, name: impl ToString) -> Self { - Self { - name: name.to_string(), - ..self - } - } - - pub fn with_params(self, params: Array) -> Self { - Self { - value: params, - ..self - } - } -} - -impl Param for Parameter -where - T: Float, - D: Dimension, -{ - type Key = ParamKind; - type Value = Array; -} - -impl Dot for Parameter -where - Array: Dot, - D: Dimension, - T: Float, -{ - type Output = O; - - fn dot(&self, rhs: &S) -> Self::Output { - self.value.dot(rhs) - } -} diff --git a/data/src/params/store.rs b/data/src/params/store.rs deleted file mode 100644 index cf451e37..00000000 --- a/data/src/params/store.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - Appellation: store - Contrib: FL03 -*/ -#![cfg(any(feature = "alloc", feature = "std"))] -use super::{ParamKind, Parameter}; -use ndarray::prelude::{Dimension, Ix2}; -use num::Float; - -#[cfg(all(feature = "alloc", no_std))] -use alloc::collections::BTreeMap as Map; -#[cfg(feature = "std")] -use std::collections::HashMap as Map; - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct ParamStore -where - T: Float, - D: Dimension, -{ - store: Map>, -} - -impl ParamStore -where - D: Dimension, - T: Float, -{ - pub fn new() -> Self { - Self { store: Map::new() } - } - - pub fn get(&self, kind: &ParamKind) -> Option<&Parameter> { - self.store.get(kind) - } - - pub fn get_mut(&mut self, kind: &ParamKind) -> Option<&mut Parameter> { - self.store.get_mut(kind) - } - - pub fn insert(&mut self, param: Parameter) { - self.store.insert(param.kind().clone(), param); - } - - pub fn remove(&mut self, kind: &ParamKind) -> Option> { - self.store.remove(kind) - } -} - -impl Extend> for ParamStore -where - D: Dimension, - T: Float, -{ - fn extend>>(&mut self, iter: I) { - for param in iter { - self.insert(param); - } - } -} - -macro_rules! impl_into_iter { - ($($p:ident)::*) => { - impl_into_iter!(@impl $($p)::*); - }; - (@impl $($p:ident)::*) => { - impl IntoIterator for ParamStore - where - D: Dimension, - T: Float, - { - type Item = (ParamKind, Parameter); - type IntoIter = $($p)::*::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.store.into_iter() - } - } - - impl<'a, T, D> IntoIterator for &'a mut ParamStore - where - D: Dimension, - T: Float, - { - type Item = (&'a ParamKind, &'a mut Parameter); - type IntoIter = $($p)::*::IterMut<'a, ParamKind, Parameter>; - - fn into_iter(self) -> Self::IntoIter { - self.store.iter_mut() - } - } - }; - -} -#[cfg(feature = "std")] -impl_into_iter!(std::collections::hash_map); -#[cfg(not(feature = "std"))] -impl_into_iter!(alloc::collections::btree_map); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_model_store() { - let (inputs, outputs) = (5, 3); - - let _shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - - let _params = ParamStore::::new(); - } -} diff --git a/data/tests/params.rs b/data/tests/params.rs deleted file mode 100644 index 134ff343..00000000 --- a/data/tests/params.rs +++ /dev/null @@ -1,40 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -extern crate concision_core as concision; -extern crate concision_data as data; - -use concision::linarr; -use data::params::{ParamKind, Parameter}; -use ndarray::linalg::Dot; -use ndarray::prelude::*; - -#[test] -fn test_parameter() { - let a = linarr::((3,)).unwrap(); - let p = linarr::((3, 3)).unwrap(); - let mut param = Parameter::::new((10, 1), ParamKind::Bias, "bias"); - param.set_params(p.clone()); - - assert_eq!(param.kind(), &ParamKind::Bias); - assert_eq!(param.name(), "bias"); - assert_eq!(param.dot(&a), p.dot(&a)); -} - -#[test] -fn test_param_kind_map() { - let name = "test"; - let other = ParamKind::other(name); - - let data = [ - (ParamKind::Bias, "bias"), - (ParamKind::Weight, "weight"), - (other.clone(), "test"), - (ParamKind::other("mask"), "mask"), - ]; - - for (kind, expected) in &data { - assert_eq!(kind.to_string(), expected.to_string()); - } -} diff --git a/models/linear/src/impls/model/impl_model.rs b/models/linear/src/impls/model/impl_model.rs index 7dcad1f4..475c66bd 100644 --- a/models/linear/src/impls/model/impl_model.rs +++ b/models/linear/src/impls/model/impl_model.rs @@ -26,24 +26,6 @@ where } } -// impl concision::Forward for Linear -// where -// D: RemoveAxis, -// LinearParams: concision::Forward, -// { -// type Output = V; - -// #[cfg_attr( -// feature = "tracing", -// tracing::instrument(skip_all, level = "debug", name = "predict", target = "linear") -// )] -// fn forward(&self, input: &U) -> Self::Output { -// #[cfg(feature = "tracing")] -// tracing::debug!("Predicting with linear model"); -// self.params().forward(input) -// } -// } - impl Predict for Linear where D: RemoveAxis, diff --git a/models/linear/src/impls/params/impl_params.rs b/models/linear/src/impls/params/impl_params.rs index 17ae9863..72df65a3 100644 --- a/models/linear/src/impls/params/impl_params.rs +++ b/models/linear/src/impls/params/impl_params.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::params::ParamsBase; -use concision::prelude::{Predict, PredictError}; +use concision::prelude::{Params, Predict, PredictError}; use core::ops::Add; use nd::linalg::Dot; use nd::*; @@ -24,6 +24,14 @@ where } } +impl Params for ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + type Elem = A; +} + impl Clone for ParamsBase where A: Clone, diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index 16042739..b0bf7fb8 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -2,14 +2,33 @@ Appellation: perceptron Contrib: FL03 */ -use crate::{Biased, Linear}; -use concision::prelude::{relu, Activate, Predict, PredictError}; +use concision::prelude::{Activate, Module, Predict, PredictError, ReLU}; use nd::prelude::*; use nd::Data; use num::traits::Zero; -pub struct ReLU; -impl Activate> for ReLU +pub struct Rho(T); + +impl Rho { + pub fn new(rho: T) -> Self { + Self(rho) + } + + pub fn get(&self) -> &T { + &self.0 + } + + pub fn get_mut(&mut self) -> &mut T { + &mut self.0 + } + + pub fn activate(&self) -> &T { + &self.0 + } +} +pub struct Relu; + +impl Activate> for Relu where A: Clone + PartialOrd + Zero, D: Dimension, @@ -18,11 +37,11 @@ where type Output = Array; fn activate(&self, args: ArrayBase) -> Self::Output { - args.mapv(relu) + args.relu() } } -impl<'a, A, S, D> Activate<&'a ArrayBase> for ReLU +impl<'a, A, S, D> Activate<&'a ArrayBase> for Relu where A: Clone + PartialOrd + Zero, D: Dimension, @@ -31,23 +50,23 @@ where type Output = Array; fn activate(&self, args: &'a ArrayBase) -> Self::Output { - args.mapv(relu) + args.relu() } } -pub struct Perceptron +pub struct Perceptron where - D: Dimension, + M: Module, { - module: Linear, + module: M, rho: F, } -impl Perceptron +impl Perceptron where - D: Dimension, + M: Module, { - pub fn new(module: Linear, rho: F) -> Self { + pub fn new(module: M, rho: F) -> Self { Self { module, rho } } @@ -59,11 +78,10 @@ where } } -impl Predict for Perceptron +impl Predict for Perceptron where - D: Dimension, F: for<'a> Activate<&'a Y, Output = Z>, - Linear: Predict, + M: Module + Predict, { type Output = Z; diff --git a/models/linear/tests/mlp.rs b/models/linear/tests/mlp.rs index 558e8e7e..3b8d020d 100644 --- a/models/linear/tests/mlp.rs +++ b/models/linear/tests/mlp.rs @@ -18,6 +18,6 @@ fn test_perceptron() { let features = Features::new(1, 300); let data = linarr::((samples, features.dmodel())).unwrap(); let layer = Linear::::lecun_normal(features, 1); - let mlp = Perceptron::::new(layer.clone(), mlp::ReLU); + let mlp = Perceptron::new(layer.clone(), mlp::Relu); assert_eq!(mlp.forward(&data), layer.forward(&data).relu()); } From 8360dd4dd80c3f0dac1e38bc845ab41f5961f199 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 14:31:09 -0500 Subject: [PATCH 04/12] delete archived code Signed-off-by: Joe McCain III --- .artifacts/archive/neural/Cargo.toml | 94 ----- .artifacts/archive/neural/src/errors/error.rs | 203 ----------- .artifacts/archive/neural/src/errors/mod.rs | 9 - .../archive/neural/src/exp/layers/config.rs | 97 ------ .../archive/neural/src/exp/layers/mod.rs | 18 - .../archive/neural/src/exp/layers/sublayer.rs | 44 --- .../archive/neural/src/exp/layers/wrapper.rs | 30 -- .artifacts/archive/neural/src/exp/mod.rs | 9 - .../archive/neural/src/exp/models/mod.rs | 38 -- .../archive/neural/src/exp/models/modules.rs | 23 -- .../archive/neural/src/exp/models/store.rs | 133 ------- .../neural/src/func/activate/activator.rs | 162 --------- .../neural/src/func/activate/binary.rs | 83 ----- .../neural/src/func/activate/linear.rs | 85 ----- .../archive/neural/src/func/activate/mod.rs | 146 -------- .../neural/src/func/activate/nl/mod.rs | 119 ------- .../neural/src/func/activate/nl/relu.rs | 93 ----- .../neural/src/func/activate/nl/sigmoid.rs | 85 ----- .../neural/src/func/activate/nl/softmax.rs | 137 -------- .../neural/src/func/activate/nl/tanh.rs | 85 ----- .../archive/neural/src/func/loss/kinds.rs | 44 --- .../archive/neural/src/func/loss/mod.rs | 32 -- .../archive/neural/src/func/loss/reg/huber.rs | 75 ---- .../archive/neural/src/func/loss/reg/mae.rs | 31 -- .../archive/neural/src/func/loss/reg/mod.rs | 112 ------ .../archive/neural/src/func/loss/reg/mse.rs | 79 ----- .artifacts/archive/neural/src/func/mod.rs | 56 --- .artifacts/archive/neural/src/func/rms.rs | 52 --- .../archive/neural/src/layers/cmp/features.rs | 105 ------ .../archive/neural/src/layers/cmp/kinds.rs | 140 -------- .../archive/neural/src/layers/cmp/mod.rs | 105 ------ .artifacts/archive/neural/src/layers/layer.rs | 287 --------------- .artifacts/archive/neural/src/layers/mod.rs | 113 ------ .../archive/neural/src/layers/params.rs | 235 ------------- .../archive/neural/src/layers/seq/mod.rs | 11 - .../neural/src/layers/seq/sequential.rs | 172 --------- .artifacts/archive/neural/src/lib.rs | 44 --- .../archive/neural/src/models/config.rs | 24 -- .artifacts/archive/neural/src/models/mod.rs | 15 - .artifacts/archive/neural/src/models/model.rs | 178 ---------- .artifacts/archive/neural/src/models/modes.rs | 64 ---- .../archive/neural/src/models/params.rs | 328 ------------------ .artifacts/archive/neural/src/neurons/mod.rs | 80 ----- .artifacts/archive/neural/src/neurons/node.rs | 251 -------------- .../archive/neural/src/neurons/perceptron.rs | 211 ----------- .../archive/neural/src/neurons/synapse.rs | 9 - .artifacts/archive/neural/src/nn/gnn/mod.rs | 30 -- .artifacts/archive/neural/src/nn/gnn/model.rs | 23 -- .artifacts/archive/neural/src/nn/gnn/tasks.rs | 18 - .artifacts/archive/neural/src/nn/kinds.rs | 157 --------- .artifacts/archive/neural/src/nn/mod.rs | 13 - .artifacts/archive/neural/src/nn/position.rs | 20 -- .artifacts/archive/neural/src/ops/compile.rs | 12 - .artifacts/archive/neural/src/ops/dropout.rs | 65 ---- .artifacts/archive/neural/src/ops/mod.rs | 32 -- .artifacts/archive/neural/src/ops/norm.rs | 114 ------ .artifacts/archive/neural/src/ops/predict.rs | 22 -- .artifacts/archive/neural/src/ops/train.rs | 27 -- .artifacts/archive/neural/src/params/iter.rs | 33 -- .artifacts/archive/neural/src/params/kinds.rs | 96 ----- .../archive/neural/src/params/masks/mask.rs | 148 -------- .../archive/neural/src/params/masks/mod.rs | 36 -- .artifacts/archive/neural/src/params/mod.rs | 92 ----- .artifacts/archive/neural/src/params/param.rs | 138 -------- .artifacts/archive/neural/src/params/store.rs | 104 ------ .../archive/neural/src/params/store/group.rs | 183 ---------- .../archive/neural/src/params/variable.rs | 16 - .artifacts/archive/neural/src/primitives.rs | 28 -- .artifacts/archive/neural/src/specs.rs | 66 ---- .artifacts/archive/neural/src/utils.rs | 24 -- .artifacts/archive/neural/tests/default.rs | 8 - .artifacts/archive/nlp/Cargo.toml | 73 ---- .../archive/nlp/src/embed/context/mod.rs | 6 - .artifacts/archive/nlp/src/embed/embedding.rs | 35 -- .artifacts/archive/nlp/src/embed/mod.rs | 23 -- .artifacts/archive/nlp/src/embed/words/mod.rs | 8 - .../archive/nlp/src/embed/words/word2vec.rs | 16 - .artifacts/archive/nlp/src/encode/mod.rs | 23 -- .../archive/nlp/src/encode/positional.rs | 164 --------- .artifacts/archive/nlp/src/lib.rs | 20 -- .artifacts/archive/nlp/src/primitives.rs | 10 - .artifacts/archive/nlp/src/specs.rs | 4 - .artifacts/archive/nlp/src/utils.rs | 4 - .artifacts/archive/nlp/tests/default.rs | 8 - .artifacts/archive/nn/Cargo.toml | 122 ------- .artifacts/archive/nn/examples/basic.rs | 93 ----- .artifacts/archive/nn/src/lib.rs | 31 -- .artifacts/archive/nn/tests/default.rs | 13 - .artifacts/archive/optim/Cargo.toml | 55 --- .artifacts/archive/optim/examples/norm.rs | 41 --- .artifacts/archive/optim/examples/sgd.rs | 31 -- .artifacts/archive/optim/src/grad/adam.rs | 310 ----------------- .artifacts/archive/optim/src/grad/descent.rs | 139 -------- .artifacts/archive/optim/src/grad/gradient.rs | 153 -------- .artifacts/archive/optim/src/grad/mod.rs | 137 -------- .artifacts/archive/optim/src/grad/modes.rs | 34 -- .artifacts/archive/optim/src/grad/sgd.rs | 251 -------------- .artifacts/archive/optim/src/lib.rs | 30 -- .artifacts/archive/optim/src/norm/kinds.rs | 72 ---- .artifacts/archive/optim/src/norm/mod.rs | 90 ----- .../archive/optim/src/norm/normalizer.rs | 40 --- .artifacts/archive/optim/src/optimizer.rs | 23 -- .artifacts/archive/optim/src/params/mod.rs | 46 --- .artifacts/archive/optim/src/primitives.rs | 18 - .artifacts/archive/optim/src/specs.rs | 22 -- .artifacts/archive/optim/src/utils.rs | 72 ---- .artifacts/archive/optim/tests/default.rs | 8 - .artifacts/archive/transformers/Cargo.toml | 54 --- .../transformers/src/attention/head.rs | 162 --------- .../archive/transformers/src/attention/mod.rs | 119 ------- .../src/attention/multi/attention.rs | 98 ------ .../transformers/src/attention/multi/mod.rs | 98 ------ .../src/attention/multi/params.rs | 48 --- .../transformers/src/attention/params/dim.rs | 222 ------------ .../src/attention/params/hyperparams.rs | 114 ------ .../transformers/src/attention/params/mod.rs | 24 -- .../transformers/src/attention/params/qkv.rs | 56 --- .../transformers/src/attention/weights.rs | 288 --------------- .../transformers/src/codec/decode/decoder.rs | 6 - .../transformers/src/codec/decode/mod.rs | 21 -- .../transformers/src/codec/decode/params.rs | 4 - .../transformers/src/codec/encode/encoder.rs | 52 --- .../transformers/src/codec/encode/mod.rs | 33 -- .../transformers/src/codec/encode/params.rs | 36 -- .../transformers/src/codec/encode/stack.rs | 29 -- .../archive/transformers/src/codec/mod.rs | 20 -- .../archive/transformers/src/ffn/mod.rs | 35 -- .../archive/transformers/src/ffn/network.rs | 63 ---- .../archive/transformers/src/ffn/params.rs | 57 --- .artifacts/archive/transformers/src/lib.rs | 46 --- .../archive/transformers/src/ops/merge.rs | 43 --- .../archive/transformers/src/ops/mod.rs | 87 ----- .../archive/transformers/src/ops/split.rs | 50 --- .../archive/transformers/src/primitives.rs | 33 -- .artifacts/archive/transformers/src/specs.rs | 4 - .../transformers/src/transform/config.rs | 16 - .../archive/transformers/src/transform/mod.rs | 17 - .../transformers/src/transform/transformer.rs | 20 -- .artifacts/archive/transformers/src/utils.rs | 4 - .../archive/transformers/tests/default.rs | 8 - core/src/nn/model/config.rs | 12 +- core/src/traits/setup.rs | 8 + data/Cargo.toml | 7 +- models/kan/src/actor.rs | 9 + models/kan/src/actor/config.rs | 17 + models/kan/src/lib.rs | 4 + models/kan/src/model/mod.rs | 4 + 147 files changed, 48 insertions(+), 10256 deletions(-) delete mode 100644 .artifacts/archive/neural/Cargo.toml delete mode 100644 .artifacts/archive/neural/src/errors/error.rs delete mode 100644 .artifacts/archive/neural/src/errors/mod.rs delete mode 100644 .artifacts/archive/neural/src/exp/layers/config.rs delete mode 100644 .artifacts/archive/neural/src/exp/layers/mod.rs delete mode 100644 .artifacts/archive/neural/src/exp/layers/sublayer.rs delete mode 100644 .artifacts/archive/neural/src/exp/layers/wrapper.rs delete mode 100644 .artifacts/archive/neural/src/exp/mod.rs delete mode 100644 .artifacts/archive/neural/src/exp/models/mod.rs delete mode 100644 .artifacts/archive/neural/src/exp/models/modules.rs delete mode 100644 .artifacts/archive/neural/src/exp/models/store.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/activator.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/binary.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/linear.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/mod.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/nl/mod.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/nl/relu.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/nl/sigmoid.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/nl/softmax.rs delete mode 100644 .artifacts/archive/neural/src/func/activate/nl/tanh.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/kinds.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/mod.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/reg/huber.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/reg/mae.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/reg/mod.rs delete mode 100644 .artifacts/archive/neural/src/func/loss/reg/mse.rs delete mode 100644 .artifacts/archive/neural/src/func/mod.rs delete mode 100644 .artifacts/archive/neural/src/func/rms.rs delete mode 100644 .artifacts/archive/neural/src/layers/cmp/features.rs delete mode 100644 .artifacts/archive/neural/src/layers/cmp/kinds.rs delete mode 100644 .artifacts/archive/neural/src/layers/cmp/mod.rs delete mode 100644 .artifacts/archive/neural/src/layers/layer.rs delete mode 100644 .artifacts/archive/neural/src/layers/mod.rs delete mode 100644 .artifacts/archive/neural/src/layers/params.rs delete mode 100644 .artifacts/archive/neural/src/layers/seq/mod.rs delete mode 100644 .artifacts/archive/neural/src/layers/seq/sequential.rs delete mode 100644 .artifacts/archive/neural/src/lib.rs delete mode 100644 .artifacts/archive/neural/src/models/config.rs delete mode 100644 .artifacts/archive/neural/src/models/mod.rs delete mode 100644 .artifacts/archive/neural/src/models/model.rs delete mode 100644 .artifacts/archive/neural/src/models/modes.rs delete mode 100644 .artifacts/archive/neural/src/models/params.rs delete mode 100644 .artifacts/archive/neural/src/neurons/mod.rs delete mode 100644 .artifacts/archive/neural/src/neurons/node.rs delete mode 100644 .artifacts/archive/neural/src/neurons/perceptron.rs delete mode 100644 .artifacts/archive/neural/src/neurons/synapse.rs delete mode 100644 .artifacts/archive/neural/src/nn/gnn/mod.rs delete mode 100644 .artifacts/archive/neural/src/nn/gnn/model.rs delete mode 100644 .artifacts/archive/neural/src/nn/gnn/tasks.rs delete mode 100644 .artifacts/archive/neural/src/nn/kinds.rs delete mode 100644 .artifacts/archive/neural/src/nn/mod.rs delete mode 100644 .artifacts/archive/neural/src/nn/position.rs delete mode 100644 .artifacts/archive/neural/src/ops/compile.rs delete mode 100644 .artifacts/archive/neural/src/ops/dropout.rs delete mode 100644 .artifacts/archive/neural/src/ops/mod.rs delete mode 100644 .artifacts/archive/neural/src/ops/norm.rs delete mode 100644 .artifacts/archive/neural/src/ops/predict.rs delete mode 100644 .artifacts/archive/neural/src/ops/train.rs delete mode 100644 .artifacts/archive/neural/src/params/iter.rs delete mode 100644 .artifacts/archive/neural/src/params/kinds.rs delete mode 100644 .artifacts/archive/neural/src/params/masks/mask.rs delete mode 100644 .artifacts/archive/neural/src/params/masks/mod.rs delete mode 100644 .artifacts/archive/neural/src/params/mod.rs delete mode 100644 .artifacts/archive/neural/src/params/param.rs delete mode 100644 .artifacts/archive/neural/src/params/store.rs delete mode 100644 .artifacts/archive/neural/src/params/store/group.rs delete mode 100644 .artifacts/archive/neural/src/params/variable.rs delete mode 100644 .artifacts/archive/neural/src/primitives.rs delete mode 100644 .artifacts/archive/neural/src/specs.rs delete mode 100644 .artifacts/archive/neural/src/utils.rs delete mode 100644 .artifacts/archive/neural/tests/default.rs delete mode 100644 .artifacts/archive/nlp/Cargo.toml delete mode 100644 .artifacts/archive/nlp/src/embed/context/mod.rs delete mode 100644 .artifacts/archive/nlp/src/embed/embedding.rs delete mode 100644 .artifacts/archive/nlp/src/embed/mod.rs delete mode 100644 .artifacts/archive/nlp/src/embed/words/mod.rs delete mode 100644 .artifacts/archive/nlp/src/embed/words/word2vec.rs delete mode 100644 .artifacts/archive/nlp/src/encode/mod.rs delete mode 100644 .artifacts/archive/nlp/src/encode/positional.rs delete mode 100644 .artifacts/archive/nlp/src/lib.rs delete mode 100644 .artifacts/archive/nlp/src/primitives.rs delete mode 100644 .artifacts/archive/nlp/src/specs.rs delete mode 100644 .artifacts/archive/nlp/src/utils.rs delete mode 100644 .artifacts/archive/nlp/tests/default.rs delete mode 100644 .artifacts/archive/nn/Cargo.toml delete mode 100644 .artifacts/archive/nn/examples/basic.rs delete mode 100644 .artifacts/archive/nn/src/lib.rs delete mode 100644 .artifacts/archive/nn/tests/default.rs delete mode 100644 .artifacts/archive/optim/Cargo.toml delete mode 100644 .artifacts/archive/optim/examples/norm.rs delete mode 100644 .artifacts/archive/optim/examples/sgd.rs delete mode 100644 .artifacts/archive/optim/src/grad/adam.rs delete mode 100644 .artifacts/archive/optim/src/grad/descent.rs delete mode 100644 .artifacts/archive/optim/src/grad/gradient.rs delete mode 100644 .artifacts/archive/optim/src/grad/mod.rs delete mode 100644 .artifacts/archive/optim/src/grad/modes.rs delete mode 100644 .artifacts/archive/optim/src/grad/sgd.rs delete mode 100644 .artifacts/archive/optim/src/lib.rs delete mode 100644 .artifacts/archive/optim/src/norm/kinds.rs delete mode 100644 .artifacts/archive/optim/src/norm/mod.rs delete mode 100644 .artifacts/archive/optim/src/norm/normalizer.rs delete mode 100644 .artifacts/archive/optim/src/optimizer.rs delete mode 100644 .artifacts/archive/optim/src/params/mod.rs delete mode 100644 .artifacts/archive/optim/src/primitives.rs delete mode 100644 .artifacts/archive/optim/src/specs.rs delete mode 100644 .artifacts/archive/optim/src/utils.rs delete mode 100644 .artifacts/archive/optim/tests/default.rs delete mode 100644 .artifacts/archive/transformers/Cargo.toml delete mode 100644 .artifacts/archive/transformers/src/attention/head.rs delete mode 100644 .artifacts/archive/transformers/src/attention/mod.rs delete mode 100644 .artifacts/archive/transformers/src/attention/multi/attention.rs delete mode 100644 .artifacts/archive/transformers/src/attention/multi/mod.rs delete mode 100644 .artifacts/archive/transformers/src/attention/multi/params.rs delete mode 100644 .artifacts/archive/transformers/src/attention/params/dim.rs delete mode 100644 .artifacts/archive/transformers/src/attention/params/hyperparams.rs delete mode 100644 .artifacts/archive/transformers/src/attention/params/mod.rs delete mode 100644 .artifacts/archive/transformers/src/attention/params/qkv.rs delete mode 100644 .artifacts/archive/transformers/src/attention/weights.rs delete mode 100644 .artifacts/archive/transformers/src/codec/decode/decoder.rs delete mode 100644 .artifacts/archive/transformers/src/codec/decode/mod.rs delete mode 100644 .artifacts/archive/transformers/src/codec/decode/params.rs delete mode 100644 .artifacts/archive/transformers/src/codec/encode/encoder.rs delete mode 100644 .artifacts/archive/transformers/src/codec/encode/mod.rs delete mode 100644 .artifacts/archive/transformers/src/codec/encode/params.rs delete mode 100644 .artifacts/archive/transformers/src/codec/encode/stack.rs delete mode 100644 .artifacts/archive/transformers/src/codec/mod.rs delete mode 100644 .artifacts/archive/transformers/src/ffn/mod.rs delete mode 100644 .artifacts/archive/transformers/src/ffn/network.rs delete mode 100644 .artifacts/archive/transformers/src/ffn/params.rs delete mode 100644 .artifacts/archive/transformers/src/lib.rs delete mode 100644 .artifacts/archive/transformers/src/ops/merge.rs delete mode 100644 .artifacts/archive/transformers/src/ops/mod.rs delete mode 100644 .artifacts/archive/transformers/src/ops/split.rs delete mode 100644 .artifacts/archive/transformers/src/primitives.rs delete mode 100644 .artifacts/archive/transformers/src/specs.rs delete mode 100644 .artifacts/archive/transformers/src/transform/config.rs delete mode 100644 .artifacts/archive/transformers/src/transform/mod.rs delete mode 100644 .artifacts/archive/transformers/src/transform/transformer.rs delete mode 100644 .artifacts/archive/transformers/src/utils.rs delete mode 100644 .artifacts/archive/transformers/tests/default.rs create mode 100644 models/kan/src/actor.rs create mode 100644 models/kan/src/actor/config.rs diff --git a/.artifacts/archive/neural/Cargo.toml b/.artifacts/archive/neural/Cargo.toml deleted file mode 100644 index 387ca3b7..00000000 --- a/.artifacts/archive/neural/Cargo.toml +++ /dev/null @@ -1,94 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-neural" -readme.workspace = true -repository.workspace = true -version.workspace = true - - -[features] -default = [ - "std", -] - -blas = [ - "ndarray/blas", - "concision-core/blas", -] - -std = [ - "concision-core/std", - "ndarray/std", - "num/std", - "strum/std", -] - -intel-mkl-system = [ - "blas", - "ndarray-linalg/intel-mkl-system", -] - -intel-mkl-static = [ - "blas", - "ndarray-linalg/intel-mkl-static", -] - -netlib-system = [ - "blas", - "ndarray-linalg/netlib-system", -] - -netlib-static = [ - "blas", - "ndarray-linalg/netlib-static", -] - -openblas-system = [ - "blas", - "ndarray-linalg/openblas-system", -] - -openblas-static = [ - "blas", - "ndarray-linalg/openblas-static", -] - -[lib] -bench = false -crate-type = ["rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-core = { features = ["full"], path = "../../core", version = "0.1.12" } - -anyhow = "1" -itertools = "0.12" -ndarray = { features = ["serde-1"], version = "0.15" } -ndarray-linalg = "0.16" -ndarray-rand = "0.14" -ndarray-stats = "0.5" -num = { features = ["rand", "serde"], version = "0.4" } -petgraph = { features = ["serde-1"], version = "0.6" } -serde = { features = ["derive"], version = "1" } -serde_json = "1" -smart-default = "0.7" -strum = { features = ["derive"], version = "0.26" } - -[dev-dependencies] - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] diff --git a/.artifacts/archive/neural/src/errors/error.rs b/.artifacts/archive/neural/src/errors/error.rs deleted file mode 100644 index 7e5b52cc..00000000 --- a/.artifacts/archive/neural/src/errors/error.rs +++ /dev/null @@ -1,203 +0,0 @@ -/* - Appellation: error - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use smart_default::SmartDefault; -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, VariantNames}; - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - SmartDefault, - VariantNames, - Deserialize, - Serialize, -)] -#[non_exhaustive] -#[serde(rename_all = "lowercase", untagged)] -#[strum(serialize_all = "lowercase")] -pub enum MlError { - Compute(ComputeError), - Data(String), - Dimension(String), - #[default] - Error(String), - Network(NetworkError), -} - -impl std::error::Error for MlError {} - -impl From> for MlError { - fn from(err: Box) -> Self { - Self::Error(err.to_string()) - } -} - -impl From for MlError { - fn from(err: String) -> Self { - Self::Error(err) - } -} - -impl From<&str> for MlError { - fn from(err: &str) -> Self { - Self::Error(err.to_string()) - } -} - -impl From for MlError { - fn from(err: NetworkError) -> Self { - Self::Network(err) - } -} - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - SmartDefault, - VariantNames, - Deserialize, - Serialize, -)] -#[non_exhaustive] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum PredictError { - Activation(String), - Arithmetic(String), - Layer(String), - Format(String), - #[default] - Other(String), -} - -impl std::error::Error for PredictError {} - -impl From> for PredictError { - fn from(err: Box) -> Self { - Self::Other(err.to_string()) - } -} - -impl From for PredictError { - fn from(err: String) -> Self { - Self::Other(err) - } -} - -impl From<&str> for PredictError { - fn from(err: &str) -> Self { - Self::Other(err.to_string()) - } -} - -impl From for PredictError { - fn from(err: anyhow::Error) -> Self { - Self::Other(err.to_string()) - } -} - -impl From for PredictError { - fn from(err: ndarray::ShapeError) -> Self { - Self::Format(err.to_string()) - } -} - -impl From for PredictError { - fn from(err: ndarray_linalg::error::LinalgError) -> Self { - Self::Arithmetic(err.to_string()) - } -} - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - SmartDefault, - VariantNames, - Deserialize, - Serialize, -)] -#[non_exhaustive] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum ComputeError { - Arithmetic(String), - #[default] - Process(String), - ShapeError(String), -} - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - SmartDefault, - VariantNames, - Deserialize, - Serialize, -)] -#[non_exhaustive] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum NetworkError { - Layer(String), - #[default] - Network(String), -} - -macro_rules! impl_from_error { - ($base:ident::$variant:ident<$($err:ty),*>) => { - $( - impl_from_error!(@impl $base::$variant<$err>); - )* - }; - (@impl $base:ident::$variant:ident<$err:ty>) => { - impl From<$err> for $base { - fn from(err: $err) -> Self { - Self::$variant(err.to_string()) - } - } - }; -} - -impl_from_error!(ComputeError::Arithmetic); diff --git a/.artifacts/archive/neural/src/errors/mod.rs b/.artifacts/archive/neural/src/errors/mod.rs deleted file mode 100644 index 9e51d373..00000000 --- a/.artifacts/archive/neural/src/errors/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: errors - Contrib: FL03 -*/ -pub use self::error::*; - -pub(crate) mod error; - -pub(crate) mod utils {} diff --git a/.artifacts/archive/neural/src/exp/layers/config.rs b/.artifacts/archive/neural/src/exp/layers/config.rs deleted file mode 100644 index 00ed5705..00000000 --- a/.artifacts/archive/neural/src/exp/layers/config.rs +++ /dev/null @@ -1,97 +0,0 @@ -/* - Appellation: config - Contrib: FL03 -*/ -use crate::layers::{LayerKind, LayerShape}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct LayerConfig { - biased: bool, - features: LayerShape, - kind: LayerKind, - name: String, -} - -impl LayerConfig { - pub fn new(biased: bool, features: LayerShape, kind: LayerKind, name: impl ToString) -> Self { - Self { - biased, - features, - kind, - name: name.to_string(), - } - } - - pub fn is_biased(&self) -> bool { - self.biased - } - - pub fn features(&self) -> &LayerShape { - &self.features - } - - pub fn features_mut(&mut self) -> &mut LayerShape { - &mut self.features - } - - pub fn kind(&self) -> LayerKind { - self.kind - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn set_biased(&mut self, biased: bool) { - self.biased = biased; - } - - pub fn set_features(&mut self, features: LayerShape) { - self.features = features; - } - - pub fn set_kind(&mut self, kind: LayerKind) { - self.kind = kind; - } - - pub fn set_name(&mut self, name: String) { - self.name = name; - } - - pub fn biased(mut self) -> Self { - self.biased = true; - self - } - - pub fn unbiased(mut self) -> Self { - self.biased = false; - self - } - - pub fn with_features(mut self, features: LayerShape) -> Self { - self.features = features; - self - } - - pub fn with_kind(mut self, kind: LayerKind) -> Self { - self.kind = kind; - self - } - - pub fn with_name(mut self, name: String) -> Self { - self.name = name; - self - } -} - -impl From for LayerConfig { - fn from(features: LayerShape) -> Self { - Self { - biased: false, - features, - kind: LayerKind::default(), - name: String::new(), - } - } -} diff --git a/.artifacts/archive/neural/src/exp/layers/mod.rs b/.artifacts/archive/neural/src/exp/layers/mod.rs deleted file mode 100644 index 0550993a..00000000 --- a/.artifacts/archive/neural/src/exp/layers/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - Appellation: exp - Contrib: FL03 -*/ -//! # Experimental Layers -pub use self::{config::*, sublayer::*, wrapper::*}; - -pub(crate) mod config; -pub(crate) mod sublayer; -pub(crate) mod wrapper; - -pub trait Layer { - type Config; - type Params; -} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/exp/layers/sublayer.rs b/.artifacts/archive/neural/src/exp/layers/sublayer.rs deleted file mode 100644 index 49f4df8b..00000000 --- a/.artifacts/archive/neural/src/exp/layers/sublayer.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - Appellation: sublayers - Contrib: FL03 -*/ -use crate::layers::Layer; -use crate::prelude::{Activate, Forward, LayerNorm, LinearActivation}; - -use ndarray::prelude::{Array2, NdFloat}; -use num::{Float, FromPrimitive}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Sublayer -where - A: Activate, - T: Float, -{ - layer: Layer, - norm: LayerNorm, -} - -impl Sublayer -where - A: Activate, - T: Float, -{ - pub fn new(layer: Layer, norm: LayerNorm) -> Self { - Self { layer, norm } - } -} - -impl Forward> for Sublayer -where - A: Activate, - T: FromPrimitive + NdFloat, -{ - type Output = Array2; - - fn forward(&self, data: &Array2) -> Self::Output { - let norm = self.norm.forward(data); - let layer = data + self.layer.forward(&norm); - layer - } -} diff --git a/.artifacts/archive/neural/src/exp/layers/wrapper.rs b/.artifacts/archive/neural/src/exp/layers/wrapper.rs deleted file mode 100644 index ee824e1a..00000000 --- a/.artifacts/archive/neural/src/exp/layers/wrapper.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - Appellation: sublayers - Contrib: FL03 -*/ -use crate::func::activate::Activate; -use crate::layers::Layer; - -use ndarray::prelude::Array2; -use num::Float; - -pub trait Wrap { - type Output; - - fn wrap(&self, obj: T) -> Self::Output; -} - -pub trait Wrapper -where - T: Float, -{ - fn apply(&self, data: &Array2) -> Array2; - - fn wrap(&self, layer: Layer) - where - A: Activate; - - fn wrapper(&self) -> &Self; - - fn wrapper_mut(&mut self) -> &mut Self; -} diff --git a/.artifacts/archive/neural/src/exp/mod.rs b/.artifacts/archive/neural/src/exp/mod.rs deleted file mode 100644 index dff39ac9..00000000 --- a/.artifacts/archive/neural/src/exp/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: exp - Contrib: FL03 -*/ -//! # Experimental -//! - -pub mod layers; -pub mod models; diff --git a/.artifacts/archive/neural/src/exp/models/mod.rs b/.artifacts/archive/neural/src/exp/models/mod.rs deleted file mode 100644 index c73329b1..00000000 --- a/.artifacts/archive/neural/src/exp/models/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - Appellation: exp - Contrib: FL03 -*/ -//! # Experimental Models -pub use self::{modules::*, store::*}; - -pub(crate) mod modules; -pub(crate) mod store; - -// use crate::prelude::Predict; -// use ndarray::prelude::Array2; -// use num::Float; - -// pub trait Model: Predict> -// where -// T: Float, -// { -// type Config; - -// fn name(&self) -> &str; - -// fn modules(&self) -> &Vec>>; - -// fn modules_mut(&mut self) -> &mut Vec>>; - -// fn register_module(&mut self, module: Box>) -> &mut Self { -// self.modules_mut().push(module); -// self -// } - -// fn get_module(&self, name: &str) -> Option<&Box>> { -// self.modules().iter().find(|m| m.name() == name) -// } -// } - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/exp/models/modules.rs b/.artifacts/archive/neural/src/exp/models/modules.rs deleted file mode 100644 index 54e88fab..00000000 --- a/.artifacts/archive/neural/src/exp/models/modules.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: modules - Contrib: FL03 -*/ -//! # Model -//! -use crate::prelude::Predict; -use ndarray::prelude::Array2; -use std::collections::HashMap; - -pub type ModuleParams = HashMap>; - -pub trait Module: Predict> { - fn get_param(&self, name: &str) -> Option<&Array2> { - self.parameters().get(name) - } - - fn name(&self) -> &str; - - fn parameters(&self) -> &HashMap>; - - fn parameters_mut(&mut self) -> &mut HashMap>; -} diff --git a/.artifacts/archive/neural/src/exp/models/store.rs b/.artifacts/archive/neural/src/exp/models/store.rs deleted file mode 100644 index 6fa95aa3..00000000 --- a/.artifacts/archive/neural/src/exp/models/store.rs +++ /dev/null @@ -1,133 +0,0 @@ -/* - Appellation: stack - Contrib: FL03 -*/ -use crate::prelude::{LayerParams, LayerPosition, LayerShape}; -use ndarray::prelude::{Dimension, Ix2}; -use ndarray::IntoDimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -// use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -// use std::ops; - -pub struct ModelStore { - store: HashMap>, -} - -impl ModelStore -where - T: Clone + Default, -{ - pub fn new() -> Self { - Self { - store: HashMap::new(), - } - } - - pub fn with_capacity(cap: usize) -> Self { - Self { - store: HashMap::with_capacity(cap), - } - } - - pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self { - // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); - let tmp = Vec::from_iter(shapes.into_iter().map(|(i, o)| LayerShape::new(i, o))); - for (i, features) in tmp.iter().enumerate() { - let position = if i == 0 { - LayerPosition::input() - } else if i == tmp.len() - 1 { - LayerPosition::output(i) - } else { - LayerPosition::hidden(i) - }; - self.store.insert(position, LayerParams::new(*features)); - } - self - } - - pub fn with_shapes(shapes: impl IntoIterator) -> Self - where - Sh: IntoDimension, - { - let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); - let mut store = HashMap::new(); - for (i, (inputs, outputs)) in tmp.iter().map(|s| s.into_pattern()).enumerate() { - let features = LayerShape::new(inputs, outputs); - let position = if i == 0 { - LayerPosition::input() - } else if i == tmp.len() - 1 { - LayerPosition::output(i) - } else { - LayerPosition::hidden(i) - }; - store.insert(position, LayerParams::new(features)); - } - Self { store } - } -} - -impl ModelStore -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - self.store.iter_mut().for_each(|(_, l)| { - *l = l.clone().init(biased); - }); - self - } - - pub fn init_bias(mut self) -> Self { - self.store - .iter_mut() - .for_each(|(_, l)| *l = l.clone().init_bias()); - self - } - - pub fn init_weight(mut self) -> Self { - self.store - .iter_mut() - .for_each(|(_, l)| *l = l.clone().init_weight()); - self - } -} - -impl IntoIterator for ModelStore -where - T: Float, -{ - type Item = (LayerPosition, LayerParams); - type IntoIter = std::collections::hash_map::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.store.into_iter() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_model_store() { - let (inputs, outputs) = (5, 3); - - let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - - let params = ModelStore::::new().build_layers(shapes).init(true); - - // validate the dimensions of the model params - // assert!(params.validate_shapes()); - - for (pos, layer) in params.into_iter() { - let shape = shapes[pos.index()]; - let features = LayerShape::new(shape.0, shape.1); - assert_eq!(layer.features(), &features); - } - } -} diff --git a/.artifacts/archive/neural/src/func/activate/activator.rs b/.artifacts/archive/neural/src/func/activate/activator.rs deleted file mode 100644 index 2a4f044f..00000000 --- a/.artifacts/archive/neural/src/func/activate/activator.rs +++ /dev/null @@ -1,162 +0,0 @@ -/* - Appellation: activator - Contrib: FL03 -*/ -use super::{Activate, Gradient}; -use ndarray::prelude::{Array, Dimension, Ix2}; -use serde::{Deserialize, Serialize}; - -pub struct Activator -where - D: Dimension, -{ - method: Box>, -} - -impl Activator -where - D: Dimension, -{ - pub fn new(method: Box>) -> Self { - Self { method } - } - - pub fn from_method(method: impl Activate + 'static) -> Self { - Self::new(Box::new(method)) - } - - pub fn boxed(&self) -> &Box> { - &self.method - } - - pub fn method(&self) -> &dyn Activate { - self.method.as_ref() - } -} - -impl Activator -where - D: Dimension, - T: Clone, -{ - pub fn linear() -> Self { - Self::new(Box::new(super::LinearActivation::new())) - } -} - -impl Activate for Activator -where - D: Dimension, - dyn Activate: Activate, -{ - fn activate(&self, args: &Array) -> Array { - self.method.activate(args) - } -} - -impl Gradient for Activator -where - D: Dimension, - dyn Activate: Gradient, -{ - fn gradient(&self, args: &Array) -> Array { - self.method.gradient(args) - } -} - -impl Clone for Activator -where - D: Dimension, - T: Clone, - dyn Activate: Clone, -{ - fn clone(&self) -> Self { - Self::new(self.method.clone()) - } -} - -impl Default for Activator -where - D: Dimension, - dyn Activate: Default, -{ - fn default() -> Self { - Self::new(Box::new(>::default())) - } -} - -// impl Copy for A where D: Dimension, T: Copy, dyn Activate: Copy {} - -impl PartialEq for Activator -where - D: Dimension, - dyn Activate: PartialEq, -{ - fn eq(&self, other: &Self) -> bool { - self.method.eq(&other.method) - } -} - -impl Eq for Activator -where - D: Dimension, - dyn Activate: Eq, -{ -} - -impl std::fmt::Debug for Activator -where - D: Dimension, - dyn Activate: std::fmt::Debug, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.method.fmt(f) - } -} - -impl std::fmt::Display for Activator -where - D: Dimension, - dyn Activate: std::fmt::Display, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.method.fmt(f) - } -} - -impl Serialize for Activator -where - D: Dimension, - dyn Activate: Serialize, -{ - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.method.serialize(serializer) - } -} - -impl<'de, T, D> Deserialize<'de> for Activator -where - D: Dimension, - dyn Activate: Deserialize<'de>, -{ - fn deserialize(deserializer: D2) -> Result - where - D2: serde::Deserializer<'de>, - { - Ok(Self::new(Box::new(>::deserialize( - deserializer, - )?))) - } -} - -impl From>> for Activator -where - D: Dimension, -{ - fn from(method: Box>) -> Self { - Self::new(method) - } -} diff --git a/.artifacts/archive/neural/src/func/activate/binary.rs b/.artifacts/archive/neural/src/func/activate/binary.rs deleted file mode 100644 index 2b7aa7ef..00000000 --- a/.artifacts/archive/neural/src/func/activate/binary.rs +++ /dev/null @@ -1,83 +0,0 @@ -/* - Appellation: binary - Contrib: FL03 -*/ -use ndarray::prelude::{Array, Dimension}; -use num::{One, Zero}; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Heavyside; - -impl Heavyside { - pub fn new() -> Self { - Self - } - - pub fn heavyside(args: &T) -> T - where - T: One + PartialOrd + Zero, - { - if args > &T::zero() { - T::one() - } else { - T::zero() - } - } -} - -// impl Activate for Heavyside -// where -// D: Dimension, -// T: Clone + One + PartialOrd + Zero, -// { -// fn activate(&self, args: &Array) -> Array { -// args.mapv(|x| Self::heavyside(&x)) -// } -// } - -// impl FnOnce<(&Array,)> for Heavyside -// where -// D: Dimension, -// T: Clone + One + PartialOrd + Zero, -// { -// type Output = Array; - -// extern "rust-call" fn call_once(self, args: (&Array,)) -> Array { -// args.mapv(|x| Self::heavyside(&x)) -// } -// } - -impl Fn<(&Array,)> for Heavyside -where - D: Dimension, - T: One + PartialOrd + Zero, -{ - extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { - args.0.map(Heavyside::heavyside) - } -} - -impl FnMut<(&Array,)> for Heavyside -where - D: Dimension, - T: One + PartialOrd + Zero, -{ - extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { - args.0.map(Heavyside::heavyside) - } -} - -impl FnOnce<(&Array,)> for Heavyside -where - D: Dimension, - T: One + PartialOrd + Zero, -{ - type Output = Array; - - extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { - args.0.map(Heavyside::heavyside) - } -} diff --git a/.artifacts/archive/neural/src/func/activate/linear.rs b/.artifacts/archive/neural/src/func/activate/linear.rs deleted file mode 100644 index 0d6608e2..00000000 --- a/.artifacts/archive/neural/src/func/activate/linear.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - Appellation: linear - Contrib: FL03 -*/ -use super::Gradient; -use ndarray::prelude::{Array, Dimension}; -use num::One; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct LinearActivation; - -impl LinearActivation { - pub fn new() -> Self { - Self::default() - } - - pub fn derivative(_x: T) -> T - where - T: One, - { - T::one() - } - - pub fn gradient(args: &Array) -> Array - where - D: Dimension, - T: Clone + One, - { - args.mapv(|x| Self::derivative(x)) - } - - pub fn linear(args: &T) -> T { - LinearActivation::method()(args) - } - - pub fn method() -> fn(&T) -> T { - |x| x.clone() - } - - pub fn rho(args: T) -> T { - args - } -} - -impl Gradient for LinearActivation -where - D: Dimension, - T: Clone + One, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(x)) - } -} - -impl Fn<(&T,)> for LinearActivation -where - T: Clone, -{ - extern "rust-call" fn call(&self, args: (&T,)) -> T { - args.0.clone() - } -} - -impl FnMut<(&T,)> for LinearActivation -where - T: Clone, -{ - extern "rust-call" fn call_mut(&mut self, args: (&T,)) -> T { - args.0.clone() - } -} - -impl FnOnce<(&T,)> for LinearActivation -where - T: Clone, -{ - type Output = T; - - extern "rust-call" fn call_once(self, args: (&T,)) -> Self::Output { - args.0.clone() - } -} diff --git a/.artifacts/archive/neural/src/func/activate/mod.rs b/.artifacts/archive/neural/src/func/activate/mod.rs deleted file mode 100644 index 729b9357..00000000 --- a/.artifacts/archive/neural/src/func/activate/mod.rs +++ /dev/null @@ -1,146 +0,0 @@ -/* - Appellation: activate - Contrib: FL03 -*/ -//! # activate -//! -//! This module contains the activation functions for the neurons. -pub use self::{activator::*, binary::*, linear::*, nl::*, utils::*}; - -pub(crate) mod activator; -pub(crate) mod binary; -pub(crate) mod linear; -pub(crate) mod nl; - -// use crate::core::prelude::ShapeResult; -use ndarray::prelude::{Array, Dimension, Ix2, IxDyn}; - -pub type ActivationFn = Box) -> Array>; - -pub type ActivateDyn = Box>; - -pub trait Rho { - fn rho(&self, args: &T) -> T; -} - -impl Rho for F -where - F: Fn(&T) -> T, -{ - fn rho(&self, args: &T) -> T { - self.call((args,)) - } -} - -pub trait Activate -where - D: Dimension, -{ - fn activate(&self, args: &Array) -> Array; -} - -pub trait ActivateExt: Activate + Clone + 'static -where - D: Dimension, -{ - fn boxed(&self) -> ActivateDyn { - Box::new(self.clone()) - } -} - -impl ActivateExt for F -where - D: Dimension, - F: Activate + Clone + 'static, -{ -} - -impl Activate for F -where - D: Dimension, - F: Fn(&Array) -> Array, -{ - fn activate(&self, args: &Array) -> Array { - self.call((args,)) - } -} - -impl Activate for Box> -where - D: Dimension, -{ - fn activate(&self, args: &Array) -> Array { - self.as_ref().activate(args) - } -} - -pub trait Gradient -where - D: Dimension, -{ - fn gradient(&self, args: &Array) -> Array; -} - -// impl Objective for fn(&Array) -> Array -// where -// D: Dimension, -// { -// fn gradient(&self, args: &Array) -> Array { -// self.call((args,)) -// } -// } - -impl Gradient for Box> -where - D: Dimension, -{ - fn gradient(&self, args: &Array) -> Array { - self.as_ref().gradient(args) - } -} - -pub(crate) mod utils { - use num::{One, Zero}; - - pub fn linear_activation(args: &T) -> T - where - T: Clone, - { - args.clone() - } - - pub fn heavyside(args: &T) -> T - where - T: One + PartialOrd + Zero, - { - if args > &T::zero() { - T::one() - } else { - T::zero() - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ndarray::array; - - #[test] - fn test_heavyside() { - let exp = array![0.0, 0.0, 1.0]; - let args = array![-1.0, 0.0, 1.0]; - - assert_eq!(Heavyside::new().activate(&args), exp); - assert_eq!(Heavyside(&args), exp); - } - - #[test] - fn test_linear() { - let exp = array![0.0, 1.0, 2.0]; - let args = array![0.0, 1.0, 2.0]; - - assert_eq!(LinearActivation::new().activate(&args), exp); - assert_eq!(LinearActivation(&args), exp); - } -} diff --git a/.artifacts/archive/neural/src/func/activate/nl/mod.rs b/.artifacts/archive/neural/src/func/activate/nl/mod.rs deleted file mode 100644 index 6abf8dab..00000000 --- a/.artifacts/archive/neural/src/func/activate/nl/mod.rs +++ /dev/null @@ -1,119 +0,0 @@ -/* - Appellation: nonlinear - Contrib: FL03 -*/ -//! # NonLinear Activation Functions -//! -//! -pub use self::{relu::*, sigmoid::*, softmax::*, tanh::*, utils::*}; - -pub(crate) mod relu; -pub(crate) mod sigmoid; -pub(crate) mod softmax; -pub(crate) mod tanh; - -pub(crate) mod utils { - use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; - use ndarray::RemoveAxis; - use num::{Float, Zero}; - - pub fn relu(args: &T) -> T - where - T: Clone + PartialOrd + Zero, - { - if args > &T::zero() { - args.clone() - } else { - T::zero() - } - } - - pub fn sigmoid(args: &T) -> T - where - T: Float, - { - T::one() / (T::one() + (-args.clone()).exp()) - } - - pub fn softmax(args: &Array) -> Array - where - D: Dimension, - T: Float, - { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) - } - - pub fn softmax_axis(args: &Array, axis: Option) -> Array - where - D: Dimension + RemoveAxis, - T: NdFloat, - { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } - - pub fn tanh(args: &T) -> T - where - T: Float, - { - args.tanh() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::RoundTo; - use crate::prelude::Activate; - use ndarray::array; - - #[test] - fn test_relu() { - let exp = array![0.0, 0.0, 3.0]; - let args = array![-1.0, 0.0, 3.0]; - - let res = ReLU::new().activate(&args); - assert_eq!(res, exp); - assert_eq!(ReLU(&args), exp); - } - - #[test] - fn test_sigmoid() { - let exp = array![0.73105858, 0.88079708, 0.95257413]; - let args = array![1.0, 2.0, 3.0]; - - let res = Sigmoid::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - let res = Sigmoid(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_softmax() { - let exp = array![0.09003057, 0.24472847, 0.66524096]; - let args = array![1.0, 2.0, 3.0]; - - let res = Softmax::new(None).activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - let res = Softmax::new(None)(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } - - #[test] - fn test_tanh() { - let exp = array![0.76159416, 0.96402758, 0.99505475]; - let args = array![1.0, 2.0, 3.0]; - - let res = Tanh::new().activate(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - let res = Tanh(&args).mapv(|i| i.round_to(8)); - assert_eq!(res, exp); - } -} diff --git a/.artifacts/archive/neural/src/func/activate/nl/relu.rs b/.artifacts/archive/neural/src/func/activate/nl/relu.rs deleted file mode 100644 index 42b6fdc3..00000000 --- a/.artifacts/archive/neural/src/func/activate/nl/relu.rs +++ /dev/null @@ -1,93 +0,0 @@ -/* - Appellation: relu - Contrib: FL03 -*/ -use crate::func::activate::Gradient; -use ndarray::prelude::{Array, Dimension}; -use num::{One, Zero}; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct ReLU; - -impl ReLU { - pub fn new() -> Self { - Self - } - - pub fn derivative(args: &T) -> T - where - T: One + PartialOrd + Zero, - { - if args > &T::zero() { - T::one() - } else { - T::zero() - } - } - - pub fn relu(args: &T) -> T - where - T: Clone + PartialOrd + Zero, - { - if args > &T::zero() { - args.clone() - } else { - T::zero() - } - } -} - -// impl Activate for ReLU -// where -// D: Dimension, -// T: Clone + PartialOrd + Zero, -// { -// fn activate(&self, x: &Array) -> Array { -// x.map(Self::relu) -// } -// } - -impl Gradient for ReLU -where - D: Dimension, - T: Clone + One + PartialOrd + Zero, -{ - fn gradient(&self, args: &Array) -> Array { - args.map(Self::derivative) - } -} - -impl Fn<(&Array,)> for ReLU -where - D: Dimension, - T: Clone + PartialOrd + Zero, -{ - extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { - args.0.map(Self::relu) - } -} - -impl FnMut<(&Array,)> for ReLU -where - D: Dimension, - T: Clone + PartialOrd + Zero, -{ - extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { - args.0.map(Self::relu) - } -} - -impl FnOnce<(&Array,)> for ReLU -where - D: Dimension, - T: Clone + PartialOrd + Zero, -{ - type Output = Array; - - extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { - args.0.map(Self::relu) - } -} diff --git a/.artifacts/archive/neural/src/func/activate/nl/sigmoid.rs b/.artifacts/archive/neural/src/func/activate/nl/sigmoid.rs deleted file mode 100644 index 142547fa..00000000 --- a/.artifacts/archive/neural/src/func/activate/nl/sigmoid.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - Appellation: sigmoid - Contrib: FL03 -*/ -use crate::func::activate::Gradient; -use ndarray::prelude::{Array, Dimension}; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Sigmoid; - -impl Sigmoid { - pub fn new() -> Self { - Self - } - - pub fn derivative(x: T) -> T - where - T: Float, - { - (-x).exp() / (T::one() + (-x).exp()).powi(2) - } - - pub fn sigmoid(x: T) -> T - where - T: Float, - { - T::one() / (T::one() + (-x).exp()) - } -} - -// impl Activate for Sigmoid -// where -// D: Dimension, -// T: Float, -// { -// fn activate(&self, x: &Array) -> Array { -// x.mapv(|x| Self::sigmoid(x)) -// } -// } - -impl Gradient for Sigmoid -where - D: Dimension, - T: Float, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(x)) - } -} - -impl Fn<(&Array,)> for Sigmoid -where - D: Dimension, - T: Float, -{ - extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { - args.0.mapv(|x| Self::sigmoid(x)) - } -} - -impl FnMut<(&Array,)> for Sigmoid -where - D: Dimension, - T: Float, -{ - extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { - args.0.mapv(|x| Self::sigmoid(x)) - } -} - -impl FnOnce<(&Array,)> for Sigmoid -where - D: Dimension, - T: Float, -{ - type Output = Array; - - extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { - args.0.mapv(|x| Self::sigmoid(x)) - } -} diff --git a/.artifacts/archive/neural/src/func/activate/nl/softmax.rs b/.artifacts/archive/neural/src/func/activate/nl/softmax.rs deleted file mode 100644 index 519ab654..00000000 --- a/.artifacts/archive/neural/src/func/activate/nl/softmax.rs +++ /dev/null @@ -1,137 +0,0 @@ -/* - Appellation: softmax - Contrib: FL03 -*/ -use crate::func::activate::Gradient; -use ndarray::prelude::{Array, Axis, Dimension, NdFloat}; -use ndarray::RemoveAxis; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Softmax { - axis: Option, -} - -impl Softmax { - pub fn new(axis: Option) -> Self { - Self { axis } - } - - pub fn axis(&self) -> Option { - self.axis - } - - pub fn softmax(args: Array) -> Array - where - D: Dimension, - T: Float, - { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) - } - - pub fn softmax_axis(&self, args: Array) -> Array - where - T: NdFloat, - D: Dimension + RemoveAxis, - { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -// impl Activate for Softmax -// where -// D: Dimension + RemoveAxis, -// T: NdFloat, -// { -// fn activate(&self, x: &Array) -> Array { -// let exp = x.mapv(|x| x.exp()); -// if let Some(axis) = self.axis { -// let denom = exp.sum_axis(Axis(axis)); -// exp / denom -// } else { -// let denom = exp.sum(); -// exp / denom -// } -// } -// } - -impl Gradient for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - fn gradient(&self, args: &Array) -> Array { - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -impl Fn<(&Array,)> for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { - let exp = args.0.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -impl FnMut<(&Array,)> for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { - let exp = args.0.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} - -impl FnOnce<(&Array,)> for Softmax -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - type Output = Array; - - extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { - let exp = args.0.mapv(|x| x.exp()); - if let Some(axis) = self.axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } - } -} diff --git a/.artifacts/archive/neural/src/func/activate/nl/tanh.rs b/.artifacts/archive/neural/src/func/activate/nl/tanh.rs deleted file mode 100644 index 2757fc2e..00000000 --- a/.artifacts/archive/neural/src/func/activate/nl/tanh.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - Appellation: tanh - Contrib: FL03 -*/ -use crate::func::activate::Gradient; -use ndarray::prelude::{Array, Dimension}; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Tanh; - -impl Tanh { - pub fn new() -> Self { - Self - } - - pub fn derivative(args: &T) -> T - where - T: Float, - { - T::one() - args.tanh().powi(2) - } - - pub fn tanh(args: &T) -> T - where - T: Float, - { - args.tanh() - } -} - -// impl Activate for Tanh -// where -// D: Dimension, -// T: Float, -// { -// fn activate(&self, x: &Array) -> Array { -// x.mapv(Float::tanh) -// } -// } - -impl Gradient for Tanh -where - D: Dimension, - T: Float, -{ - fn gradient(&self, args: &Array) -> Array { - args.mapv(|x| Self::derivative(&x)) - } -} - -impl Fn<(&Array,)> for Tanh -where - D: Dimension, - T: Float, -{ - extern "rust-call" fn call(&self, args: (&Array,)) -> Self::Output { - args.0.mapv(T::tanh) - } -} - -impl FnMut<(&Array,)> for Tanh -where - D: Dimension, - T: Float, -{ - extern "rust-call" fn call_mut(&mut self, args: (&Array,)) -> Self::Output { - args.0.mapv(T::tanh) - } -} - -impl FnOnce<(&Array,)> for Tanh -where - D: Dimension, - T: Float, -{ - type Output = Array; - - extern "rust-call" fn call_once(self, args: (&Array,)) -> Self::Output { - args.0.mapv(T::tanh) - } -} diff --git a/.artifacts/archive/neural/src/func/loss/kinds.rs b/.artifacts/archive/neural/src/func/loss/kinds.rs deleted file mode 100644 index 92bddbc8..00000000 --- a/.artifacts/archive/neural/src/func/loss/kinds.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum LossKind { - Classification, - #[default] - Regression, -} - -impl LossKind { - pub fn classification() -> Self { - Self::Classification - } - - pub fn regression() -> Self { - Self::Regression - } -} diff --git a/.artifacts/archive/neural/src/func/loss/mod.rs b/.artifacts/archive/neural/src/func/loss/mod.rs deleted file mode 100644 index a8c79cc2..00000000 --- a/.artifacts/archive/neural/src/func/loss/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - Appellation: loss - Contrib: FL03 -*/ -//! # Loss Functions -//! -//! Loss functions consider the differences between predicted and target outputs. -//! Overall, neural network models aim to minimize the average loss by adjusting certain hyperparameters, -//! the weights and biases. - -pub use self::kinds::*; - -pub(crate) mod kinds; - -pub mod reg; - -pub trait Loss { - type Output; - - fn loss(&self, pred: &T, target: &T) -> Self::Output; -} - -pub trait LossWith { - type Output; - - fn loss(&self, other: &Self) -> Self::Output; -} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/func/loss/reg/huber.rs b/.artifacts/archive/neural/src/func/loss/reg/huber.rs deleted file mode 100644 index a2abf340..00000000 --- a/.artifacts/archive/neural/src/func/loss/reg/huber.rs +++ /dev/null @@ -1,75 +0,0 @@ -/* - Appellation: regress - Contrib: FL03 -*/ -use crate::func::loss::Loss; -use ndarray::prelude::{Array, Dimension, NdFloat}; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct HuberLoss -where - T: Float, -{ - delta: T, -} - -impl HuberLoss -where - T: Float, -{ - pub fn new(delta: T) -> Self { - Self { delta } - } - - pub fn delta(&self) -> T { - self.delta - } - - pub fn delta_mut(&mut self) -> &mut T { - &mut self.delta - } - - pub fn set_delta(&mut self, delta: T) { - self.delta = delta; - } - - pub fn with_delta(mut self, delta: T) -> Self { - self.delta = delta; - self - } -} - -impl Loss> for HuberLoss -where - D: Dimension, - T: NdFloat, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - let half = T::from(0.5).unwrap(); - let mut loss = T::zero(); - for (x, y) in pred.iter().cloned().zip(target.iter().cloned()) { - let diff = x - y; - if diff.abs() <= self.delta() { - // If the difference is sufficiently small, use the squared error. - loss += half * diff.powi(2); - } else { - // Otherwise, use a variant of the absolute error. - loss += self.delta * (diff.abs() - half * self.delta()); - } - } - loss / T::from(pred.len()).unwrap() - } -} - -impl From for HuberLoss -where - T: Float, -{ - fn from(delta: T) -> Self { - Self::new(delta) - } -} diff --git a/.artifacts/archive/neural/src/func/loss/reg/mae.rs b/.artifacts/archive/neural/src/func/loss/reg/mae.rs deleted file mode 100644 index 4d34c182..00000000 --- a/.artifacts/archive/neural/src/func/loss/reg/mae.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - Appellation: mae - Contrib: FL03 -*/ -use crate::func::loss::Loss; -use ndarray::prelude::{Array, Dimension, NdFloat}; -use num::FromPrimitive; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct MeanAbsoluteError; - -impl MeanAbsoluteError { - pub fn new() -> Self { - Self - } -} - -impl Loss> for MeanAbsoluteError -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - (pred - target).mapv(T::abs).mean().unwrap() - } -} diff --git a/.artifacts/archive/neural/src/func/loss/reg/mod.rs b/.artifacts/archive/neural/src/func/loss/reg/mod.rs deleted file mode 100644 index f196fb9d..00000000 --- a/.artifacts/archive/neural/src/func/loss/reg/mod.rs +++ /dev/null @@ -1,112 +0,0 @@ -/* - Appellation: reg - Contrib: FL03 -*/ -pub use self::{huber::*, mae::*, mse::*, utils::*}; - -pub(crate) mod huber; -pub(crate) mod mae; -pub(crate) mod mse; - -use crate::func::loss::Loss; -use ndarray::prelude::{Array, Dimension, NdFloat}; -use num::FromPrimitive; - -pub enum RegressiveLoss { - Huber(HuberLoss), - MeanAbsoluteError, - MeanSquaredError, - Other(String), -} - -pub trait RegressiveLosses { - fn huber(&self, delta: T, other: &Self) -> T; - fn mae(&self, other: &Self) -> T; - fn mse(&self, other: &Self) -> T; -} - -impl RegressiveLosses for Array -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - fn huber(&self, delta: T, other: &Self) -> T { - HuberLoss::new(delta).loss(self, other) - } - - fn mae(&self, other: &Self) -> T { - MeanAbsoluteError::new().loss(self, other) - } - - fn mse(&self, other: &Self) -> T { - MeanSquaredError::new().loss(self, other) - } -} - -pub(crate) mod utils { - use ndarray::prelude::{Array, Dimension, NdFloat}; - use num::FromPrimitive; - - pub fn mae(pred: &Array, target: &Array) -> Option - where - D: Dimension, - T: FromPrimitive + NdFloat, - { - (pred - target).mapv(T::abs).mean() - } - - pub fn mse(pred: &Array, target: &Array) -> Option - where - D: Dimension, - T: FromPrimitive + NdFloat, - { - (pred - target).mapv(|x| x.powi(2)).mean() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::{GenerateRandom, RoundTo}; - use crate::func::loss::Loss; - use ndarray::prelude::{Array, Ix2}; - use ndarray_stats::DeviationExt; - - #[test] - fn test_mae() { - let (m, n) = (3, 3); - let shape = (m, n); - - let ns = m * n; - - let targets: Array = Array::linspace(0.0, ns as f64, ns) - .into_shape(shape) - .unwrap(); - let pred = Array::::uniform_between(3.0, shape); - - let loss = MeanAbsoluteError::new().loss(&pred, &targets).round_to(4); - - let exp = targets.mean_abs_err(&pred).unwrap().round_to(4); - - assert_eq!(&loss, &exp); - } - - #[test] - fn test_mse() { - let (m, n) = (3, 3); - let shape = (m, n); - - let ns = m * n; - - let targets: Array = Array::linspace(0.0, ns as f64, ns) - .into_shape(shape) - .unwrap(); - let pred = Array::::uniform_between(3.0, shape); - - let loss = MeanSquaredError::new().loss(&pred, &targets).round_to(4); - - let exp = targets.mean_sq_err(&pred).unwrap().round_to(4); - - assert_eq!(&loss, &exp); - } -} diff --git a/.artifacts/archive/neural/src/func/loss/reg/mse.rs b/.artifacts/archive/neural/src/func/loss/reg/mse.rs deleted file mode 100644 index 95007d8d..00000000 --- a/.artifacts/archive/neural/src/func/loss/reg/mse.rs +++ /dev/null @@ -1,79 +0,0 @@ -/* - Appellation: mse - Contrib: FL03 -*/ -use crate::func::loss::Loss; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; -use num::FromPrimitive; -use serde::{Deserialize, Serialize}; -use std::ops; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct MeanSquaredError; - -impl MeanSquaredError { - pub fn new() -> Self { - Self - } - - pub fn mse(&self, pred: &Array, target: &Array) -> T - where - D: Dimension, - T: FromPrimitive + NdFloat, - { - (pred - target).mapv(|x| x.powi(2)).mean().unwrap() - } - - pub fn wg( - data: &Array2, - target: &Array, - bias: &Array, - weights: &Array, - ) -> T - where - D: Dimension, - T: FromPrimitive + NdFloat, - Array2: Dot, Output = Array>, - { - let pred = data.dot(&weights.t().to_owned()) + bias.clone(); - let error = target - &pred; - let dw = data.t().to_owned().dot(&error) * (-T::from(2).unwrap()); - dw.mean().unwrap() - } - - pub fn partial( - data: &Array, - bias: &Array1, - slope: &Array, - target: &Array1, - ) -> (T, T) - where - D: Dimension, - T: FromPrimitive + NdFloat, - Array: Dot>, - as Dot>>::Output: ops::Add, Output = Array> - + ops::Sub, Output = Array> - + ops::Mul>, - Array1: ops::Sub, Output = Array>, - { - let predicted = data.dot(&slope.t().to_owned()) + bias.clone(); - let w = data.dot(&(target.clone() - predicted.clone())) * (-T::from(2).unwrap()); - let b = (target.clone() - predicted) * (-T::from(2).unwrap()); - (w.mean().unwrap(), b.mean().unwrap()) - } -} - -impl Loss> for MeanSquaredError -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - type Output = T; - - fn loss(&self, pred: &Array, target: &Array) -> Self::Output { - (pred - target).mapv(|x| x.powi(2)).mean().unwrap() - } -} diff --git a/.artifacts/archive/neural/src/func/mod.rs b/.artifacts/archive/neural/src/func/mod.rs deleted file mode 100644 index a29197e9..00000000 --- a/.artifacts/archive/neural/src/func/mod.rs +++ /dev/null @@ -1,56 +0,0 @@ -/* - Appellation: func - Contrib: FL03 -*/ -//! # Functional -//! -//! This module implements several functional aspects of the neural network. -//! -//! ## Activation -//! -//! The activation functions are implemented as structs that implement the `Fn` trait. -//! -//! ## Loss -//! -//! The loss functions are implemented as structs that implement the `Fn` trait. -pub use self::{rms::*, utils::*}; - -pub mod activate; -pub mod loss; - -pub(crate) mod rms; - -pub(crate) mod utils { - use ndarray::linalg::Dot; - use ndarray::prelude::{Array, Dimension}; - use num::Float; - use std::ops; - - pub fn lin( - args: &Array, - weights: &Array, - bias: &Array, - ) -> Array - where - A: Dimension, - D: Dimension, - O: Dimension, - T: Float, - Array: Dot, Output = Array>, - Array: ops::Add, Output = Array>, - { - args.dot(weights) + bias.clone() - } - - pub fn slope_intercept(args: T, slope: T, intercept: T) -> T - where - T: ops::Add + ops::Mul, - { - args * slope + intercept - } -} - -#[cfg(test)] -mod tests { - // use super::*; -} diff --git a/.artifacts/archive/neural/src/func/rms.rs b/.artifacts/archive/neural/src/func/rms.rs deleted file mode 100644 index f42dc3c7..00000000 --- a/.artifacts/archive/neural/src/func/rms.rs +++ /dev/null @@ -1,52 +0,0 @@ -/* - Appellation: rms - Contrib: FL03 -*/ -use ndarray::prelude::{Array, Dimension}; -use num::{Float, FromPrimitive}; - -pub fn rms(args: &Array) -> Option -where - D: Dimension, - T: Float + FromPrimitive, -{ - if let Some(avg) = args.mapv(|xs| xs.powi(2)).mean() { - return Some(avg.sqrt()); - } - None -} - -pub trait RootMeanSquare { - fn rms(&self) -> T; -} - -// impl RootMeanSquare for S where S: ExactSizeIterator, T: Float { -// fn rms(&self) -> T { -// let sum = self.iter().fold(T::zero(), |acc, x| acc + x.powi(2)); -// (sum / T::from(self.len()).unwrap()).sqrt() -// } -// } - -impl RootMeanSquare for Array -where - D: Dimension, - T: Float + FromPrimitive, -{ - fn rms(&self) -> T { - (self.mapv(|xs| xs.powi(2)).mean().unwrap_or_else(T::zero)).sqrt() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::{linarr, RoundTo}; - use ndarray::prelude::Ix1; - - #[test] - fn test_rms() { - let v = linarr::(5).expect("Failed to create array"); - let rms = v.rms().round_to(5); - assert_eq!(rms, 3.31662); - } -} diff --git a/.artifacts/archive/neural/src/layers/cmp/features.rs b/.artifacts/archive/neural/src/layers/cmp/features.rs deleted file mode 100644 index 742dac1c..00000000 --- a/.artifacts/archive/neural/src/layers/cmp/features.rs +++ /dev/null @@ -1,105 +0,0 @@ -/* - Appellation: features - Contrib: FL03 -*/ -use super::Features; -use ndarray::prelude::{Dimension, Ix2}; -use ndarray::IntoDimension; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct LayerShape { - pub inputs: usize, - pub outputs: usize, -} - -impl LayerShape { - pub fn new(inputs: usize, outputs: usize) -> Self { - Self { inputs, outputs } - } - - pub fn from_dimension(shape: impl IntoDimension) -> Self { - let dim = shape.into_dimension(); - let (outputs, inputs) = dim.into_pattern(); - Self::new(inputs, outputs) - } - - pub fn neuron(inputs: usize) -> Self { - Self::new(inputs, 1) - } - - pub fn uniform_scale(&self) -> T { - (T::one() / T::from(self.inputs()).unwrap()).sqrt() - } -} - -impl std::fmt::Display for LayerShape { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "({}, {})", self.inputs, self.outputs) - } -} - -impl Features for LayerShape { - fn inputs(&self) -> usize { - self.inputs - } - - fn outputs(&self) -> usize { - self.outputs - } -} - -impl IntoDimension for LayerShape { - type Dim = Ix2; - - fn into_dimension(self) -> Self::Dim { - ndarray::Ix2(self.outputs, self.inputs) - } -} - -impl From for Ix2 { - fn from(features: LayerShape) -> Self { - features.into_dimension() - } -} - -impl From for ndarray::IxDyn { - fn from(features: LayerShape) -> Self { - ndarray::IxDyn(&[features.outputs, features.inputs]) - } -} - -impl From for [usize; 2] { - fn from(features: LayerShape) -> Self { - [features.outputs, features.inputs] - } -} - -impl From<[usize; 2]> for LayerShape { - fn from(features: [usize; 2]) -> Self { - Self { - inputs: features[1], - outputs: features[0], - } - } -} - -impl From for (usize, usize) { - fn from(features: LayerShape) -> Self { - (features.outputs, features.inputs) - } -} - -impl From<(usize, usize)> for LayerShape { - fn from((inputs, outputs): (usize, usize)) -> Self { - Self { inputs, outputs } - } -} - -impl From for LayerShape { - fn from(inputs: usize) -> Self { - Self { inputs, outputs: 1 } - } -} diff --git a/.artifacts/archive/neural/src/layers/cmp/kinds.rs b/.artifacts/archive/neural/src/layers/cmp/kinds.rs deleted file mode 100644 index 9fa587bc..00000000 --- a/.artifacts/archive/neural/src/layers/cmp/kinds.rs +++ /dev/null @@ -1,140 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum LayerKind { - #[default] - Input = 0, - Hidden, - Output, -} - -impl LayerKind { - pub fn input() -> Self { - Self::Input - } - - pub fn hidden() -> Self { - Self::Hidden - } - - pub fn output() -> Self { - Self::Output - } - - pub fn create_kind(idx: usize, layers: usize) -> Self { - if idx == 0 { - Self::Input - } else if idx == layers - 1 { - Self::Output - } else { - Self::Hidden - } - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct LayerPosition { - pub idx: usize, - pub kind: LayerKind, -} - -impl LayerPosition { - pub fn new(idx: usize, kind: LayerKind) -> Self { - Self { idx, kind } - } - - pub fn input() -> Self { - Self::new(0, LayerKind::Input) - } - - pub fn hidden(idx: usize) -> Self { - Self::new(idx, LayerKind::Hidden) - } - - pub fn output(idx: usize) -> Self { - Self::new(idx, LayerKind::Output) - } - - pub fn is_input(&self) -> bool { - self.kind().is_input() - } - - pub fn is_hidden(&self) -> bool { - self.kind().is_hidden() - } - - pub fn is_output(&self) -> bool { - self.kind().is_output() - } - - pub fn index(&self) -> usize { - self.idx - } - - pub fn kind(&self) -> &LayerKind { - &self.kind - } -} - -impl AsRef for LayerPosition { - fn as_ref(&self) -> &usize { - &self.idx - } -} - -impl AsRef for LayerPosition { - fn as_ref(&self) -> &LayerKind { - &self.kind - } -} - -impl AsMut for LayerPosition { - fn as_mut(&mut self) -> &mut usize { - &mut self.idx - } -} - -impl AsMut for LayerPosition { - fn as_mut(&mut self) -> &mut LayerKind { - &mut self.kind - } -} - -impl From for usize { - fn from(pos: LayerPosition) -> Self { - pos.idx - } -} - -impl From for LayerKind { - fn from(pos: LayerPosition) -> Self { - pos.kind - } -} diff --git a/.artifacts/archive/neural/src/layers/cmp/mod.rs b/.artifacts/archive/neural/src/layers/cmp/mod.rs deleted file mode 100644 index 3916a626..00000000 --- a/.artifacts/archive/neural/src/layers/cmp/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -/* - Appellation: cmp - Contrib: FL03 -*/ -//! # Layers -pub use self::{features::*, kinds::*}; - -pub(crate) mod features; -pub(crate) mod kinds; - -use ndarray::prelude::{Dimension, Ix2}; -use ndarray::IntoDimension; -use num::Float; - -pub trait Features { - fn inputs(&self) -> usize; - - fn network(&self) -> usize { - self.inputs() * self.outputs() - } - - fn outputs(&self) -> usize; - - fn in_by_out(&self) -> (usize, usize) { - (self.inputs(), self.outputs()) - } - - fn out_by_in(&self) -> (usize, usize) { - (self.outputs(), self.inputs()) - } - - fn input_scale(&self) -> T { - (T::one() / T::from(self.inputs()).unwrap()).sqrt() - } -} - -impl Features for usize { - fn inputs(&self) -> usize { - *self - } - - fn outputs(&self) -> usize { - 1 - } -} - -impl Features for (usize, usize) { - fn inputs(&self) -> usize { - self.1 - } - - fn outputs(&self) -> usize { - self.0 - } -} - -impl Features for [usize; 2] { - fn inputs(&self) -> usize { - self[1] - } - - fn outputs(&self) -> usize { - self[0] - } -} - -pub trait FeaturesExt: Features + IntoDimension -where - D: Dimension, -{ - fn new(inputs: usize, outputs: usize) -> Self; - - fn single(inputs: usize) -> Self - where - Self: Sized, - { - Self::new(inputs, 1) - } -} - -impl FeaturesExt for (usize, usize) { - fn new(inputs: usize, outputs: usize) -> Self { - (outputs, inputs) - } -} - -pub trait FromFeatures { - fn from_features(features: Sh) -> Self; -} - -pub trait IntoFeatures { - fn into_features(self) -> LayerShape; -} - -impl IntoFeatures for S -where - S: Into, -{ - fn into_features(self) -> LayerShape { - self.into() - } -} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/layers/layer.rs b/.artifacts/archive/neural/src/layers/layer.rs deleted file mode 100644 index 14d5c2cb..00000000 --- a/.artifacts/archive/neural/src/layers/layer.rs +++ /dev/null @@ -1,287 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use super::{LayerParams, LayerShape}; -use crate::func::activate::{Activate, Gradient, LinearActivation}; -use crate::prelude::{Features, Forward, Node, Perceptron}; -use ndarray::prelude::{Array2, Ix1, NdFloat}; -use ndarray::LinalgScalar; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use ndarray_stats::DeviationExt; -use num::{Float, Signed}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct Layer -where - A: Activate, -{ - activator: A, - features: LayerShape, - name: String, - params: LayerParams, -} - -impl Layer -where - A: Activate + Default, - T: Default, -{ - pub fn new(activator: A, features: LayerShape, name: impl ToString) -> Self { - Self { - activator, - features, - name: name.to_string(), - params: LayerParams::new(features), - } - } - - pub fn from_features(inputs: usize, outputs: usize) -> Self { - let features = LayerShape::new(inputs, outputs); - Self { - activator: A::default(), - features, - name: String::new(), - params: LayerParams::new(features), - } - } -} - -impl Layer -where - A: Activate, -{ - pub fn activator(&self) -> &A { - &self.activator - } - - pub fn features(&self) -> &LayerShape { - &self.features - } - - pub fn features_mut(&mut self) -> &mut LayerShape { - &mut self.features - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn params(&self) -> &LayerParams { - &self.params - } - - pub fn params_mut(&mut self) -> &mut LayerParams { - &mut self.params - } - - pub fn set_name(&mut self, name: impl ToString) { - self.name = name.to_string(); - } - - pub fn set_node(&mut self, idx: usize, neuron: &Perceptron) - where - A: Activate, - T: Clone + Default, - { - self.params.set_node(idx, neuron.node().clone()); - } - - pub fn validate_layer(&self, other: &Self, next: bool) -> bool { - if next { - return self.features().inputs() == other.features().outputs(); - } - self.features().outputs() == other.features().inputs() - } - - pub fn with_name(mut self, name: impl ToString) -> Self { - self.name = name.to_string(); - self - } -} - -impl Layer -where - A: Activate + Clone + 'static, - T: Clone, -{ - pub fn as_dyn(&self) -> Layer>> { - Layer { - activator: Box::new(self.activator.clone()), - features: self.features.clone(), - name: self.name.clone(), - params: self.params.clone(), - } - } -} - -impl Layer -where - A: Activate, - T: LinalgScalar + Signed, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: F) - where - F: Fn(&Array2) -> Array2, - { - let grad = gradient(&self.params.weights()); - self.params.weights_mut().scaled_add(-gamma, &grad); - } - - pub fn update_with_gradient(&mut self, gamma: T, grad: &Array2) { - self.params.weights_mut().scaled_add(-gamma, grad); - } -} - -impl Layer -where - A: Activate, - T: NdFloat, -{ - pub fn linear(&self, args: &Array2) -> Array2 { - self.params().forward(args) - } -} - -impl Layer -where - A: Activate + Gradient, - T: NdFloat + Signed, -{ - pub fn grad(&mut self, gamma: T, args: &Array2, targets: &Array2) -> T { - let ns = T::from(args.shape()[0]).unwrap(); - let pred = self.forward(args); - - let scale = T::from(2).unwrap() * ns; - - let errors = &pred - targets; - let dz = errors * self.activator.gradient(&pred); - let dw = args.t().dot(&dz) / scale; - - self.params_mut().weights_mut().scaled_add(-gamma, &dw.t()); - - let loss = targets - .mean_sq_err(&pred) - .expect("Failed to calculate loss"); - T::from(loss).unwrap() - } -} - -impl Layer -where - A: Activate, - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - self.params = self.params.init(biased); - self - } -} - -impl Features for Layer -where - A: Activate, - T: Float, -{ - fn inputs(&self) -> usize { - self.features.inputs() - } - - fn outputs(&self) -> usize { - self.features.outputs() - } -} - -// impl Forward> for Layer -// where -// A: Activate, -// D: Dimension, -// T: NdFloat, -// Array: Dot, Output = Array>, -// { -// type Output = Array2; - -// fn forward(&self, args: &Array2) -> Self::Output { -// self.activator.activate(&self.linear(args)) -// } -// } - -impl Forward> for Layer -where - A: Activate, - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, args: &Array2) -> Self::Output { - self.activator.activate(&self.linear(args)) - } -} - -// impl PartialOrd for Layer -// where -// A: Activate + PartialEq, -// T: Float, -// { -// fn partial_cmp(&self, other: &Self) -> Option { -// self.position.partial_cmp(&other.position) -// } -// } - -// impl From for Layer -// where -// A: Activate + Default, -// S: IntoDimension -// T: Float, -// { -// fn from(features: LayerShape) -> Self { -// Self::new(features, LayerPosition::input()) -// } -// } - -impl From for Layer -where - A: Activate + Default, - T: Default, -{ - fn from(features: LayerShape) -> Self { - Self { - activator: A::default(), - features, - name: String::new(), - params: LayerParams::new(features), - } - } -} - -impl IntoIterator for Layer -where - A: Activate + Default, - T: Float, -{ - type Item = Node; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.params.into_iter() - } -} - -impl FromIterator> for Layer -where - A: Activate + Default, - T: Clone + Default, -{ - fn from_iter>>(nodes: I) -> Self { - let params = LayerParams::from_iter(nodes); - Self { - activator: A::default(), - features: *params.features(), - name: String::new(), - params, - } - } -} diff --git a/.artifacts/archive/neural/src/layers/mod.rs b/.artifacts/archive/neural/src/layers/mod.rs deleted file mode 100644 index 3cf0a6b0..00000000 --- a/.artifacts/archive/neural/src/layers/mod.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - Appellation: layers - Contrib: FL03 -*/ -//! # Layers -pub use self::{cmp::*, layer::*, params::*, utils::*}; - -pub(crate) mod cmp; -pub(crate) mod layer; -pub(crate) mod params; -// pub(crate) mod stack; - -pub mod seq; - -use crate::prelude::{Activate, ActivateDyn, Forward, Node}; -use ndarray::prelude::{Array2, Ix2}; -// use ndarray::IntoDimension; -use num::Float; - -pub type LayerDyn = Layer>; - -pub trait L: Forward> -where - A: Activate, - T: Float, -{ - fn activator(&self) -> &A; - - fn features(&self) -> LayerShape; - - fn name(&self) -> &str; - - fn params(&self) -> &LayerParams; - - fn is_biased(&self) -> bool; -} - -pub trait FFNLayer: Forward> + L -where - A: Activate, - T: Float, -{ - fn forward(&self, args: &Node) -> Node; -} - -// pub trait LayerExt: L -// where -// T: Float, -// { -// type Rho: Activate; -// } - -pub(crate) mod utils { - use crate::prelude::{Activate, Features, Layer}; - use num::Float; - - pub fn validate_layers(layers: I) -> bool - where - A: Activate, - T: Float, - I: IntoIterator>, - { - let layers = layers.into_iter().collect::>(); - let depth = layers.len(); - let mut dim = true; - for (i, layer) in layers[..(depth - 1)].into_iter().enumerate() { - dim = dim && layer.inputs() == layers[i + 1].outputs(); - } - dim - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::linarr; - use crate::prelude::{Forward, Node, Softmax}; - use ndarray::prelude::Ix2; - - #[test] - fn test_layer() { - let (samples, inputs, outputs) = (20, 5, 3); - let features = LayerShape::new(inputs, outputs); - - let args = linarr::((samples, inputs)).unwrap(); - - let layer = Layer::::from(features).init(true); - - let pred = layer.forward(&args); - - assert_eq!(pred.dim(), (samples, outputs)); - - let nodes = (0..outputs) - .map(|_| Node::::new(inputs).init(true)) - .collect::>(); - let layer = Layer::::from_iter(nodes); - assert_eq!(layer.features(), &features); - } - - #[test] - fn test_layer_iter() { - let (_samples, inputs, outputs) = (20, 5, 3); - let features = LayerShape::new(inputs, outputs); - - let layer = Layer::::from(features).init(true); - - for node in layer.into_iter() { - assert!(node.is_biased()); - assert_eq!(node.features(), inputs); - assert_eq!(node.bias().as_ref().unwrap().dim(), ()); - } - } -} diff --git a/.artifacts/archive/neural/src/layers/params.rs b/.artifacts/archive/neural/src/layers/params.rs deleted file mode 100644 index e690d0d5..00000000 --- a/.artifacts/archive/neural/src/layers/params.rs +++ /dev/null @@ -1,235 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use super::LayerShape; -use crate::core::prelude::GenerateRandom; -use crate::prelude::{Features, Forward, Node}; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Axis, Dimension, NdFloat}; -use ndarray::{LinalgScalar, ScalarOperand}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::traits::{Float, NumAssignOps}; -use serde::{Deserialize, Serialize}; -use std::ops::{self, Neg}; - -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -pub struct LayerParams { - bias: Option>, - pub features: LayerShape, - weights: Array2, -} - -impl LayerParams { - pub fn bias(&self) -> &Option> { - &self.bias - } - - pub fn bias_mut(&mut self) -> &mut Option> { - &mut self.bias - } - - pub fn features(&self) -> &LayerShape { - &self.features - } - - pub fn features_mut(&mut self) -> &mut LayerShape { - &mut self.features - } - - pub fn is_biased(&self) -> bool { - self.bias.is_some() - } - - pub fn set_bias(&mut self, bias: Option>) { - self.bias = bias; - } - - pub fn set_weights(&mut self, weights: Array2) { - self.weights = weights; - } - - pub fn weights(&self) -> &Array2 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array2 { - &mut self.weights - } - - pub fn with_bias(mut self, bias: Option>) -> Self { - self.bias = bias; - self - } - - pub fn with_weights(mut self, weights: Array2) -> Self { - self.weights = weights; - self - } -} - -impl LayerParams -where - T: Default, -{ - pub fn new(features: LayerShape) -> Self { - Self::create(false, features) - } - - pub fn create(biased: bool, features: LayerShape) -> Self { - let bias = if biased { - Some(Array1::default(features.outputs())) - } else { - None - }; - Self { - bias, - features, - weights: Array2::default(features.out_by_in()), - } - } - - pub fn biased(features: LayerShape) -> Self { - Self::create(true, features) - } - - pub fn reset(&mut self) - where - T: NumAssignOps + ScalarOperand, - { - if let Some(bias) = self.bias() { - self.bias = Some(Array1::default(bias.dim())); - } - self.weights *= T::default(); - } - - pub fn set_node(&mut self, idx: usize, node: Node) - where - T: Clone, - { - if let Some(bias) = node.bias() { - if !self.is_biased() { - let mut tmp = Array1::default(self.features().outputs()); - tmp.index_axis_mut(Axis(0), idx).assign(bias); - self.bias = Some(tmp); - } - self.bias - .as_mut() - .unwrap() - .index_axis_mut(Axis(0), idx) - .assign(bias); - } - - self.weights_mut() - .index_axis_mut(Axis(0), idx) - .assign(&node.weights()); - } -} - -impl LayerParams -where - T: LinalgScalar + Neg + 'static, -{ - pub fn update_with_gradient(&mut self, gamma: T, gradient: &Array2) { - self.weights_mut().scaled_add(-gamma, gradient); - } -} - -impl LayerParams -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.bias = Some(Array1::uniform_between(dk, self.features().outputs())); - self - } - - pub fn init_weight(mut self) -> Self { - let dk = (T::one() / T::from(self.features().inputs()).unwrap()).sqrt(); - self.weights = Array2::uniform_between(dk, self.features().out_by_in()); - self - } -} - -impl Features for LayerParams -where - T: Float, -{ - fn inputs(&self) -> usize { - self.features.inputs() - } - - fn outputs(&self) -> usize { - self.features.outputs() - } -} - -impl Forward> for LayerParams -where - D: Dimension, - T: NdFloat, - Array: Dot, Output = Array> + ops::Add, Output = Array>, -{ - type Output = Array; - - fn forward(&self, input: &Array) -> Self::Output { - let w = self.weights().t().to_owned(); - if let Some(bias) = self.bias() { - return input.dot(&w) + bias.clone(); - } - input.dot(&w) - } -} - -impl IntoIterator for LayerParams -where - T: Clone, -{ - type Item = Node; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - if let Some(bias) = self.bias() { - return self - .weights() - .axis_iter(Axis(0)) - .zip(bias.axis_iter(Axis(0))) - .map(|(w, b)| (w.to_owned(), b.to_owned()).into()) - .collect::>() - .into_iter(); - } - self.weights() - .axis_iter(Axis(0)) - .map(|w| (w.to_owned(), None).into()) - .collect::>() - .into_iter() - } -} - -impl FromIterator> for LayerParams -where - T: Clone + Default, -{ - fn from_iter>>(nodes: I) -> Self { - let nodes = nodes.into_iter().collect::>(); - let mut iter = nodes.iter(); - let node = iter.next().unwrap(); - let shape = LayerShape::new(node.features(), nodes.len()); - let mut params = LayerParams::create(true, shape); - params.set_node(0, node.clone()); - for (i, node) in iter.into_iter().enumerate() { - params.set_node(i + 1, node.clone()); - } - params - } -} diff --git a/.artifacts/archive/neural/src/layers/seq/mod.rs b/.artifacts/archive/neural/src/layers/seq/mod.rs deleted file mode 100644 index 74a592d6..00000000 --- a/.artifacts/archive/neural/src/layers/seq/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -/* - Appellation: seq - Contrib: FL03 -*/ - -pub use self::sequential::*; - -pub(crate) mod sequential; - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/layers/seq/sequential.rs b/.artifacts/archive/neural/src/layers/seq/sequential.rs deleted file mode 100644 index e876a330..00000000 --- a/.artifacts/archive/neural/src/layers/seq/sequential.rs +++ /dev/null @@ -1,172 +0,0 @@ -/* - Appellation: sequential - Contrib: FL03 -*/ -use crate::prelude::Forward; -use serde::{Deserialize, Serialize}; - -pub struct Sequential { - layers: Vec>>, -} - -impl Sequential { - pub fn new() -> Self { - Self { layers: Vec::new() } - } - - pub fn include(mut self, layer: L) -> Self - where - L: Forward + 'static, - { - self.layers.push(Box::new(layer)); - self - } - - pub fn push(&mut self, layer: L) - where - L: Forward + 'static, - { - self.layers.push(Box::new(layer)); - } -} - -impl AsRef<[Box>]> for Sequential { - fn as_ref(&self) -> &[Box>] { - &self.layers - } -} - -impl AsMut<[Box>]> for Sequential { - fn as_mut(&mut self) -> &mut [Box>] { - &mut self.layers - } -} - -impl Extend>> for Sequential { - fn extend>>>(&mut self, iter: I) { - self.layers.extend(iter); - } -} - -impl Forward for Sequential -where - T: Clone, -{ - type Output = T; - - fn forward(&self, input: &T) -> Self::Output { - let mut output = input.clone(); - for layer in &self.layers { - output = layer.forward(&output); - } - output - } -} - -impl FromIterator>> for Sequential { - fn from_iter>>>(iter: I) -> Self { - Self { - layers: Vec::from_iter(iter), - } - } -} - -impl IntoIterator for Sequential { - type Item = Box>; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.layers.into_iter() - } -} - -impl Clone for Sequential -where - Box>: Clone, -{ - fn clone(&self) -> Self { - Self { - layers: self.layers.clone(), - } - } -} - -impl Default for Sequential { - fn default() -> Self { - Self::new() - } -} - -impl std::fmt::Debug for Sequential -where - Box>: std::fmt::Debug, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Sequential") - .field("layers", &self.layers) - .finish() - } -} - -impl PartialEq for Sequential -where - Box>: PartialEq, -{ - fn eq(&self, other: &Self) -> bool { - self.layers == other.layers - } -} - -impl Eq for Sequential where Box>: Eq {} - -impl std::hash::Hash for Sequential -where - Box>: std::hash::Hash, -{ - fn hash(&self, state: &mut H) { - self.layers.hash(state); - } -} - -impl PartialOrd for Sequential -where - Box>: PartialOrd, -{ - fn partial_cmp(&self, other: &Self) -> Option { - self.layers.partial_cmp(&other.layers) - } -} - -impl Ord for Sequential -where - Box>: Ord, -{ - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.layers.cmp(&other.layers) - } -} - -impl<'a, T> Deserialize<'a> for Sequential -where - Box>: Deserialize<'a>, -{ - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'a>, - { - let layers = Vec::>>::deserialize(deserializer)?; - Ok(Self { layers }) - } -} - -impl Serialize for Sequential -where - Box>: Serialize, -{ - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.layers.serialize(serializer) - } -} diff --git a/.artifacts/archive/neural/src/lib.rs b/.artifacts/archive/neural/src/lib.rs deleted file mode 100644 index f62c54c1..00000000 --- a/.artifacts/archive/neural/src/lib.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - Appellation: neural - Contrib: FL03 -*/ -//! # concision-neural -//! -//! This library implements the neural network primitives and specifications. -//! -#![feature(fn_traits, unboxed_closures)] -extern crate concision_core as concision; - -pub use self::{primitives::*, specs::*, utils::*}; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod errors; -pub mod func; -pub mod layers; -pub mod models; -pub mod neurons; -pub mod nn; -pub mod ops; -pub mod params; - -#[doc(hidden)] -pub mod exp; - -pub(crate) use concision as core; - -pub mod prelude { - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; - - pub use crate::errors::*; - pub use crate::func::{activate::*, loss::*, rms::*}; - pub use crate::layers::*; - pub use crate::neurons::*; - pub use crate::nn::*; - pub use crate::ops::*; - pub use crate::params::*; -} diff --git a/.artifacts/archive/neural/src/models/config.rs b/.artifacts/archive/neural/src/models/config.rs deleted file mode 100644 index d5fee799..00000000 --- a/.artifacts/archive/neural/src/models/config.rs +++ /dev/null @@ -1,24 +0,0 @@ -/* - Appellation: config - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct ModelConfig { - pub layers: usize, -} - -impl ModelConfig { - pub fn new(layers: usize) -> Self { - Self { layers } - } - - pub fn layers(&self) -> usize { - self.layers - } - - pub fn n_hidden(&self) -> usize { - self.layers - 2 - } -} diff --git a/.artifacts/archive/neural/src/models/mod.rs b/.artifacts/archive/neural/src/models/mod.rs deleted file mode 100644 index a8c537d1..00000000 --- a/.artifacts/archive/neural/src/models/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -/* - Appellation: models - Contrib: FL03 -*/ -//! # Model -//! -pub use self::{config::*, model::*, modes::*, params::*}; - -pub(crate) mod config; -pub(crate) mod model; -pub(crate) mod modes; -pub(crate) mod params; - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/models/model.rs b/.artifacts/archive/neural/src/models/model.rs deleted file mode 100644 index 025e11e6..00000000 --- a/.artifacts/archive/neural/src/models/model.rs +++ /dev/null @@ -1,178 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use super::{ModelConfig, ModelParams}; -use crate::prelude::{Forward, Gradient, LayerParams}; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array1, Array2, Dimension, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use ndarray_stats::DeviationExt; -use num::{Float, Signed}; -use std::ops; - -#[derive(Clone, Debug)] -pub struct Model -where - T: Float, -{ - config: ModelConfig, - params: ModelParams, -} - -impl Model -where - T: Float, -{ - pub fn new(config: ModelConfig) -> Self { - Self { - config, - params: ModelParams::new(), - } - } - - pub fn config(&self) -> &ModelConfig { - &self.config - } - - pub fn config_mut(&mut self) -> &mut ModelConfig { - &mut self.config - } - - pub fn params(&self) -> &ModelParams { - &self.params - } - - pub fn params_mut(&mut self) -> &mut ModelParams { - &mut self.params - } - - pub fn with_params(mut self, params: ModelParams) -> Self { - self.params = params; - self - } -} - -impl Model -where - T: NdFloat + Signed, -{ - pub fn gradient( - &mut self, - data: &Array2, - targets: &Array2, - gamma: T, - grad: impl Gradient, - ) -> anyhow::Result { - // the number of layers in the model - let depth = self.params().len(); - // the gradients for each layer - let mut grads = Vec::with_capacity(self.params().len()); - // a store for the predictions of each layer - let mut store = vec![data.clone()]; - // compute the predictions for each layer - for layer in self.clone().into_iter() { - let pred = layer.forward(&store.last().unwrap()); - store.push(pred); - } - // compute the error for the last layer - let error = store.last().unwrap() - targets; - // compute the error gradient for the last layer - let dz = &error * grad.gradient(&error); - // push the error gradient for the last layer - grads.push(dz.clone()); - - for i in (1..depth).rev() { - // get the weights for the current layer - let wt = self.params[i].weights().t(); - // compute the delta for the current layer w.r.t. the previous layer - let dw = grads.last().unwrap().dot(&wt); - // compute the gradient w.r.t. the current layer's predictions - let dp = grad.gradient(&store[i]); - // compute the gradient for the current layer - let gradient = dw * &dp; - grads.push(gradient); - } - // reverse the gradients so that they are in the correct order - grads.reverse(); - // update the parameters for each layer - for i in 0..depth { - let gradient = &store[i].t().dot(&grads[i]); - self.params[i] - .weights_mut() - .scaled_add(-gamma, &gradient.t()); - } - let loss = self.forward(data).mean_sq_err(targets)?; - Ok(loss) - } -} - -impl Model -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - self.params = self.params.init(biased); - self - } -} - -// impl FromIterator for Model -// where -// T: Float, -// { -// fn from_iter(iter: I) -> Self -// where -// I: IntoIterator, -// { -// let params = ModelParam::from_iter(iter); -// Self { - -// } -// } -// } - -// impl FromIterator> for Model -// where -// T: Float, -// { -// fn from_iter(iter: I) -> Self -// where -// I: IntoIterator>, -// { -// Self { -// children: iter.into_iter().collect(), -// } -// } -// } -impl IntoIterator for Model -where - T: Float, -{ - type Item = LayerParams; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.params.into_iter() - } -} - -impl Forward> for Model -where - D: Dimension, - T: NdFloat, - Array: Dot, Output = Array> + ops::Add, Output = Array>, -{ - type Output = Array; - - fn forward(&self, input: &Array) -> Self::Output { - let mut store = vec![input.clone()]; - for layer in self.clone().into_iter() { - let pred = layer.forward(&store.last().unwrap()); - store.push(pred); - } - store.last().unwrap().clone() - } -} diff --git a/.artifacts/archive/neural/src/models/modes.rs b/.artifacts/archive/neural/src/models/modes.rs deleted file mode 100644 index 964e9ab7..00000000 --- a/.artifacts/archive/neural/src/models/modes.rs +++ /dev/null @@ -1,64 +0,0 @@ -/* - Appellation: modes - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Mode { - Predict, - #[default] - Test, - Train, -} - -impl Mode { - pub fn predict() -> Self { - Self::Predict - } - - pub fn test() -> Self { - Self::Test - } - - pub fn train() -> Self { - Self::Train - } -} - -impl From for Mode { - fn from(mode: usize) -> Self { - match mode % Mode::VARIANTS.len() { - 0 => Self::Predict, - 1 => Self::Test, - _ => Self::Train, - } - } -} - -impl From for usize { - fn from(mode: Mode) -> Self { - mode as usize - } -} diff --git a/.artifacts/archive/neural/src/models/params.rs b/.artifacts/archive/neural/src/models/params.rs deleted file mode 100644 index 5f990ad3..00000000 --- a/.artifacts/archive/neural/src/models/params.rs +++ /dev/null @@ -1,328 +0,0 @@ -/* - Appellation: stack - Contrib: FL03 -*/ -use crate::prelude::{Features, Forward, LayerParams, LayerShape}; -use ndarray::prelude::{Array2, Dimension, Ix2, NdFloat}; -use ndarray::IntoDimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -use serde::{Deserialize, Serialize}; -use std::ops; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct ModelParams { - children: Vec>, -} - -impl ModelParams { - pub fn new() -> Self { - Self { - children: Vec::new(), - } - } - - pub fn with_capacity(depth: usize) -> Self { - Self { - children: Vec::with_capacity(depth), - } - } - - pub fn with_shapes(shapes: impl IntoIterator) -> Self - where - Sh: IntoDimension, - T: Clone + Default, - { - let tmp = Vec::from_iter(shapes.into_iter().map(IntoDimension::into_dimension)); - let mut children = Vec::new(); - for (inputs, outputs) in tmp.iter().map(|s| s.into_pattern()) { - let features = LayerShape::new(inputs, outputs); - children.push(LayerParams::new(features)); - } - Self { children } - } - - pub fn is_empty(&self) -> bool { - self.children.is_empty() - } - - pub fn build_layers(mut self, shapes: impl IntoIterator) -> Self - where - T: Clone + Default, - { - // let shapes = shapes.into_iter().map(|s| (s.inputs(), s.outputs())); - for (inputs, outputs) in shapes.into_iter() { - let features = LayerShape::new(inputs, outputs); - self.children.push(LayerParams::new(features)); - } - self - } - - pub fn len(&self) -> usize { - self.children.len() - } - - pub fn params(&self) -> &[LayerParams] { - &self.children - } - - pub fn params_mut(&mut self) -> &mut [LayerParams] { - &mut self.children - } - - pub fn pop(&mut self) -> Option> { - self.children.pop() - } - - pub fn push(&mut self, params: LayerParams) { - self.children.push(params); - } - - pub fn validate_shapes(&self) -> bool { - let mut dim = true; - for (i, layer) in self.children[..(self.len() - 1)].iter().enumerate() { - dim = dim && layer.features().outputs() == self.children[i + 1].features().inputs(); - } - dim - } -} - -impl ModelParams -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - self.children - .iter_mut() - .for_each(|l| *l = l.clone().init(biased)); - self - } - - pub fn init_bias(mut self) -> Self { - self.children - .iter_mut() - .for_each(|l| *l = l.clone().init_bias()); - self - } - - pub fn init_weight(mut self) -> Self { - self.children - .iter_mut() - .for_each(|l| *l = l.clone().init_weight()); - self - } -} - -impl Forward> for ModelParams -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, input: &Array2) -> Array2 { - let mut iter = self.children.iter(); - - let mut output = iter.next().unwrap().forward(input); - for layer in iter { - output = layer.forward(&output); - } - output - } -} - -impl AsRef<[LayerParams]> for ModelParams { - fn as_ref(&self) -> &[LayerParams] { - &self.children - } -} - -impl AsMut<[LayerParams]> for ModelParams { - fn as_mut(&mut self) -> &mut [LayerParams] { - &mut self.children - } -} - -impl FromIterator for ModelParams -where - T: Clone + Default, -{ - fn from_iter(iter: I) -> Self - where - I: IntoIterator, - { - Self { - children: iter.into_iter().map(LayerParams::new).collect(), - } - } -} - -impl FromIterator> for ModelParams -where - T: Float, -{ - fn from_iter(iter: I) -> Self - where - I: IntoIterator>, - { - Self { - children: iter.into_iter().collect(), - } - } -} - -impl IntoIterator for ModelParams -where - T: Float, -{ - type Item = LayerParams; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.children.into_iter() - } -} - -impl ops::Index for ModelParams -where - T: Float, -{ - type Output = LayerParams; - - fn index(&self, index: usize) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.children[index] - } -} - -impl ops::Index> for ModelParams -where - T: Float, -{ - type Output = [LayerParams]; - - fn index(&self, index: ops::Range) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut> for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: ops::Range) -> &mut Self::Output { - &mut self.children[index] - } -} - -impl ops::Index> for ModelParams -where - T: Float, -{ - type Output = [LayerParams]; - - fn index(&self, index: ops::RangeFrom) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut> for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: ops::RangeFrom) -> &mut Self::Output { - &mut self.children[index] - } -} - -impl ops::Index for ModelParams -where - T: Float, -{ - type Output = [LayerParams]; - - fn index(&self, index: ops::RangeFull) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: ops::RangeFull) -> &mut Self::Output { - &mut self.children[index] - } -} - -impl ops::Index> for ModelParams -where - T: Float, -{ - type Output = [LayerParams]; - - fn index(&self, index: ops::RangeInclusive) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut> for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: ops::RangeInclusive) -> &mut Self::Output { - &mut self.children[index] - } -} - -impl ops::Index> for ModelParams -where - T: Float, -{ - type Output = [LayerParams]; - - fn index(&self, index: ops::RangeTo) -> &Self::Output { - &self.children[index] - } -} - -impl ops::IndexMut> for ModelParams -where - T: Float, -{ - fn index_mut(&mut self, index: ops::RangeTo) -> &mut Self::Output { - &mut self.children[index] - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::prelude::LayerShape; - - #[test] - fn test_model_params() { - let (inputs, outputs) = (5, 3); - - let shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - - let params = ModelParams::::new().build_layers(shapes).init(true); - - // validate the dimensions of the model params - assert!(params.validate_shapes()); - - for (layer, shape) in params.into_iter().zip(&shapes) { - assert_eq!(layer.features(), &LayerShape::new(shape.0, shape.1)); - } - } -} diff --git a/.artifacts/archive/neural/src/neurons/mod.rs b/.artifacts/archive/neural/src/neurons/mod.rs deleted file mode 100644 index 1794906c..00000000 --- a/.artifacts/archive/neural/src/neurons/mod.rs +++ /dev/null @@ -1,80 +0,0 @@ -/* - Appellation: neurons - Contrib: FL03 -*/ -//! # neurons -pub use self::{node::*, perceptron::*, synapse::*}; - -pub(crate) mod node; -pub(crate) mod perceptron; -pub(crate) mod synapse; - -use crate::func::activate::Activate; -use crate::ops::Predict; -use ndarray::prelude::Ix1; -use std::collections::HashMap; - -pub trait Neuron: Predict { - type Rho: Activate; - - fn params(&self) -> HashMap; - - fn rho(&self) -> &Self::Rho; -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::func::activate::{softmax, Activate, Softmax}; - use crate::prelude::Forward; - // use lazy_static::lazy_static; - use ndarray::prelude::{array, Array1, Ix1}; - - fn _artificial( - args: &Array1, - bias: Option>, - rho: impl Activate, - weights: &Array1, - ) -> Array1 { - let bias = bias.unwrap_or_else(|| Array1::zeros(args.len())); - let linear = args.dot(weights) + bias; - rho.activate(&linear) - } - - #[test] - fn test_neuron() { - let bias = 0.0; - - let data = array![[10.0, 10.0, 6.0, 1.0, 8.0]]; - let weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - let neuron = Perceptron::::new(5).with_weights(weights.clone()); - - let linear = data.dot(&weights) + bias; - let exp = softmax(&linear); - - assert_eq!(exp, neuron.forward(&data)); - } - - // #[test] - // fn test_node() { - // let bias = ndarray::Array1::::zeros(4); - - // let a_data = array![10.0, 10.0, 6.0, 1.0, 8.0]; - // let a_weights = array![2.0, 1.0, 10.0, 1.0, 7.0]; - // let a = Neuron::new(softmax, bias.clone(), a_weights.clone()); - // let node_a = Node::new(a.clone()).with_data(a_data.clone()); - - // let exp = _artificial(&a_data, Some(bias.clone()), Softmax::default(), &a_weights); - // assert_eq!(node_a.process(), exp); - - // let b_data = array![0.0, 9.0, 3.0, 5.0, 3.0]; - // let b_weights = array![2.0, 8.0, 8.0, 0.0, 3.0]; - - // let b = Neuron::new(softmax, bias.clone(), b_weights.clone()); - // let node_b = Node::new(b.clone()).with_data(b_data.clone()); - // let exp = _artificial(&b_data, Some(bias), Softmax::default(), &b_weights); - // assert_eq!(node_b.process(), exp); - - // assert_eq!(node_a.dot() + node_b.dot(), 252.0); - // } -} diff --git a/.artifacts/archive/neural/src/neurons/node.rs b/.artifacts/archive/neural/src/neurons/node.rs deleted file mode 100644 index 04df98aa..00000000 --- a/.artifacts/archive/neural/src/neurons/node.rs +++ /dev/null @@ -1,251 +0,0 @@ -/* - Appellation: node - Contrib: FL03 -*/ -use crate::core::prelude::GenerateRandom; - -use crate::prelude::Forward; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array0, Array1, Array2, Dimension}; -use ndarray::{LinalgScalar, RemoveAxis, ScalarOperand}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::{Float, Num}; -use std::ops::{self, Neg}; - -#[derive(Clone, Debug, PartialEq)] -pub struct Node { - bias: Option>, - features: usize, - weights: Array1, -} - -impl Node -where - T: Default, -{ - pub fn create(biased: bool, features: usize) -> Self { - let bias = if biased { - Some(Array0::default(())) - } else { - None - }; - Self { - bias, - features, - weights: Array1::default(features), - } - } - - pub fn biased(features: usize) -> Self { - Self::create(true, features) - } - - pub fn new(features: usize) -> Self { - Self::create(false, features) - } -} -impl Node { - pub fn bias(&self) -> Option<&Array0> { - self.bias.as_ref() - } - - pub fn bias_mut(&mut self) -> Option<&mut Array0> { - self.bias.as_mut() - } - - pub fn features(&self) -> usize { - self.features - } - - pub fn is_biased(&self) -> bool { - self.bias.is_some() - } - - pub fn set_bias(&mut self, bias: Option>) { - self.bias = bias; - } - - pub fn set_features(&mut self, features: usize) { - self.features = features; - } - - pub fn set_weights(&mut self, weights: Array1) { - self.weights = weights; - } - - pub fn weights(&self) -> &Array1 { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut Array1 { - &mut self.weights - } - - pub fn with_bias(mut self, bias: Option>) -> Self { - self.bias = bias; - self - } - - pub fn with_features(mut self, features: usize) -> Self { - self.features = features; - self - } - - pub fn with_weights(mut self, weights: Array1) -> Self { - self.weights = weights; - self - } -} - -impl Node -where - T: Num + ScalarOperand + 'static, - Array2: Dot, Output = Array1>, -{ - pub fn linear(&self, data: &Array2) -> Array1 { - let w = self.weights().t().to_owned(); - if let Some(bias) = self.bias() { - data.dot(&w) + bias - } else { - data.dot(&w) - } - } -} -impl Node -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.features).unwrap()).sqrt(); - self.bias = Some(Array0::uniform_between(dk, ())); - self - } - - pub fn init_weight(mut self) -> Self { - let features = self.features; - let dk = (T::one() / T::from(features).unwrap()).sqrt(); - self.weights = Array1::uniform_between(dk, features); - self - } -} - -impl Node -where - T: LinalgScalar + Neg, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: G) - where - G: Fn(&Array1) -> Array1, - { - let grad = gradient(self.weights()); - self.weights_mut().scaled_add(-gamma, &grad); - } - - pub fn activate(&self, data: &Array2, activator: A) -> Array1 - where - A: Fn(&Array1) -> Array1, - { - activator(&self.forward(data)) - } -} - -impl Forward> for Node -where - D: Dimension + RemoveAxis, - T: Clone, - Array: Dot, Output = Array>, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn forward(&self, data: &Array) -> Self::Output { - let w = self.weights().t().to_owned(); - if let Some(bias) = self.bias() { - return data.dot(&w) + bias.clone(); - } - data.dot(&w) - } -} - -impl FromIterator for Node -where - T: Float, -{ - fn from_iter(iter: I) -> Self - where - I: IntoIterator, - { - let weights = Array1::::from_iter(iter); - Self { - bias: None, - features: weights.len(), - weights, - } - } -} - -impl From<(Array1, Array0)> for Node { - fn from((weights, bias): (Array1, Array0)) -> Self { - Self { - bias: Some(bias), - features: weights.len(), - weights, - } - } -} - -impl From<(Array1, T)> for Node -where - T: Num + ScalarOperand, -{ - fn from((weights, bias): (Array1, T)) -> Self { - Self { - bias: Some(Array0::ones(()) * bias), - features: weights.len(), - weights, - } - } -} - -impl From<(Array1, Option)> for Node -where - T: Num + ScalarOperand, -{ - fn from((weights, bias): (Array1, Option)) -> Self { - let bias = if let Some(b) = bias { - Some(Array0::ones(()) * b) - } else { - None - }; - Self { - bias, - features: weights.len(), - weights, - } - } -} - -impl From<(Array1, Option>)> for Node { - fn from((weights, bias): (Array1, Option>)) -> Self { - Self { - bias, - features: weights.len(), - weights, - } - } -} - -impl From> for (Array1, Option>) { - fn from(node: Node) -> Self { - (node.weights, node.bias) - } -} diff --git a/.artifacts/archive/neural/src/neurons/perceptron.rs b/.artifacts/archive/neural/src/neurons/perceptron.rs deleted file mode 100644 index 28c6b56e..00000000 --- a/.artifacts/archive/neural/src/neurons/perceptron.rs +++ /dev/null @@ -1,211 +0,0 @@ -/* - Appellation: neuron - Contrib: FL03 -*/ -use super::Node; -use crate::prelude::{Activate, Forward, LinearActivation}; -use ndarray::prelude::{Array0, Array1, Array2, Ix1, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -/// Artificial Neuron -#[derive(Clone, Debug, PartialEq)] -pub struct Perceptron -where - A: Activate, -{ - activation: A, - node: Node, -} - -impl Perceptron -where - A: Activate, -{ - pub fn new(features: usize) -> Self - where - A: Default, - T: Default, - { - Self { - activation: A::default(), - node: Node::create(false, features), - } - } - - pub fn node(&self) -> &Node { - &self.node - } - - pub fn node_mut(&mut self) -> &mut Node { - &mut self.node - } - - pub fn features(&self) -> usize { - self.node.features() - } - - pub fn params(&self) -> &Node { - &self.node - } - - pub fn params_mut(&mut self) -> &mut Node { - &mut self.node - } - - pub fn rho(&self) -> &A { - &self.activation - } - - pub fn with_bias(mut self, bias: Option>) -> Self { - self.node = self.node.with_bias(bias); - self - } - - pub fn with_rho>(self, rho: B) -> Perceptron { - Perceptron { - activation: rho, - node: self.node, - } - } - - pub fn with_node(mut self, node: Node) -> Self { - self.node = node; - self - } - - pub fn with_weights(mut self, weights: Array1) -> Self { - self.node = self.node.with_weights(weights); - self - } - - pub fn weights(&self) -> &Array1 { - self.node.weights() - } - - pub fn weights_mut(&mut self) -> &mut Array1 { - self.node.weights_mut() - } - - pub fn set_weights(&mut self, weights: Array1) { - self.node.set_weights(weights); - } -} - -impl Perceptron -where - T: NdFloat, - A: Activate, -{ - pub fn apply_gradient(&mut self, gamma: T, gradient: G) - where - G: Fn(&Array1) -> Array1, - { - let grad = gradient(self.node().weights()); - self.update_with_gradient(gamma, &grad); - } - - pub fn update_with_gradient(&mut self, gamma: T, grad: &Array1) { - self.node.weights_mut().scaled_add(-gamma, grad); - } -} - -impl Perceptron -where - T: Float + SampleUniform, - A: Activate, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - self.node = self.node.init_bias(); - self - } - - pub fn init_weight(mut self) -> Self { - self.node = self.node.init_weight(); - self - } -} - -impl Forward> for Perceptron -where - T: NdFloat, - A: Activate, -{ - type Output = Array1; - - fn forward(&self, args: &Array2) -> Self::Output { - let linstep = self.params().forward(args); - self.rho().activate(&linstep) - } -} - -impl From<(Array1, Array0)> for Perceptron -where - T: Float, - A: Activate + Default, -{ - fn from((weights, bias): (Array1, Array0)) -> Self { - Self { - activation: A::default(), - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, T)> for Perceptron -where - T: NdFloat, - A: Activate + Default, -{ - fn from((weights, bias): (Array1, T)) -> Self { - Self { - activation: A::default(), - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, Array0, A)> for Perceptron -where - T: Float, - A: Activate, -{ - fn from((weights, bias, activation): (Array1, Array0, A)) -> Self { - Self { - activation, - node: Node::from((weights, bias)), - } - } -} - -impl From<(Array1, T, A)> for Perceptron -where - T: NdFloat, - A: Activate, -{ - fn from((weights, bias, activation): (Array1, T, A)) -> Self { - Self { - activation, - node: Node::from((weights, bias)), - } - } -} - -impl From> for (Array1, Option>) -where - T: Float, - A: Activate, -{ - fn from(neuron: Perceptron) -> Self { - neuron.node().clone().into() - } -} diff --git a/.artifacts/archive/neural/src/neurons/synapse.rs b/.artifacts/archive/neural/src/neurons/synapse.rs deleted file mode 100644 index f04a1c80..00000000 --- a/.artifacts/archive/neural/src/neurons/synapse.rs +++ /dev/null @@ -1,9 +0,0 @@ -/* - Appellation: synapse - Contrib: FL03 -*/ - -pub struct Synapse { - pub layer: usize, - pub node: usize, -} diff --git a/.artifacts/archive/neural/src/nn/gnn/mod.rs b/.artifacts/archive/neural/src/nn/gnn/mod.rs deleted file mode 100644 index 321efbe5..00000000 --- a/.artifacts/archive/neural/src/nn/gnn/mod.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - Appellation: graph - Contrib: FL03 -*/ -//! # Graph Neural Network -//! -pub use self::{model::*, tasks::*}; - -pub(crate) mod model; -pub(crate) mod tasks; - -use num::Float; - -pub trait GNN -where - T: Float, -{ - type G; - - fn depth(&self) -> usize { - self.layers().len() - } - - fn layers(&self) -> &[Self::G]; -} - -pub(crate) mod utils {} - -#[cfg(tets)] -mod tests {} diff --git a/.artifacts/archive/neural/src/nn/gnn/model.rs b/.artifacts/archive/neural/src/nn/gnn/model.rs deleted file mode 100644 index 0cea9291..00000000 --- a/.artifacts/archive/neural/src/nn/gnn/model.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use crate::prelude::Node; - -use num::Float; -use petgraph::prelude::{Directed, Graph}; - -pub enum Edges { - Biased { bias: f64, weights: Vec }, - - Unbiased { weights: Vec }, - - Empty, -} - -pub struct GraphModel -where - T: Float, -{ - graph: Graph, K>, -} diff --git a/.artifacts/archive/neural/src/nn/gnn/tasks.rs b/.artifacts/archive/neural/src/nn/gnn/tasks.rs deleted file mode 100644 index 128d00c9..00000000 --- a/.artifacts/archive/neural/src/nn/gnn/tasks.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use num::Float; - -pub trait GraphTask -where - T: Float, -{ - type G; - - fn depth(&self) -> usize { - self.layers().len() - } - - fn layers(&self) -> &[Self::G]; -} diff --git a/.artifacts/archive/neural/src/nn/kinds.rs b/.artifacts/archive/neural/src/nn/kinds.rs deleted file mode 100644 index e79b5655..00000000 --- a/.artifacts/archive/neural/src/nn/kinds.rs +++ /dev/null @@ -1,157 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum NetworkKind { - #[serde(alias = "convolution", alias = "conv", alias = "cnn")] - Convolution = 0, - #[default] - #[serde(alias = "feed_forward", alias = "ffn")] - FeedForward = 1, - #[serde(alias = "graph", alias = "gnn")] - Graph = 2, - #[serde(alias = "recurrent", alias = "rnn")] - Recurrent = 3, -} - -impl NetworkKind { - pub fn cnn() -> Self { - Self::Convolution - } - - pub fn ffn() -> Self { - Self::FeedForward - } - - pub fn gnn() -> Self { - Self::Graph - } - - pub fn rnn() -> Self { - Self::Recurrent - } -} - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Learning { - Reinforcement = 0, - #[default] - Supervised = 1, - Unsupervised = 2, -} - -impl Learning { - pub fn reinforcement() -> Self { - Self::Reinforcement - } - - pub fn supervised() -> Self { - Self::Supervised - } - - pub fn unsupervised() -> Self { - Self::Unsupervised - } -} - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum NetworkType { - #[serde(alias = "autoencoder", alias = "ae")] - Autoencoder = 0, - #[default] - #[serde(alias = "classifier", alias = "clf")] - Classifier = 1, - #[serde(alias = "regressor", alias = "reg")] - Regressor = 2, -} - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum NetworkStyle { - #[default] - Deep, - Shallow, -} diff --git a/.artifacts/archive/neural/src/nn/mod.rs b/.artifacts/archive/neural/src/nn/mod.rs deleted file mode 100644 index 1144614d..00000000 --- a/.artifacts/archive/neural/src/nn/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: nn - Contrib: FL03 -*/ -//! # Neural Network -pub use self::{kinds::*, position::*}; - -pub(crate) mod kinds; -pub(crate) mod position; - -pub mod gnn; - -pub trait NeuralNetwork {} diff --git a/.artifacts/archive/neural/src/nn/position.rs b/.artifacts/archive/neural/src/nn/position.rs deleted file mode 100644 index 324d08e1..00000000 --- a/.artifacts/archive/neural/src/nn/position.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - Appellation: position - Contrib: FL03 -*/ - -pub struct Position { - index: usize, -} - -impl Position { - pub fn new(index: usize) -> Self { - Self { index } - } - - pub fn index(&self) -> usize { - self.index - } -} - -pub struct Scope {} diff --git a/.artifacts/archive/neural/src/ops/compile.rs b/.artifacts/archive/neural/src/ops/compile.rs deleted file mode 100644 index 9cd6b7c6..00000000 --- a/.artifacts/archive/neural/src/ops/compile.rs +++ /dev/null @@ -1,12 +0,0 @@ -/* - Appellation: compile - Contrib: FL03 -*/ -use crate::core::prelude::BoxResult; -use crate::func::loss::Loss; - -pub trait Compile { - type Opt; - - fn compile(&mut self, loss: impl Loss, optimizer: Self::Opt) -> BoxResult<()>; -} diff --git a/.artifacts/archive/neural/src/ops/dropout.rs b/.artifacts/archive/neural/src/ops/dropout.rs deleted file mode 100644 index 94db8acc..00000000 --- a/.artifacts/archive/neural/src/ops/dropout.rs +++ /dev/null @@ -1,65 +0,0 @@ -/* - Appellation: dropout - Contrib: FL03 -*/ -use crate::prelude::Forward; -use ndarray::prelude::{Array, Dimension}; -use ndarray::ScalarOperand; -use ndarray_rand::rand_distr::Bernoulli; -use ndarray_rand::RandomExt; -use num::Num; -use serde::{Deserialize, Serialize}; - -pub fn dropout(array: &Array, p: f64) -> Array -where - D: Dimension, - T: Num + ScalarOperand, -{ - // Create a Bernoulli distribution for dropout - let distribution = Bernoulli::new(p).unwrap(); - - // Create a mask of the same shape as the input array - let mask: Array = Array::random(array.dim(), distribution); - let mask = mask.mapv(|x| if x { T::zero() } else { T::one() }); - - // Element-wise multiplication to apply dropout - array * mask -} - -#[derive(Clone, Copy, Debug, Deserialize, PartialEq, PartialOrd, Serialize)] -pub struct Dropout { - axis: Option, - p: f64, -} - -impl Dropout { - pub fn new(axis: Option, p: f64) -> Self { - Self { axis, p } - } - - pub fn dropout(&self, array: &Array) -> Array - where - D: Dimension, - T: Num + ScalarOperand, - { - dropout(array, self.p) - } -} - -impl Default for Dropout { - fn default() -> Self { - Self::new(None, 0.5) - } -} - -impl Forward> for Dropout -where - D: Dimension, - T: Num + ScalarOperand, -{ - type Output = Array; - - fn forward(&self, input: &Array) -> Self::Output { - dropout(input, self.p) - } -} diff --git a/.artifacts/archive/neural/src/ops/mod.rs b/.artifacts/archive/neural/src/ops/mod.rs deleted file mode 100644 index 38b6980d..00000000 --- a/.artifacts/archive/neural/src/ops/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -/* - Appellation: ops - Contrib: FL03 -*/ -pub use self::{compile::*, dropout::*, norm::*, predict::*, train::*}; - -pub(crate) mod compile; -pub(crate) mod dropout; -pub(crate) mod norm; -pub(crate) mod predict; -pub(crate) mod train; - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::RoundTo; - use crate::prelude::Forward; - use ndarray::prelude::{array, Array, Ix2}; - - #[test] - fn test_layer_norm() { - let features = 4; - let data: Array = Array::linspace(1., 4., 4) - .into_shape((1, features)) - .unwrap(); - let norm = LayerNorm::::new((1, features)); - let normed = norm.forward(&data); - let rounded = normed.map(|x| x.round_to(4)); - let exp = array![[-1.1619, -0.3873, 0.3873, 1.1619]]; - assert_eq!(rounded, exp); - } -} diff --git a/.artifacts/archive/neural/src/ops/norm.rs b/.artifacts/archive/neural/src/ops/norm.rs deleted file mode 100644 index 167017be..00000000 --- a/.artifacts/archive/neural/src/ops/norm.rs +++ /dev/null @@ -1,114 +0,0 @@ -/* - Appellation: norm - Contrib: FL03 -*/ -use crate::prelude::Forward; -use ndarray::prelude::{Array, Axis, Dimension, Ix2, NdFloat}; -use ndarray::{IntoDimension, RemoveAxis}; -use num::{FromPrimitive, Num}; -use serde::{Deserialize, Serialize}; - -pub fn norm(x: &Array) -> Array -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - let epsilon = T::from(1e-6).unwrap(); - // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean().expect("mean_axis failed"); - - let std = x.std(T::one()); - (x.clone() - mean) / (std + epsilon) -} - -pub fn norma(x: &Array, axis: usize) -> Array -where - D: RemoveAxis, - T: FromPrimitive + NdFloat, -{ - let axis = Axis(axis); - let epsilon = T::from(1e-6).unwrap(); - // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean_axis(axis.clone()).expect("mean_axis failed"); - - let std = x.std_axis(axis, T::one()); - (x.clone() - mean) / (std + epsilon) -} - -pub fn norm_and_scale( - x: &Array, - alpha: &Array, - beta: &Array, -) -> Array -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - let epsilon = T::from(1e-6).unwrap(); - // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = x.mean().unwrap_or_else(T::zero); - // Normalize the activations. - let norm = (x - mean) / (x.std(T::one()) + epsilon); - - // Scale and shift the normalized activations with learnable parameters alpha and beta. - norm * alpha.clone() + beta.clone() -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct LayerNorm -where - D: Dimension, - T: Num, -{ - alpha: Array, - beta: Array, -} - -impl LayerNorm -where - D: Dimension, - T: Clone + Num, -{ - pub fn new(dim: impl IntoDimension) -> Self { - let dim = dim.into_dimension(); - Self { - alpha: Array::ones(dim.clone()), - beta: Array::zeros(dim), - } - } - - pub fn alpha(&self) -> &Array { - &self.alpha - } - - pub fn alpha_mut(&mut self) -> &mut Array { - &mut self.alpha - } - - pub fn beta(&self) -> &Array { - &self.beta - } - - pub fn beta_mut(&mut self) -> &mut Array { - &mut self.beta - } -} - -impl Forward> for LayerNorm -where - D: Dimension, - T: FromPrimitive + NdFloat, -{ - type Output = Array; - - fn forward(&self, data: &Array) -> Self::Output { - let epsilon = T::from(1e-6).unwrap(); - // Calculate the mean and standard deviation of the activations along the feature axis. - let mean = data.mean().unwrap_or_else(T::zero); - // Normalize the activations. - let norm = (data - mean) / (data.std(T::one()) + epsilon); - - // Scale and shift the normalized activations with learnable parameters alpha and beta. - norm * self.alpha() + self.beta() - } -} diff --git a/.artifacts/archive/neural/src/ops/predict.rs b/.artifacts/archive/neural/src/ops/predict.rs deleted file mode 100644 index a682eb84..00000000 --- a/.artifacts/archive/neural/src/ops/predict.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* - Appellation: predict - Contrib: FL03 -*/ -use crate::prelude::{Forward, PredictError}; - -pub trait Predict { - type Output; - - fn predict(&self, input: &T) -> Result; -} - -impl Predict for S -where - S: Forward, -{ - type Output = O; - - fn predict(&self, input: &T) -> Result { - Ok(self.forward(input)) - } -} diff --git a/.artifacts/archive/neural/src/ops/train.rs b/.artifacts/archive/neural/src/ops/train.rs deleted file mode 100644 index dc6aa7b6..00000000 --- a/.artifacts/archive/neural/src/ops/train.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - Appellation: train - Contrib: FL03 -*/ -use crate::core::prelude::BoxResult; -use ndarray::prelude::{Array2, Axis}; - -pub trait Train { - fn train(&mut self, input: &Array2, target: &Array2) -> BoxResult; - - fn train_batch( - &mut self, - batch_size: usize, - input: &Array2, - target: &Array2, - ) -> BoxResult - where - T: Clone + std::iter::Sum, - { - let res = input - .axis_chunks_iter(Axis(0), batch_size) - .zip(target.axis_chunks_iter(Axis(0), batch_size)) - .map(|(x, y)| self.train(&x.to_owned(), &y.to_owned()).expect("")) - .sum(); - Ok(res) - } -} diff --git a/.artifacts/archive/neural/src/params/iter.rs b/.artifacts/archive/neural/src/params/iter.rs deleted file mode 100644 index c682dc15..00000000 --- a/.artifacts/archive/neural/src/params/iter.rs +++ /dev/null @@ -1,33 +0,0 @@ -/* - Appellation: iter - Contrib: FL03 -*/ - -pub struct Entry { - key: K, - value: V, -} - -impl Entry { - pub fn new(key: K, value: V) -> Self { - Self { key, value } - } - - pub fn key(&self) -> &K { - &self.key - } - - pub fn value(&self) -> &V { - &self.value - } - - pub fn value_mut(&mut self) -> &mut V { - &mut self.value - } -} - -pub struct IntoIter; - -pub struct Iter; - -pub struct IterMut; diff --git a/.artifacts/archive/neural/src/params/kinds.rs b/.artifacts/archive/neural/src/params/kinds.rs deleted file mode 100644 index 03f5d2ac..00000000 --- a/.artifacts/archive/neural/src/params/kinds.rs +++ /dev/null @@ -1,96 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -pub trait ParamType: ToString { - fn kind(&self) -> String; -} - -impl ParamType for T -where - T: ToString, -{ - fn kind(&self) -> String { - self.to_string() - } -} - -#[derive( - AsRefStr, - Clone, - Debug, - Default, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - VariantNames, -)] -#[non_exhaustive] -#[repr(usize)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "lowercase", tag = "kind") -)] -#[strum(serialize_all = "lowercase")] -pub enum ParamKind { - #[default] - Bias, - Weight, - Other(String), -} - -impl ParamKind { - pub fn bias() -> Self { - Self::Bias - } - - pub fn weight() -> Self { - Self::Weight - } - - pub fn other(name: impl ToString) -> Self { - Self::Other(name.to_string()) - } -} - -impl std::fmt::Display for ParamKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let content = match self { - ParamKind::Bias => "bias", - ParamKind::Weight => "weight", - ParamKind::Other(name) => name, - }; - write!(f, "{}", content) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn test_param_kind_map() { - let name = "test"; - let other = ParamKind::other(name); - - let data = [ - (ParamKind::Bias, 0), - (ParamKind::Weight, 1), - (other.clone(), 2), - (ParamKind::other("mask"), 3), - ]; - let store = HashMap::::from_iter(data); - assert_eq!(store.get(&ParamKind::Bias), Some(&0)); - assert_eq!(store.get(&other), Some(&2)); - } -} diff --git a/.artifacts/archive/neural/src/params/masks/mask.rs b/.artifacts/archive/neural/src/params/masks/mask.rs deleted file mode 100644 index 54de8921..00000000 --- a/.artifacts/archive/neural/src/params/masks/mask.rs +++ /dev/null @@ -1,148 +0,0 @@ -/* - Appellation: mask - Contrib: FL03 -*/ -use core::ops; -use ndarray::prelude::{Array, Array2, Dimension}; -use ndarray::ScalarOperand; -use ndarray_rand::rand_distr::uniform::{SampleUniform, Uniform}; -use ndarray_rand::RandomExt; -use num::traits::{Float, NumOps}; -use smart_default::SmartDefault; -use strum::EnumIs; - -#[derive(Clone, Debug, EnumIs, PartialEq, SmartDefault)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub enum Mask { - Masked(Array2), - #[default] - Unmasked, -} - -impl Mask -where - T: NumOps + ScalarOperand, -{ - pub fn forward(&self, data: &Array2) -> Array2 { - match self { - Self::Masked(bias) => data + bias, - Self::Unmasked => data.clone(), - } - } -} - -impl Mask -where - T: Float + SampleUniform, -{ - pub fn uniform(size: usize) -> Self { - let ds = (T::from(size).unwrap()).sqrt(); - let dist = Uniform::new(-ds, ds); - let mask = Array2::::random((size, size), dist); - Self::Masked(mask) - } -} - -impl From for Mask -where - T: Float + SampleUniform, -{ - fn from(size: usize) -> Self { - let ds = (T::from(size).unwrap()).sqrt(); - let dist = Uniform::new(-ds, ds); - let mask = Array2::::random((size, size), dist); - Self::Masked(mask) - } -} - -impl From> for Mask { - fn from(bias: Array2) -> Self { - Self::Masked(bias) - } -} - -impl From>> for Mask { - fn from(bias: Option>) -> Self { - match bias { - Some(bias) => Self::Masked(bias), - None => Self::Unmasked, - } - } -} - -impl From> for Option> { - fn from(bias: Mask) -> Self { - match bias { - Mask::Masked(bias) => Some(bias), - Mask::Unmasked => None, - } - } -} - -impl ops::Add> for Mask -where - D: Dimension, - T: NumOps + ScalarOperand, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, rhs: Array) -> Self::Output { - use Mask::*; - if let Masked(bias) = self { - return rhs.clone() + bias; - } - rhs.clone() - } -} - -impl ops::Add<&Array> for Mask -where - D: Dimension, - T: NumOps + ScalarOperand, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, rhs: &Array) -> Self::Output { - use Mask::*; - if let Masked(bias) = self { - return rhs.clone() + bias; - } - rhs.clone() - } -} - -impl ops::Add> for Array -where - D: Dimension, - T: NumOps + ScalarOperand, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, bias: Mask) -> Self::Output { - use Mask::*; - if let Masked(bias) = bias { - return self.clone() + bias; - } - self.clone() - } -} - -impl ops::Add<&Mask> for Array -where - D: Dimension, - T: NumOps + ScalarOperand, - Array: ops::Add, Output = Array>, -{ - type Output = Array; - - fn add(self, bias: &Mask) -> Self::Output { - use Mask::*; - if let Masked(m) = bias.clone() { - return self.clone() + m; - } - self.clone() - } -} diff --git a/.artifacts/archive/neural/src/params/masks/mod.rs b/.artifacts/archive/neural/src/params/masks/mod.rs deleted file mode 100644 index db9ce4cd..00000000 --- a/.artifacts/archive/neural/src/params/masks/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -/* - Appellation: masks - Contrib: FL03 -*/ -//! # Mask -pub use self::{mask::*, utils::*}; - -pub(crate) mod mask; - -pub trait Masked { - fn mask(&self) -> &Mask; - fn mask_mut(&mut self) -> &mut Mask; -} - -pub(crate) mod utils { - use super::Mask; - use ndarray::prelude::Array2; - use ndarray_rand::rand_distr::uniform::SampleUniform; - use ndarray_rand::rand_distr::Uniform; - use ndarray_rand::RandomExt; - use num::Float; - - pub fn rmask_uniform(features: usize) -> Mask - where - T: Float + SampleUniform, - { - let ds = (T::one() / T::from(features).unwrap()).sqrt(); - Mask::from(Array2::::random( - (features, features), - Uniform::new(-ds, ds), - )) - } -} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/neural/src/params/mod.rs b/.artifacts/archive/neural/src/params/mod.rs deleted file mode 100644 index 0c3c1373..00000000 --- a/.artifacts/archive/neural/src/params/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -//! # Parameters -//! -//! ## Overview -//! -pub use self::{iter::*, kinds::*, param::*, variable::*}; - -pub(crate) mod iter; -pub(crate) mod kinds; -pub(crate) mod param; -pub(crate) mod variable; - -pub mod masks; -pub mod store; - -use ndarray::{Array, ArrayBase, Data, Dimension}; -use std::collections::HashMap; - -pub trait Param { - type Dim: Dimension; - - fn kind(&self) -> &ParamKind; - - fn name(&self) -> &str; -} - -pub trait Biased { - type Dim: Dimension; - /// Returns an owned reference to the bias of the layer. - fn bias(&self) -> &Array; - /// Returns a mutable reference to the bias of the layer. - fn bias_mut(&mut self) -> &mut Array; - /// Sets the bias of the layer. - fn set_bias(&mut self, bias: Array); -} - -pub trait Weighted { - type Dim: Dimension; - /// Returns an owned reference to the weights of the layer. - fn weights(&self) -> &Array; - /// Returns a mutable reference to the weights of the layer. - fn weights_mut(&mut self) -> &mut Array; - /// Sets the weights of the layer. - fn set_weights(&mut self, weights: Array); -} - -pub trait Params { - fn get(&self, param: &K) -> Option<&V>; - - fn get_mut(&mut self, param: &K) -> Option<&mut V>; -} - -/* - ********* implementations ********* -*/ -impl Params> for HashMap> -where - S: Data, - D: Dimension, - K: core::cmp::Eq + core::hash::Hash, -{ - fn get(&self, param: &K) -> Option<&ArrayBase> { - self.get(param) - } - - fn get_mut(&mut self, param: &K) -> Option<&mut ArrayBase> { - self.get_mut(param) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use concision::linarr; - use ndarray::linalg::Dot; - use ndarray::prelude::{Ix1, Ix2}; - - #[test] - fn test_parameter() { - let a = linarr::((3,)).unwrap(); - let p = linarr::((3, 3)).unwrap(); - let mut param = Parameter::::new((10, 1), ParamKind::Bias, "bias"); - param.set_params(p.clone()); - - assert_eq!(param.kind(), &ParamKind::Bias); - assert_eq!(param.name(), "bias"); - assert_eq!(param.dot(&a), p.dot(&a)); - } -} diff --git a/.artifacts/archive/neural/src/params/param.rs b/.artifacts/archive/neural/src/params/param.rs deleted file mode 100644 index 15eb30e3..00000000 --- a/.artifacts/archive/neural/src/params/param.rs +++ /dev/null @@ -1,138 +0,0 @@ -/* - Appellation: param - Contrib: FL03 -*/ -use super::{Param, ParamKind}; -use concision::GenerateRandom; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Dimension, Ix2}; -use ndarray::IntoDimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -#[derive(Clone, Debug, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Parameter -where - T: Float, - D: Dimension, -{ - features: D, - kind: ParamKind, - name: String, - params: Array, -} - -impl Parameter -where - T: Float, - D: Dimension, -{ - pub fn new( - features: impl IntoDimension, - kind: ParamKind, - name: impl ToString, - ) -> Self { - let features = features.into_dimension(); - Self { - features: features.clone(), - kind, - name: name.to_string(), - params: Array::zeros(features), - } - } - - pub fn kind(&self) -> &ParamKind { - &self.kind - } - - pub fn kind_mut(&mut self) -> &mut ParamKind { - &mut self.kind - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn name_mut(&mut self) -> &mut String { - &mut self.name - } - - pub fn params(&self) -> &Array { - &self.params - } - - pub fn params_mut(&mut self) -> &mut Array { - &mut self.params - } - - pub fn set_kind(&mut self, kind: ParamKind) { - self.kind = kind; - } - - pub fn set_name(&mut self, name: String) { - self.name = name; - } - - pub fn set_params(&mut self, params: Array) { - self.params = params; - } - - pub fn with_kind(mut self, kind: ParamKind) -> Self { - self.kind = kind; - self - } - - pub fn with_name(mut self, name: impl ToString) -> Self { - self.name = name.to_string(); - self - } - - pub fn with_params(mut self, params: Array) -> Self { - self.params = params; - self - } -} - -impl Parameter -where - D: Dimension, - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init_uniform(mut self, dk: T) -> Self { - let dim = self.params.dim(); - self.params = Array::uniform_between(dk, dim); - self - } -} - -impl Param for Parameter -where - T: Float, - D: Dimension, -{ - type Dim = D; - - fn kind(&self) -> &ParamKind { - &self.kind - } - - fn name(&self) -> &str { - &self.name - } -} - -impl Dot for Parameter -where - Array: Dot, - D: Dimension, - T: Float, -{ - type Output = O; - - fn dot(&self, rhs: &S) -> Self::Output { - self.params.dot(rhs) - } -} diff --git a/.artifacts/archive/neural/src/params/store.rs b/.artifacts/archive/neural/src/params/store.rs deleted file mode 100644 index 1a13e701..00000000 --- a/.artifacts/archive/neural/src/params/store.rs +++ /dev/null @@ -1,104 +0,0 @@ -/* - Appellation: store - Contrib: FL03 -*/ - -pub mod group; - -use crate::params::{ParamKind, Parameter}; -#[cfg(not(feature = "std"))] -use alloc::collections::{btree_map, BTreeMap}; -use ndarray::prelude::{Dimension, Ix2}; -use num::Float; -#[cfg(feature = "std")] -use std::collections::{btree_map, BTreeMap}; - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct ParamStore -where - T: Float, - D: Dimension, -{ - store: BTreeMap>, -} - -impl ParamStore -where - D: Dimension, - T: Float, -{ - pub fn new() -> Self { - Self { - store: BTreeMap::new(), - } - } - - pub fn get(&self, kind: &ParamKind) -> Option<&Parameter> { - self.store.get(kind) - } - - pub fn get_mut(&mut self, kind: &ParamKind) -> Option<&mut Parameter> { - self.store.get_mut(kind) - } - - pub fn insert(&mut self, param: Parameter) { - self.store.insert(param.kind().clone(), param); - } - - pub fn remove(&mut self, kind: &ParamKind) -> Option> { - self.store.remove(kind) - } -} - -impl Extend> for ParamStore -where - D: Dimension, - T: Float, -{ - fn extend>>(&mut self, iter: I) { - for param in iter { - self.insert(param); - } - } -} - -impl IntoIterator for ParamStore -where - D: Dimension, - T: Float, -{ - type Item = (ParamKind, Parameter); - type IntoIter = btree_map::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.store.into_iter() - } -} - -impl<'a, T, D> IntoIterator for &'a mut ParamStore -where - D: Dimension, - T: Float, -{ - type Item = (&'a ParamKind, &'a mut Parameter); - type IntoIter = btree_map::IterMut<'a, ParamKind, Parameter>; - - fn into_iter(self) -> Self::IntoIter { - self.store.iter_mut() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_model_store() { - let (inputs, outputs) = (5, 3); - - let _shapes = [(inputs, outputs), (outputs, outputs), (outputs, 1)]; - - let _params = ParamStore::::new(); - } -} diff --git a/.artifacts/archive/neural/src/params/store/group.rs b/.artifacts/archive/neural/src/params/store/group.rs deleted file mode 100644 index d6716891..00000000 --- a/.artifacts/archive/neural/src/params/store/group.rs +++ /dev/null @@ -1,183 +0,0 @@ -/* - Appellation: group - Contrib: FL03 -*/ -use crate::params::{Biased, Weighted}; -use concision::GenerateRandom; -use core::ops; -use ndarray::linalg::Dot; -use ndarray::{Array, Axis, Dimension, IntoDimension, Ix2, NdFloat, RemoveAxis}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ParamGroup -where - D: Dimension, -{ - bias: Array, - features: D, - weights: Array, -} - -impl ParamGroup -where - T: Float, - D: RemoveAxis, -{ - pub fn new(dim: impl IntoDimension) -> Self { - let dim = dim.into_dimension(); - Self { - bias: Array::zeros(dim.remove_axis(Axis(dim.ndim() - 1))), - features: dim.clone(), - weights: Array::zeros(dim), - } - } -} - -impl ParamGroup -where - T: Float, - D: Dimension, -{ - pub fn features(&self) -> &D { - &self.features - } - - pub fn inputs(&self) -> usize { - self.weights.shape().last().unwrap().clone() - } - - pub fn outputs(&self) -> usize { - if self.features.ndim() == 1 { - return 1; - } - self.weights.shape().first().unwrap().clone() - } -} - -impl ParamGroup -where - D: Dimension, - T: NdFloat, - Self: Biased + Weighted, -{ - pub fn linear(&self, data: &Array) -> Array - where - Array: Dot, Output = Array> - + ops::Add, Output = Array>, - { - data.dot(&self.weights().t().to_owned()) + self.bias().clone() - } -} - -impl ParamGroup -where - D: Dimension + RemoveAxis, - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); - } - self.init_weight() - } - - pub fn init_bias(mut self) -> Self { - let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); - self.bias = Array::uniform_between( - dk, - self.features() - .remove_axis(Axis(self.features().ndim() - 1)) - .clone(), - ); - self - } - - pub fn init_weight(mut self) -> Self { - let dk = (T::one() / T::from(self.inputs()).unwrap()).sqrt(); - self.weights = Array::uniform_between(dk, self.features().clone()); - self - } -} - -impl Biased for ParamGroup -where - D: RemoveAxis, - T: Float, -{ - type Dim = D::Smaller; - - fn bias(&self) -> &Array { - &self.bias - } - - fn bias_mut(&mut self) -> &mut Array { - &mut self.bias - } - - fn set_bias(&mut self, bias: Array) { - self.bias = bias; - } -} - -impl Weighted for ParamGroup -where - D: Dimension, - T: Float, -{ - type Dim = D; - - fn weights(&self) -> &Array { - &self.weights - } - - fn weights_mut(&mut self) -> &mut Array { - &mut self.weights - } - - fn set_weights(&mut self, weights: Array) { - self.weights = weights; - } -} - -#[cfg(feature = "serde")] - -mod impl_serde { - use super::*; - use serde::{Deserialize, Serialize}; - impl<'a, T, D> Deserialize<'a> for ParamGroup - where - T: Deserialize<'a> + Float, - D: Deserialize<'a> + Dimension, - ::Smaller: Deserialize<'a> + Dimension, - { - fn deserialize(deserializer: Der) -> Result - where - Der: serde::Deserializer<'a>, - { - let (bias, features, weights) = Deserialize::deserialize(deserializer)?; - Ok(Self { - bias, - features, - weights, - }) - } - } - - impl Serialize for ParamGroup - where - T: Float + Serialize, - D: Dimension + RemoveAxis + Serialize, - ::Smaller: Dimension + Serialize, - { - fn serialize(&self, serializer: Ser) -> Result - where - Ser: serde::Serializer, - { - (self.bias(), self.features(), self.weights()).serialize(serializer) - } - } -} diff --git a/.artifacts/archive/neural/src/params/variable.rs b/.artifacts/archive/neural/src/params/variable.rs deleted file mode 100644 index a2394f90..00000000 --- a/.artifacts/archive/neural/src/params/variable.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - Appellation: variable - Contrib: FL03 -*/ -//! # Variables -//! -//! ## Overview -//! Variables extend the functionality of the 'Parameter' by enabling mutability. -//! - -pub struct Variable; - -pub enum P { - Param, - Variable(Box), -} diff --git a/.artifacts/archive/neural/src/primitives.rs b/.artifacts/archive/neural/src/primitives.rs deleted file mode 100644 index fa82351d..00000000 --- a/.artifacts/archive/neural/src/primitives.rs +++ /dev/null @@ -1,28 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, types::*}; - -mod constants { - pub const DEFAULT_BUFFER: usize = 1024; -} - -mod types { - use crate::prelude::{Forward, MlError}; - use ndarray::prelude::{Array, Ix2}; - - pub type BoxError = Box; - - pub type BoxResult = core::result::Result; - - pub type BoxedFunction = Box T>; - /// - pub type ForwardDyn = Box, Output = Array>>; - /// - pub type LayerBias = ndarray::Array1; - - pub type LayerWeight = ndarray::Array2; - - pub type Result = core::result::Result; -} diff --git a/.artifacts/archive/neural/src/specs.rs b/.artifacts/archive/neural/src/specs.rs deleted file mode 100644 index 899f8559..00000000 --- a/.artifacts/archive/neural/src/specs.rs +++ /dev/null @@ -1,66 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ -use crate::ops::{Compile, Predict}; - -pub trait Backward: Forward { - fn backward(&mut self, args: &T, grad: &T); -} - -pub trait Forward { - type Output; - - fn forward(&self, args: &T) -> Self::Output; -} - -pub trait ForwardIter: Forward + IntoIterator -where - I: Forward, - T: Clone, -{ - fn forward_iter(&self, args: &T) -> Vec; -} - -impl ForwardIter for S -where - S: Clone + Forward + IntoIterator, - I: Forward, - T: Clone, -{ - fn forward_iter(&self, args: &T) -> Vec { - let mut store = vec![args.clone()]; - - for item in self.clone().into_iter() { - let res = item.forward(store.last().unwrap()); - store.push(res) - } - - store - } -} - -// impl ForwardIter for S -// where -// S: Forward + IntoIterator>, -// { -// } - -pub trait Batched { - type Output; - - fn batch(&self, batch_size: usize) -> Self::Output; -} - -pub trait Module: Compile + Predict { - type Config; - type Params; - - fn config(&self) -> &Self::Config; - - fn id(&self) -> &str; - - fn name(&self) -> &str; - - fn params(&self) -> &Self::Params; -} diff --git a/.artifacts/archive/neural/src/utils.rs b/.artifacts/archive/neural/src/utils.rs deleted file mode 100644 index 9b7b2532..00000000 --- a/.artifacts/archive/neural/src/utils.rs +++ /dev/null @@ -1,24 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use ndarray::prelude::Array; -use ndarray::{Dimension, IntoDimension}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::Uniform; -use ndarray_rand::RandomExt; -use num::Float; - -pub fn generate_uniform_arr(axis: usize, dim: impl IntoDimension) -> Array -where - D: Dimension, - T: Float + SampleUniform, -{ - let shape: D = dim.into_dimension(); - let dk = { - let k = T::from(shape[axis]).unwrap(); - (T::one() / k).sqrt() - }; - let dist = Uniform::new(-dk, dk); - Array::::random(shape, dist) -} diff --git a/.artifacts/archive/neural/tests/default.rs b/.artifacts/archive/neural/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/.artifacts/archive/neural/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/.artifacts/archive/nlp/Cargo.toml b/.artifacts/archive/nlp/Cargo.toml deleted file mode 100644 index 5aa6fc48..00000000 --- a/.artifacts/archive/nlp/Cargo.toml +++ /dev/null @@ -1,73 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-nlp" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [ - "std" -] - -full = [ - "default", - "serde", -] -blas = [ - "ndarray/blas", -] - -serde = [ - "dep:serde", - "serde-ext", -] - -serde-ext = [ - "dep:serde_json", - "ndarray/serde-1", - "num/serde", -] - -std = [ - "ndarray/std", - "num/std", - "strum/std", -] - -[lib] -bench = false -crate-type = ["rlib"] -doctest = false -test = false - -[build-dependencies] - -[dependencies] - -anyhow.workspace = true -finalfusion = "0.18" -ndarray = "0.15" -num = { features = ["rand"], version = "0.4" } -serde = { features = ["derive"], optional = true, version = "1" } -serde_json = { optional = true, version = "1" } -smart-default.workspace = true -strum.workspace = true -tokenizers = "0.19" - -[dev-dependencies] -concision-core = { path = "../../core", version = "0.1.12" } - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] diff --git a/.artifacts/archive/nlp/src/embed/context/mod.rs b/.artifacts/archive/nlp/src/embed/context/mod.rs deleted file mode 100644 index 8ccae58f..00000000 --- a/.artifacts/archive/nlp/src/embed/context/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: context - Contrib: FL03 -*/ -//! # Contextual Embedding -//! diff --git a/.artifacts/archive/nlp/src/embed/embedding.rs b/.artifacts/archive/nlp/src/embed/embedding.rs deleted file mode 100644 index dfc4cc98..00000000 --- a/.artifacts/archive/nlp/src/embed/embedding.rs +++ /dev/null @@ -1,35 +0,0 @@ -/* - Appellation: embedding - Contrib: FL03 -*/ -use ndarray::{Array2, Dim, IntoDimension}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -pub struct Embedding { - data: Array2, -} - -impl Embedding { - pub fn new(dim: Dim<[usize; 2]>) -> Self { - Self { - data: Array2::zeros(dim), - } - } -} - -impl std::fmt::Display for Embedding { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) - } -} - -impl From for Embedding -where - D: IntoDimension>, -{ - fn from(dim: D) -> Self { - Self::new(dim.into_dimension()) - } -} diff --git a/.artifacts/archive/nlp/src/embed/mod.rs b/.artifacts/archive/nlp/src/embed/mod.rs deleted file mode 100644 index 2b414b65..00000000 --- a/.artifacts/archive/nlp/src/embed/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: embed - Contrib: FL03 -*/ -//! # Embedding -//! -//! Natural language processing applications rely heavily on the use of embeddings. -//! Embeddings are generally broken down into two categories: contextual and static. -//! Popular static solutions such as word2vec and GloVe are used to create a static -//! embedding of a word or phrase. Contextual embeddings, on the other hand, are -//! generated by a model that takes into account the context of the word or phrase -//! in question. This module provides a variety of embedding solutions for use in -//! natural language processing applications. -pub use self::embedding::*; - -pub(crate) mod embedding; - -pub mod context; -pub mod words; - -pub trait Embed { - fn embed(&self) -> Embedding; -} diff --git a/.artifacts/archive/nlp/src/embed/words/mod.rs b/.artifacts/archive/nlp/src/embed/words/mod.rs deleted file mode 100644 index efbd8c89..00000000 --- a/.artifacts/archive/nlp/src/embed/words/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -/* - Appellation: words - Contrib: FL03 -*/ -//! # Word Embedding -//! - -pub mod word2vec; diff --git a/.artifacts/archive/nlp/src/embed/words/word2vec.rs b/.artifacts/archive/nlp/src/embed/words/word2vec.rs deleted file mode 100644 index 7d123428..00000000 --- a/.artifacts/archive/nlp/src/embed/words/word2vec.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - Appellation: word2vec - Contrib: FL03 -*/ -//! # word2vec -//! -//! word2vec is a popular static embedding solution that is used in a variety of -//! natural language processing applications. This module provides a word2vec -//! implementation that can be used to generate static embeddings for words and -//! phrases. -//! -//! ## Methods -//! -//! ### Continuous Bag of Words (CBOW) -//! -//! ### Skip-Gram diff --git a/.artifacts/archive/nlp/src/encode/mod.rs b/.artifacts/archive/nlp/src/encode/mod.rs deleted file mode 100644 index f5dbeeac..00000000 --- a/.artifacts/archive/nlp/src/encode/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: encode - Contrib: FL03 -*/ -pub mod positional; - -use ndarray::prelude::{Array, Array2}; -use ndarray::Dimension; - -pub trait Encode { - type Output; - - fn encode(&self, data: &T) -> Self::Output; -} - -pub trait EncodeArr { - type Dim: Dimension; - - fn encode(&self, data: &Array) -> Array2; -} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/nlp/src/encode/positional.rs b/.artifacts/archive/nlp/src/encode/positional.rs deleted file mode 100644 index f647344a..00000000 --- a/.artifacts/archive/nlp/src/encode/positional.rs +++ /dev/null @@ -1,164 +0,0 @@ -/* - Appellation: positional - Contrib: FL03 -*/ -use ndarray::prelude::Array2; -use ndarray::ScalarOperand; -use num::Float; -use serde::{Deserialize, Serialize}; - -pub fn create_positional( - model: usize, - seq_len: usize, - samples: Option, -) -> Array2 { - let n = T::from(samples.unwrap_or(10000)).unwrap(); - let d = T::from(model).unwrap(); - let denom = |pos: T, x: T| pos / T::powf(n, (T::from(2).unwrap() * x) / d); - let mut p = Array2::zeros((seq_len, model)); - for i in 0..seq_len { - for j in 0..model / 2 { - let u = T::from(i).unwrap(); - let v = T::from(j).unwrap(); - p[[i, 2 * j]] = denom(u, v).sin(); - p[[i, 2 * j + 1]] = denom(u, v + T::one()).cos(); - } - } - p -} - -#[derive(Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)] -pub struct PositionalEncoder { - params: PositionalEncoderParams, - pe: Array2, -} - -impl PositionalEncoder -where - T: Float, -{ - pub fn new(model: usize, sequence: usize, samples: usize) -> Self { - Self { - params: PositionalEncoderParams::new(model, sequence, samples), - pe: Array2::zeros((sequence, model)), - } - } - - pub fn init(mut self) -> Self { - self.pe = create_positional::( - self.params.model(), - self.params.sequence(), - Some(self.params.samples()), - ); - self - } - - pub fn params(&self) -> PositionalEncoderParams { - self.params - } - - pub fn positional(&self) -> &Array2 { - &self.pe - } -} - -impl PositionalEncoder -where - T: Float + ScalarOperand, -{ - pub fn encode(&self, data: &Array2) -> Array2 { - let x = data * T::from(self.params().model()).unwrap().sqrt(); - x + self.positional() - } -} - -pub trait IntoParams { - type Params; - - fn into_params(self) -> Self::Params; -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct PositionalEncoderParams { - pub model: usize, - pub sequence: usize, - pub samples: usize, -} - -impl PositionalEncoderParams { - pub fn new(model: usize, sequence: usize, samples: usize) -> Self { - Self { - model, - sequence, - samples, - } - } - - pub fn std(sequence: usize) -> Self { - Self::new(512, sequence, 10000) - } - - pub fn model(&self) -> usize { - self.model - } - - pub fn sequence(&self) -> usize { - self.sequence - } - - pub fn samples(&self) -> usize { - self.samples - } - - pub fn set_model(&mut self, model: usize) { - self.model = model; - } - - pub fn set_sequence(&mut self, sequence: usize) { - self.sequence = sequence; - } - - pub fn set_samples(&mut self, samples: usize) { - self.samples = samples; - } - - pub fn with_model(mut self, model: usize) -> Self { - self.model = model; - self - } - - pub fn with_sequence(mut self, sequence: usize) -> Self { - self.sequence = sequence; - self - } - - pub fn with_samples(mut self, samples: usize) -> Self { - self.samples = samples; - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::RoundTo; - use ndarray::prelude::{array, Array}; - - #[test] - fn test_positional_encoding() { - let data = Array::linspace(1., 4., 4).into_shape((1, 4)).unwrap(); - let encoder = PositionalEncoder::new(4, 4, 10000).init(); - - let pe = encoder.positional(); - assert_eq!(pe.dim(), (4, 4)); - assert_eq!(pe.row(0), array![0.0, 1.0, 0.0, 1.0]); - - let encoded = encoder.encode(&data); - let rounded = encoded.mapv(|x| x.round_to(4)); - - assert_eq!(rounded[[0, 0]], 2.0); - assert_eq!(rounded[[1, 0]], 2.8415); - } -} diff --git a/.artifacts/archive/nlp/src/lib.rs b/.artifacts/archive/nlp/src/lib.rs deleted file mode 100644 index 75150baa..00000000 --- a/.artifacts/archive/nlp/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - Appellation: concision-nlp - Contrib: FL03 -*/ -//! # Natural Language Processing - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod embed; -pub mod encode; - -#[cfg(test)] -pub use concision_core as core; - -pub mod prelude { - pub use crate::embed::*; - pub use crate::encode::*; -} diff --git a/.artifacts/archive/nlp/src/primitives.rs b/.artifacts/archive/nlp/src/primitives.rs deleted file mode 100644 index 51d14049..00000000 --- a/.artifacts/archive/nlp/src/primitives.rs +++ /dev/null @@ -1,10 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ - -mod constants {} - -mod statics {} - -mod types {} diff --git a/.artifacts/archive/nlp/src/specs.rs b/.artifacts/archive/nlp/src/specs.rs deleted file mode 100644 index 1d8faa71..00000000 --- a/.artifacts/archive/nlp/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ diff --git a/.artifacts/archive/nlp/src/utils.rs b/.artifacts/archive/nlp/src/utils.rs deleted file mode 100644 index 752dabaf..00000000 --- a/.artifacts/archive/nlp/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ diff --git a/.artifacts/archive/nlp/tests/default.rs b/.artifacts/archive/nlp/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/.artifacts/archive/nlp/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/.artifacts/archive/nn/Cargo.toml b/.artifacts/archive/nn/Cargo.toml deleted file mode 100644 index f6daa451..00000000 --- a/.artifacts/archive/nn/Cargo.toml +++ /dev/null @@ -1,122 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-nn" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [ - "std", -] - -full = [ - "default", - "approx", - "nlp", - "optim", - "serde", - "transformers", -] - -approx = [] - -nlp = [ - "dep:concision-nlp" -] - -optim = [ - "dep:concision-optim", -] - -s4 = [ - "dep:concision-s4", -] - -serde = [] - -std = [] - -transformers = [ - "dep:concision-transformers" -] - -blas = [ - "concision-neural/blas", - "concision-nlp/blas", - "concision-optim/blas", - "concision-s4/blas", - "transformers?/blas", -] - -intel-mkl-system = [ - "blas", - "concision-neural/intel-mkl-system", - "concision-s4/intel-mkl-system", -] - -intel-mkl-static = [ - "blas", - "concision-neural/intel-mkl-static", - "concision-s4/intel-mkl-static", -] - -netlib-system = [ - "blas", - "concision-neural/netlib-system", - "concision-s4/netlib-system", -] - -netlib-static = [ - "blas", - "concision-neural/netlib-static", - "concision-s4/netlib-static", -] - -openblas-system = [ - "blas", - "concision-neural/openblas-system", - "concision-s4/openblas-system", -] - -openblas-static = [ - "blas", - "concision-neural/openblas-static", - "concision-s4/openblas-static", -] - -[lib] -bench = false -crate-type = ["rlib"] -doctest = false -test = false - -[build-dependencies] - -[dependencies] -concision-neural = { path = "../neural", version = "0.1.12" } -concision-nlp = { optional = true, path = "../nlp", version = "0.1.12" } -concision-optim = { optional = true, path = "../optim", version = "0.1.12" } -concision-s4 = { optional = true, path = "../s4", version = "0.1.12" } -transformers = { optional = true, path = "../transformers", version = "0.1.12" } - - -[dev-dependencies] -anyhow = "1" -approx = "0.5" -concision = { path = "../../../concision" } -ndarray = { features = ["approx-0_5", "serde-1"], version = "0.15" } - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] \ No newline at end of file diff --git a/.artifacts/archive/nn/examples/basic.rs b/.artifacts/archive/nn/examples/basic.rs deleted file mode 100644 index 0f4e9e89..00000000 --- a/.artifacts/archive/nn/examples/basic.rs +++ /dev/null @@ -1,93 +0,0 @@ -/* - Appellation: gradients - Contrib: FL03 -*/ -extern crate concision as cnc; -extern crate concision_nn as nn; - -use cnc::prelude::linarr; -use nn::models::{Model, ModelConfig, ModelParams}; -use nn::prelude::{Features, Forward, Layer, LayerShape, Sigmoid}; - -use ndarray::prelude::{Array1, Ix2}; - -fn main() -> anyhow::Result<()> { - let (samples, inputs) = (20, 8); - let outputs = 4; - - let features = LayerShape::new(inputs, outputs); - - let (epochs, gamma) = (100000, 0.0005); - - // sample_gradient(epochs, features, gamma, samples)?; - - sample_model(epochs, features, gamma, samples)?; - - Ok(()) -} - -pub fn sample_gradient( - epochs: usize, - features: LayerShape, - gamma: f64, - samples: usize, -) -> anyhow::Result<()> { - // Generate some example data - let x = linarr::((samples, features.inputs()))?; - let mut y = linarr::((samples, features.outputs()))?; - y.map_inplace(|ys| *ys = ys.powi(2)); - - let mut model = Layer::::from(features).init(false); - println!( - "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", - &y.shape(), - model.forward(&x) - ); - - let mut losses = Array1::zeros(epochs); - for e in 0..epochs { - // let cost = gradient(gamma, &mut model, &x, &y, Sigmoid); - let cost = model.grad(gamma, &x, &y); - losses[e] = cost; - } - println!("Losses:\n\n{:?}\n", &losses); - println!("Trained:\n\n{:?}", model.forward(&x)); - Ok(()) -} - -pub fn sample_model( - epochs: usize, - features: LayerShape, - gamma: f64, - samples: usize, -) -> anyhow::Result<()> { - let mut losses = Array1::zeros(epochs); - - // Generate some example data - let x = linarr::((samples, features.inputs()))?; - let y = linarr::((samples, features.outputs()))?; - - let mut shapes = vec![features]; - shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - - let config = ModelConfig::new(4); - let params = ModelParams::::from_iter(shapes); - let mut model = Model::::new(config).with_params(params).init(false); - // let mut opt = Grad::new(gamma, model.clone(), Sigmoid); - - println!( - "Targets (dim):\t{:?}\nPredictions:\n\n{:?}\n", - &y.shape(), - model.forward(&x) - ); - - for e in 0..epochs { - let cost = model.gradient(&x, &y, gamma, Sigmoid)?; - // let cost = opt.step(&x, &y)?; - // let cost = model.grad(gamma, &x, &y); - losses[e] = cost; - } - println!("Losses:\n\n{:?}\n", &losses); - println!("Trained:\n\n{:?}", model.forward(&x)); - Ok(()) -} diff --git a/.artifacts/archive/nn/src/lib.rs b/.artifacts/archive/nn/src/lib.rs deleted file mode 100644 index 5f29aacb..00000000 --- a/.artifacts/archive/nn/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -/* - Appellation: concision - Contrib: FL03 -*/ -//! # Concision -//! -//! Concision aims to be a complete machine learning library written in pure Rust. -//! -#![crate_name = "concision_nn"] - -pub use concision_neural::*; -#[cfg(feature = "nlp")] -pub use concision_nlp as nlp; -#[cfg(feature = "optim")] -pub use concision_optim as optim; -#[cfg(feature = "s4")] -pub use concision_s4 as s4; -#[cfg(feature = "transformers")] -pub use transformers as transformers; - -pub mod prelude { - pub use concision_neural::prelude::*; - #[cfg(feature = "nlp")] - pub use concision_nlp::prelude::*; - #[cfg(feature = "optim")] - pub use concision_optim::prelude::*; - #[cfg(feature = "s4")] - pub use concision_s4::prelude::*; - #[cfg(feature = "transformers")] - pub use concision_transformers::prelude::*; -} diff --git a/.artifacts/archive/nn/tests/default.rs b/.artifacts/archive/nn/tests/default.rs deleted file mode 100644 index e68a9cf5..00000000 --- a/.artifacts/archive/nn/tests/default.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: default - Contrib: FL03 -*/ -#![cfg(test)] - -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/.artifacts/archive/optim/Cargo.toml b/.artifacts/archive/optim/Cargo.toml deleted file mode 100644 index 0da4d329..00000000 --- a/.artifacts/archive/optim/Cargo.toml +++ /dev/null @@ -1,55 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "concision-optim" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -blas = [ - "ndarray/blas", - "concision-core/blas", - "concision-neural/blas", -] - -[lib] -bench = false -crate-type = ["rlib"] -doctest = false -test = false - -[build-dependencies] - -[dependencies] -concision-core = { features = ["full"], path = "../../core", version = "0.1.12" } -concision-neural = { path = "../neural" } - -anyhow = "1" -itertools = "0.12" -ndarray = { features = ["serde-1"], version = "0.15" } -ndarray-rand = "0.14" -ndarray-stats = "0.5" -num = { features = ["rand", "serde"], version = "0.4" } -rand = "0.8" -serde = { features = ["derive"], version = "1" } -serde_json = "1" -smart-default = "0.7" -strum = { features = ["derive"], version = "0.26" } - -[dev-dependencies] - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] diff --git a/.artifacts/archive/optim/examples/norm.rs b/.artifacts/archive/optim/examples/norm.rs deleted file mode 100644 index 827b9a59..00000000 --- a/.artifacts/archive/optim/examples/norm.rs +++ /dev/null @@ -1,41 +0,0 @@ -use concision_neural::prelude::{Features, LayerShape}; -use concision_optim::prelude::Norm; -use ndarray::prelude::Array; - -fn main() -> anyhow::Result<()> { - let (samples, inputs) = (20, 3); - let outputs = 8; - - let features = LayerShape::new(inputs, outputs); - - // basic_descent(epochs, features, gamma)?; - - sample_norm(features, samples)?; - - Ok(()) -} - -pub fn sample_norm(features: impl Features, samples: usize) -> anyhow::Result<()> { - let n = samples * features.inputs(); - let args = Array::linspace(1., n as f64, n) - .into_shape((samples, features.inputs())) - .unwrap(); - - println!( - "Norms:\n\nL0: {:?}\nL1: {:?}\nL2: {:?}\n", - &args.l0(), - &args.l1(), - &args.l2() - ); - let args = Array::linspace(1., features.inputs() as f64, features.inputs()) - .into_shape(features.inputs()) - .unwrap(); - println!( - "Norms:\n\nL0: {:?}\nL1: {:?}\nL2: {:?}\n", - &args.l0(), - &args.l1(), - &args.l2() - ); - - Ok(()) -} diff --git a/.artifacts/archive/optim/examples/sgd.rs b/.artifacts/archive/optim/examples/sgd.rs deleted file mode 100644 index 791ef83b..00000000 --- a/.artifacts/archive/optim/examples/sgd.rs +++ /dev/null @@ -1,31 +0,0 @@ -use concision_core::prelude::linarr; -use concision_neural::prelude::{Layer, LayerShape, Sigmoid}; -use concision_optim::grad::sgd::sgd; -use ndarray::prelude::Array2; - -fn main() -> anyhow::Result<()> { - let (samples, inputs) = (20, 10); - let outputs = 5; - - let features = LayerShape::new(inputs, outputs); - - let (batch_size, epochs, gamma) = (20, 4, 0.01); - // Generate some example data - let (x, y) = sample_data::(inputs, outputs, samples)?; - - let mut model = Layer::::from(features); - - let cost = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); - println!("Losses {:?}", cost); - Ok(()) -} - -fn sample_data( - inputs: usize, - outputs: usize, - samples: usize, -) -> anyhow::Result<(Array2, Array2)> { - let x = linarr((samples, inputs)).unwrap(); // (samples, inputs) - let y = linarr((samples, outputs)).unwrap(); // (samples, outputs) - Ok((x, y)) -} diff --git a/.artifacts/archive/optim/src/grad/adam.rs b/.artifacts/archive/optim/src/grad/adam.rs deleted file mode 100644 index 8da534e1..00000000 --- a/.artifacts/archive/optim/src/grad/adam.rs +++ /dev/null @@ -1,310 +0,0 @@ -/* - Appellation: adam - Contrib: FL03 -*/ -//! # Adam (Adaptive Moment Estimation) optimizer -//! -//! The `Adam (Adaptive Moment Estimation)` optimizer is an adaptive learning rate algorithm used -//! in gradient descent and machine learning, such as for training neural networks to solve deep -//! learning problems. Boasting memory-efficient fast convergence rates, it sets and iteratively -//! updates learning rates individually for each model parameter based on the gradient history. -//! -//! ## Algorithm: -//! -//! Given: -//! - α is the learning rate -//! - (β_1, β_2) are the exponential decay rates for moment estimates -//! - ϵ is any small value to prevent division by zero -//! - g_t are the gradients at time step t -//! - m_t are the biased first moment estimates of the gradient at time step t -//! - v_t are the biased second raw moment estimates of the gradient at time step t -//! - θ_t are the model parameters at time step t -//! - t is the time step -//! -//! Required: -//! θ_0 -//! -//! Initialize: -//! m_0 <- 0 -//! v_0 <- 0 -//! t <- 0 -//! -//! while θ_t not converged do -//! m_t = β_1 * m_{t−1} + (1 − β_1) * g_t -//! v_t = β_2 * v_{t−1} + (1 − β_2) * g_t^2 -//! m_hat_t = m_t / 1 - β_1^t -//! v_hat_t = v_t / 1 - β_2^t -//! θ_t = θ_{t-1} − α * m_hat_t / (sqrt(v_hat_t) + ϵ) -//! -//! ## Resources: -//! - Adam: A Method for Stochastic Optimization (by Diederik P. Kingma and Jimmy Ba): -//! - [https://arxiv.org/abs/1412.6980] -//! - PyTorch Adam optimizer: -//! - [https://pytorch.org/docs/stable/generated/torch.optim.Adam.html#torch.optim.Adam] -//! -use ndarray::prelude::{Array1, NdFloat}; -use num::Float; - -pub struct Adam -where - T: Float, -{ - learning_rate: T, // alpha: initial step size for iterative optimization - betas: (T, T), // betas: exponential decay rates for moment estimates - epsilon: T, // epsilon: prevent division by zero - m: Array1, // m: biased first moment estimate of the gradient vector - v: Array1, // v: biased second raw moment estimate of the gradient vector - t: usize, // t: time step -} - -impl Adam -where - T: Float, -{ - pub fn new( - learning_rate: Option, - betas: Option<(T, T)>, - epsilon: Option, - params_len: usize, - ) -> Self { - let gamma = T::from(1e-3).unwrap(); - let betas = betas.unwrap_or((T::from(0.9).unwrap(), T::from(0.999).unwrap())); - let epsilon = epsilon.unwrap_or(T::from(1e-8).unwrap()); - Adam { - learning_rate: learning_rate.unwrap_or(gamma), // typical good default lr - betas, // typical good default decay rates - epsilon, // typical good default epsilon - m: Array1::zeros(params_len), // first moment vector elements all initialized to zero - v: Array1::zeros(params_len), // second moment vector elements all initialized to zero - t: 0, // time step initialized to zero - } - } -} - -impl Adam -where - T: NdFloat, -{ - pub fn step(&mut self, gradients: &Array1) -> Array1 { - let mut model_params = Array1::zeros(gradients.len()); - self.t += 1; - - for i in 0..gradients.len() { - // update biased first moment estimate and second raw moment estimate - self.m[i] = self.betas.0 * self.m[i] + (T::one() - self.betas.0) * gradients[i]; - self.v[i] = self.betas.1 * self.v[i] + (T::one() - self.betas.1) * gradients[i].powi(2); - - // compute bias-corrected first moment estimate and second raw moment estimate - let m_hat = self.m[i] / (T::one() - self.betas.0.powi(self.t as i32)); - let v_hat = self.v[i] / (T::one() - self.betas.1.powi(self.t as i32)); - - // update model parameters - model_params[i] -= self.learning_rate * m_hat / (v_hat.sqrt() + self.epsilon); - } - model_params // return updated model parameters - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ndarray::prelude::array; - - #[test] - fn test_adam_init_default_values() { - let optimizer = Adam::new(None, None, None, 1); - - assert_eq!(optimizer.learning_rate, 0.001); - assert_eq!(optimizer.betas, (0.9, 0.999)); - assert_eq!(optimizer.epsilon, 1e-8); - assert_eq!(optimizer.m, array![0.0]); - assert_eq!(optimizer.v, array![0.0]); - assert_eq!(optimizer.t, 0); - } - - #[test] - fn test_adam_init_custom_lr_value() { - let optimizer = Adam::new(Some(0.9), None, None, 2); - - assert_eq!(optimizer.learning_rate, 0.9); - assert_eq!(optimizer.betas, (0.9, 0.999)); - assert_eq!(optimizer.epsilon, 1e-8); - assert_eq!(optimizer.m, array![0.0, 0.0]); - assert_eq!(optimizer.v, array![0.0, 0.0]); - assert_eq!(optimizer.t, 0); - } - - #[test] - fn test_adam_init_custom_betas_value() { - let optimizer = Adam::new(None, Some((0.8, 0.899)), None, 3); - - assert_eq!(optimizer.learning_rate, 0.001); - assert_eq!(optimizer.betas, (0.8, 0.899)); - assert_eq!(optimizer.epsilon, 1e-8); - assert_eq!(optimizer.m, array![0.0, 0.0, 0.0]); - assert_eq!(optimizer.v, array![0.0, 0.0, 0.0]); - assert_eq!(optimizer.t, 0); - } - - #[test] - fn test_adam_init_custom_epsilon_value() { - let optimizer = Adam::new(None, None, Some(1e-10), 4); - - assert_eq!(optimizer.learning_rate, 0.001); - assert_eq!(optimizer.betas, (0.9, 0.999)); - assert_eq!(optimizer.epsilon, 1e-10); - assert_eq!(optimizer.m, array![0.0, 0.0, 0.0, 0.0]); - assert_eq!(optimizer.v, array![0.0, 0.0, 0.0, 0.0]); - assert_eq!(optimizer.t, 0); - } - - #[test] - fn test_adam_init_all_custom_values() { - let optimizer = Adam::new(Some(1.0), Some((0.001, 0.099)), Some(1e-1), 5); - - assert_eq!(optimizer.learning_rate, 1.0); - assert_eq!(optimizer.betas, (0.001, 0.099)); - assert_eq!(optimizer.epsilon, 1e-1); - assert_eq!(optimizer.m, Array1::::zeros(5)); - assert_eq!(optimizer.v, Array1::::zeros(5)); - assert_eq!(optimizer.t, 0); - } - - #[test] - fn test_adam_step_default_params() { - let gradients = array![-1.0, 2.0, -3.0, 4.0, -5.0, 6.0, -7.0, 8.0]; - - let mut optimizer = Adam::new(None, None, None, 8); - let updated_params = optimizer.step(&gradients); - - assert_eq!( - updated_params, - array![ - 0.0009999999900000003, - -0.000999999995, - 0.0009999999966666666, - -0.0009999999975, - 0.000999999998, - -0.0009999999983333334, - 0.0009999999985714286, - -0.00099999999875 - ] - ); - } - - #[test] - fn test_adam_step_custom_params() { - let gradients = array![9.0, -8.0, 7.0, -6.0, 5.0, -4.0, 3.0, -2.0, 1.0]; - - let mut optimizer = Adam::new(Some(0.005), Some((0.5, 0.599)), Some(1e-5), 9); - let updated_params = optimizer.step(&gradients); - - assert_eq!( - updated_params, - array![ - -0.004999994444450618, - 0.004999993750007813, - -0.004999992857153062, - 0.004999991666680556, - -0.004999990000020001, - 0.004999987500031251, - -0.004999983333388888, - 0.004999975000124999, - -0.0049999500004999945 - ] - ); - } - - #[test] - fn test_adam_step_empty_gradients_array() { - let gradients = Array1::::zeros(0); - - let mut optimizer = Adam::new(None, None, None, 0); - let updated_params = optimizer.step(&gradients); - - assert_eq!(updated_params, Array1::::zeros(0)); - } - - #[ignore] - #[test] - fn test_adam_step_iteratively_until_convergence_with_default_params() { - const CONVERGENCE_THRESHOLD: f64 = 1e-5; - let gradients = array![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - - let mut optimizer = Adam::new(None, None, None, 6); - - let mut model_params = Array1::::zeros(6); - let mut updated_params: Array1 = optimizer.step(&gradients); - - while (updated_params - .iter() - .zip(model_params.iter()) - .map(|(x, y)| x - y) - .collect::>()) - .iter() - .map(|&x| x.powi(2)) - .sum::() - .sqrt() - > CONVERGENCE_THRESHOLD - { - model_params = updated_params; - updated_params = optimizer.step(&gradients); - } - - // assert!(updated_params < Array1::::from_elem(6, CONVERGENCE_THRESHOLD)); - assert_ne!(updated_params, model_params); - assert_eq!( - updated_params, - array![ - -0.0009999999899999931, - -0.0009999999949999929, - -0.0009999999966666597, - -0.0009999999974999929, - -0.0009999999979999927, - -0.0009999999983333263 - ] - ); - } - - #[ignore] - #[test] - fn test_adam_step_iteratively_until_convergence_with_custom_params() { - const CONVERGENCE_THRESHOLD: f64 = 1e-7; - let gradients = array![7.0, -8.0, 9.0, -10.0, 11.0, -12.0, 13.0]; - - let mut optimizer = Adam::new(Some(0.005), Some((0.8, 0.899)), Some(1e-5), 7); - - let mut model_params = Array1::::zeros(7); - let mut updated_params = optimizer.step(&gradients); - - while (updated_params - .iter() - .zip(model_params.iter()) - .map(|(x, y)| x - y) - .collect::>()) - .iter() - .map(|&x| x.powi(2)) - .sum::() - .sqrt() - > CONVERGENCE_THRESHOLD - { - model_params = updated_params; - updated_params = optimizer.step(&gradients); - } - - // assert!(updated_params < vec![CONVERGENCE_THRESHOLD; 7]); - assert_ne!(updated_params, model_params); - assert_eq!( - updated_params, - array![ - -0.004999992857153061, - 0.004999993750007814, - -0.0049999944444506185, - 0.004999995000005001, - -0.004999995454549587, - 0.004999995833336807, - -0.004999996153849113 - ] - ); - } -} diff --git a/.artifacts/archive/optim/src/grad/descent.rs b/.artifacts/archive/optim/src/grad/descent.rs deleted file mode 100644 index fc4fbb65..00000000 --- a/.artifacts/archive/optim/src/grad/descent.rs +++ /dev/null @@ -1,139 +0,0 @@ -/* - Appellation: grad - Contrib: FL03 -*/ -use crate::neural::prelude::{Forward, Gradient, Layer, Sigmoid}; -use ndarray::prelude::{Array2, Ix2, NdFloat}; -use ndarray_stats::DeviationExt; -use num::{Float, Signed}; - -#[derive(Clone)] -pub struct GradientDescent -where - O: Gradient, - T: Float, -{ - gamma: T, - model: Layer, - objective: O, - store: Vec>, -} - -impl GradientDescent -where - O: Gradient, - T: Float, -{ - pub fn new(gamma: T, model: Layer, objective: O) -> Self { - Self { - gamma, - model, - objective, - store: Vec::new(), - } - } - - pub fn gamma(&self) -> T { - self.gamma - } - - pub fn gamma_mut(&mut self) -> &mut T { - &mut self.gamma - } - - pub fn model(&self) -> &Layer { - &self.model - } - - pub fn model_mut(&mut self) -> &mut Layer { - &mut self.model - } - - pub fn set_gamma(&mut self, gamma: T) { - self.gamma = gamma; - } - - pub fn set_model(&mut self, model: Layer) { - self.model = model; - } - - pub fn with_gamma(mut self, gamma: T) -> Self { - self.gamma = gamma; - self - } - - pub fn with_model(mut self, model: Layer) -> Self { - self.model = model; - self - } -} - -impl GradientDescent -where - O: Gradient, - T: NdFloat + Signed, -{ - pub fn gradient(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { - let lr = self.gamma(); - let ns = T::from(data.shape()[0]).unwrap(); - let pred = self.model.forward(data); - - let scale = T::from(2).unwrap() * ns; - - let errors = &pred - targets; - let dz = errors * self.objective.gradient(&pred); - let dw = data.t().dot(&dz) / scale; - - self.model_mut() - .update_with_gradient(lr, &dw.t().to_owned()); - - let loss = targets.mean_sq_err(&self.model().forward(data))?; - Ok(T::from(loss).unwrap()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::neural::prelude::{LayerShape, Sigmoid}; - use ndarray::prelude::{Array, Array2}; - - fn sample_data(inputs: usize, outputs: usize, samples: usize) -> (Array2, Array2) { - let m = samples * inputs; - let n = samples * outputs; - let x = Array::linspace(1., m as f64, m) - .into_shape((samples, inputs)) - .unwrap(); - let y = Array::linspace(1., n as f64, n) - .into_shape((samples, outputs)) - .unwrap(); - (x, y) - } - - #[test] - fn test_descent() { - let (samples, inputs, outputs) = (20, 5, 3); - - let (_epochs, gamma) = (1, 0.01); - // Generate some example data - let (x, y) = sample_data(inputs, outputs, samples); - let features = LayerShape::new(inputs, outputs); - let model = Layer::from(features).init(true); - - let mut grad = GradientDescent::new(gamma, model, Sigmoid); - - let l1 = { - let tmp = grad.gradient(&x, &y); - assert!(tmp.is_ok()); - tmp.unwrap() - }; - - let l2 = { - let tmp = grad.gradient(&x, &y); - assert!(tmp.is_ok()); - tmp.unwrap() - }; - - assert!(l1 > l2); - } -} diff --git a/.artifacts/archive/optim/src/grad/gradient.rs b/.artifacts/archive/optim/src/grad/gradient.rs deleted file mode 100644 index 1d895224..00000000 --- a/.artifacts/archive/optim/src/grad/gradient.rs +++ /dev/null @@ -1,153 +0,0 @@ -/* - Appellation: grad - Contrib: FL03 -*/ -use ndarray::prelude::{Array2, Axis, NdFloat}; -use ndarray_stats::DeviationExt; -use neural::models::ModelParams; -use neural::prelude::{Forward, Gradient, Sigmoid}; -use num::{Float, Signed}; - -pub struct Grad -where - O: Gradient, - T: Float, -{ - gamma: T, - params: ModelParams, - objective: O, -} - -impl Grad -where - O: Gradient, - T: Float, -{ - pub fn new(gamma: T, params: ModelParams, objective: O) -> Self { - Self { - gamma, - params, - objective, - } - } - - pub fn gamma(&self) -> T { - self.gamma - } - - pub fn gamma_mut(&mut self) -> &mut T { - &mut self.gamma - } - - pub fn objective(&self) -> &O { - &self.objective - } - - pub fn model(&self) -> &ModelParams { - &self.params - } - - pub fn model_mut(&mut self) -> &mut ModelParams { - &mut self.params - } -} - -impl Grad -where - O: Gradient, - T: NdFloat + Signed, -{ - pub fn gradient(&mut self, data: &Array2, targets: &Array2) -> anyhow::Result { - let lr = self.gamma(); - // the number of layers in the model - let depth = self.model().len(); - // the gradients for each layer - let mut grads = Vec::with_capacity(depth); - // a store for the predictions of each layer - let mut store = vec![data.clone()]; - // compute the predictions for each layer - for layer in self.model().clone().into_iter() { - let pred = layer.forward(&store.last().unwrap()); - store.push(pred); - } - // compute the error for the last layer - let error = store.last().unwrap() - targets; - // compute the error gradient for the last layer - let dz = &error * self.objective.gradient(&error); - // push the error gradient for the last layer - grads.push(dz.clone()); - - for i in (1..depth).rev() { - // get the weights for the current layer - let wt = self.params[i].weights().t(); - // compute the delta for the current layer w.r.t. the previous layer - let dw = grads.last().unwrap().dot(&wt); - // compute the gradient w.r.t. the current layer's predictions - let dp = self.objective.gradient(&store[i]); - // compute the gradient for the current layer - let gradient = dw * &dp; - grads.push(gradient); - } - // reverse the gradients so that they are in the correct order - grads.reverse(); - // update the parameters for each layer - for i in 0..depth { - let grad = &grads[i]; - let wg = &store[i].t().dot(grad); - let _bg = grad.sum_axis(Axis(0)); - self.params[i].weights_mut().scaled_add(-lr, &wg.t()); - } - let loss = self.model().forward(data).mean_sq_err(targets)?; - Ok(loss) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use concision::prelude::linarr; - use neural::models::ModelParams; - use neural::prelude::{Features, LayerShape, Sigmoid}; - - use ndarray::Ix2; - - pub fn assert_ok(result: Result) -> T - where - E: std::fmt::Debug, - T: std::fmt::Debug, - { - assert!(result.is_ok(), "{:?}", result); - result.unwrap() - } - - #[test] - fn test_gradient() { - let (samples, inputs) = (20, 5); - let outputs = 4; - - let _shape = (samples, inputs); - - let features = LayerShape::new(inputs, outputs); - - let x = linarr::((samples, features.inputs())).unwrap(); - let y = linarr::((samples, features.outputs())).unwrap(); - - let mut shapes = vec![features]; - shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - - let mut model = ModelParams::::from_iter(shapes).init(true); - - let mut grad = Grad::new(0.01, model.clone(), Sigmoid); - - let mut losses = Vec::new(); - - for _epoch in 0..3 { - let loss = assert_ok(grad.gradient(&x, &y)); - losses.push(loss); - } - - model = grad.model().clone(); - - assert!(losses.first().unwrap() > losses.last().unwrap()); - } -} diff --git a/.artifacts/archive/optim/src/grad/mod.rs b/.artifacts/archive/optim/src/grad/mod.rs deleted file mode 100644 index e5d26d0a..00000000 --- a/.artifacts/archive/optim/src/grad/mod.rs +++ /dev/null @@ -1,137 +0,0 @@ -/* - Appellation: grad - Contrib: FL03 -*/ -//! # Gradient Descent -pub use self::{descent::*, gradient::*, modes::*, utils::*}; - -pub(crate) mod descent; -pub(crate) mod gradient; -pub(crate) mod modes; - -pub mod adam; -pub mod sgd; - -pub struct BatchParams { - pub batch_size: usize, -} - -pub struct DescentParams { - pub batch_size: usize, - pub epochs: usize, - pub gamma: f64, // learning rate - - pub lambda: f64, // decay rate - pub mu: f64, // momentum rate - pub nesterov: bool, - pub tau: f64, // momentum damper -} - -pub(crate) mod utils { - use concision::prelude::BoxResult; - use core::ops::Sub; - use ndarray::linalg::Dot; - use ndarray::{Array, Array2, Dimension, NdFloat}; - use ndarray_stats::DeviationExt; - use neural::exp::models::Module; - use neural::prelude::Gradient; - use num::{FromPrimitive, Signed}; - - pub fn gradient<'a, T, D>( - gamma: T, - model: &mut Box>>, - data: &Array2, - targets: &Array, - grad: impl Gradient, - ) -> BoxResult - where - D: Dimension + 'a, - T: FromPrimitive + NdFloat + Signed, - Array2: Dot, Output = Array>, - &'a Array2: Sub<&'a Array, Output = Array>, - { - let (_samples, _inputs) = data.dim(); - let pred = model.predict(data)?; - - let ns = T::from(data.len()).unwrap(); - - let errors = &pred - targets; - // compute the gradient of the objective function w.r.t. the model's weights - let dz = &errors * grad.gradient(&pred); - // compute the gradient of the objective function w.r.t. the model's weights - let dw = data.t().to_owned().dot(&dz) / ns; - // let dw = - model.params().bias() * dz + data.t().to_owned().dot(&dz) / ns; - // compute the gradient of the objective function w.r.t. the model's bias - // let db = dz.sum_axis(Axis(0)) / ns; - // // Apply the gradients to the model's learnable parameters - // model.params_mut().bias_mut().scaled_add(-gamma, &db.t()); - for p in model.parameters_mut().values_mut() { - p.scaled_add(-gamma, &dw.t()); - } - - let loss = targets - .mean_sq_err(&model.predict(data)?) - .expect("Error when calculating the MSE of the model"); - Ok(loss) - } -} - -#[cfg(test)] -mod tests { - use concision::linarr; - use ndarray::{Array1, Ix2}; - use neural::models::ModelParams; - use neural::prelude::{Features, Forward, Layer, LayerShape, LinearActivation}; - - #[test] - fn test_gradient_descent() { - let (epochs, _gamma) = (10, 0.001); - let (samples, inputs) = (20, 5); - let outputs = 4; - - let _shape = (samples, inputs); - - let features = LayerShape::new(inputs, outputs); - - let _x = linarr::((samples, features.inputs())).unwrap(); - let _y = linarr::((samples, features.outputs())).unwrap(); - - let mut shapes = vec![features]; - shapes.extend((0..3).map(|_| LayerShape::new(features.outputs(), features.outputs()))); - - let mut _params = ModelParams::::from_iter(shapes).init(true); - - let mut _losses = Array1::::zeros(epochs); - // for e in 0..epochs { - // let cost = gradient_descent(gamma, &mut Box::new(model), Sigmoid, &x, &y) - // .expect("Gradient Descent Error"); - // losses[e] = cost; - // } - // assert_eq!(losses.len(), epochs); - } - - #[test] - fn test_gradient() { - let (samples, inputs, outputs) = (20, 5, 1); - - let (_epochs, _gamma) = (10, 0.001); - - let features = LayerShape::new(inputs, outputs); - - // Generate some example data - let x = linarr::((samples, features.inputs())).unwrap(); - let _y = linarr::((samples, features.outputs())).unwrap(); - - let model = Layer::::from(features).init(true); - - let _pred = model.forward(&x); - - // let mut losses = Array1::zeros(epochs); - // for e in 0..epochs { - // let cost = gradient(gamma, &mut model, &x, &y, Sigmoid).unwrap(); - // losses[e] = cost; - // } - // assert_eq!(losses.len(), epochs); - // assert!(losses.first() > losses.last()); - } -} diff --git a/.artifacts/archive/optim/src/grad/modes.rs b/.artifacts/archive/optim/src/grad/modes.rs deleted file mode 100644 index 8bfb1a60..00000000 --- a/.artifacts/archive/optim/src/grad/modes.rs +++ /dev/null @@ -1,34 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Mode { - Batch, - #[default] - Descent, - Stochastic, -} diff --git a/.artifacts/archive/optim/src/grad/sgd.rs b/.artifacts/archive/optim/src/grad/sgd.rs deleted file mode 100644 index 611b4e13..00000000 --- a/.artifacts/archive/optim/src/grad/sgd.rs +++ /dev/null @@ -1,251 +0,0 @@ -/* - Appellation: sgd - Contrib: FL03 -*/ -//! # Stochastic Gradient Descent (SGD) -//! -//! - -use neural::prelude::{Activate, Features, Forward, Layer}; -// use crate::prelude::ObjectiveFn; -use ndarray::{s, Array1, Array2, Axis, Ix2, NdFloat}; -use ndarray_stats::DeviationExt; -use num::{Float, FromPrimitive, Signed}; -use rand::seq::SliceRandom; - -pub fn sgd( - x: &Array2, - y: &Array2, - model: &mut Layer, - epochs: usize, - learning_rate: f64, - batch_size: usize, -) -> anyhow::Result> -where - A: Clone + Activate, -{ - let layer = model.clone(); - let features = layer.features(); - let (samples, _inputs) = x.dim(); - let mut indices: Vec = (0..samples).collect(); - let mut losses = Array1::::zeros(epochs); - - for epoch in 0..epochs { - indices.shuffle(&mut rand::thread_rng()); - let pos = &indices[..batch_size]; - - let xs = x.select(Axis(0), pos); - let ys = y.select(Axis(0), pos); - - let pred = model.forward(&xs); - let _error = &pred - &ys; - - for batch in (0..samples).step_by(batch_size) { - let mut gradient = Array2::zeros((features.outputs(), features.inputs())); - - for i in batch..(batch + batch_size).min(samples) { - let idx = indices[i]; - - let input = x - .slice(s![idx, ..]) - .to_shape((1, features.inputs()))? - .to_owned(); // (1, inputs) - let prediction = model.forward(&input); // (1, outputs) - - let inner = y - &prediction; - let partial_w = (-2.0 / batch_size as f64) * input.dot(&inner); - let _partial_b = (-2.0 / batch_size as f64) * inner; - gradient -= partial_w.sum(); - // let mut weights = model.weights_mut().slice_mut(s![]) - // model.set_weights(weights) - - let cost = y.mean_sq_err(&prediction)?; - losses[epoch] += cost; - // let error = &prediction - y[idx]; - println!("Cost:\t{:?}", &cost); - // gradient += &(input * cost); - } - gradient /= batch_size as f64; - model - .params_mut() - .weights_mut() - .scaled_add(-learning_rate, &gradient.t()); - - println!("Gradient:\n{:?}", &gradient); - } - losses /= batch_size as f64; - } - - Ok(losses) -} - -pub fn sgd_step( - x: &Array2, - y: &Array1, - model: &mut Layer, - _learning_rate: f64, - batch_size: usize, -) -> anyhow::Result -where - A: Clone + Activate, -{ - let layer = model.clone(); - let features = layer.features(); - let (_samples, _inputs) = x.dim(); - let mut indices: Vec = (0..features.outputs()).collect(); - let losses = 0.0; - - indices.shuffle(&mut rand::thread_rng()); - let pos = &indices[..batch_size]; - - let xs = x.select(Axis(0), pos); - let _ys = y.select(Axis(0), pos); - - let _pred = model.forward(&xs); - - Ok(losses) -} - -pub struct Sgd { - batch_size: usize, - gamma: f64, // learning rate - model: Layer, -} - -impl Sgd { - pub fn batch_size(&self) -> usize { - self.batch_size - } - - pub fn gamma(&self) -> f64 { - self.gamma - } - - pub fn model(&self) -> &Layer { - &self.model - } - - pub fn step(&mut self) -> f64 { - let loss = 0.0; - - loss - } -} - -impl Iterator for Sgd { - type Item = Array1; - - fn next(&mut self) -> Option { - None - } -} - -pub struct StochasticGradientDescent -where - T: Float, -{ - batch_size: usize, - epochs: usize, - gamma: T, // learning rate - model: Layer, -} - -impl StochasticGradientDescent -where - T: Float, -{ - pub fn new(batch_size: usize, epochs: usize, gamma: T, model: Layer) -> Self { - Self { - batch_size, - epochs, - gamma, - model, - } - } - - pub fn batch_size(&self) -> usize { - self.batch_size - } - - pub fn epochs(&self) -> usize { - self.epochs - } - - pub fn gamma(&self) -> T { - self.gamma - } - - pub fn model(&self) -> &Layer { - &self.model - } -} - -impl StochasticGradientDescent -where - T: Default + FromPrimitive + NdFloat + Signed, -{ - pub fn sgd(&mut self, x: &Array2, y: &Array2) -> Array1 { - let (samples, inputs) = x.dim(); - let mut indices: Vec = (0..samples).collect(); - let mut losses = Array1::::zeros(self.epochs); - - for epoch in 0..self.epochs { - indices.shuffle(&mut rand::thread_rng()); - - for batch_start in (0..samples).step_by(self.batch_size) { - let batch_end = (batch_start + self.batch_size).min(samples); - let mut gradient = Array2::zeros((inputs, self.model().features().outputs())); - - for i in batch_start..batch_end { - let idx = indices[i]; - let input = x - .slice(s![idx, ..]) - .to_shape((1, inputs)) - .expect("") - .to_owned(); // (1, inputs) - let prediction = self.model.forward(&input); // (1, outputs) - let error = &prediction - y; - gradient += &(&input * &error.t()).t(); - } - - gradient /= T::from(self.batch_size).unwrap(); - self.model.update_with_gradient(self.gamma, &gradient); - - println!("Gradient:\n{:?}", &gradient); - let loss = y.mean_sq_err(&self.model.forward(x)).unwrap(); - println!("Epoch: {:?}\nLoss:\n{:?}", &epoch, &loss); - losses[epoch] += gradient.mean().unwrap_or_default(); - } - losses[epoch] /= T::from(self.batch_size).unwrap(); - } - losses - } -} - -#[cfg(test)] -mod tests { - use super::*; - use concision::linarr; - use neural::prelude::{LayerShape, Sigmoid}; - - #[test] - fn test_sgd() { - let (samples, inputs) = (20, 5); - let outputs = 4; - - let features = LayerShape::new(inputs, outputs); - - let (_bs, _epochs, _gamma) = (10, 1, 0.01); - // Generate some example data - let x = linarr::((samples, inputs)).unwrap(); - let _y = linarr::((samples, outputs)).unwrap(); - - let model = Layer::::from(features).init(true); - - let _pred = model.forward(&x); - - // let mut sgd = StochasticGradientDescent::new(batch_size, epochs, gamma, model); - // sgd.sgd(&x, &y); - // let sgd = sgd(&x, &y, &mut model, epochs, gamma, batch_size).unwrap(); - } -} diff --git a/.artifacts/archive/optim/src/lib.rs b/.artifacts/archive/optim/src/lib.rs deleted file mode 100644 index dd4b11cd..00000000 --- a/.artifacts/archive/optim/src/lib.rs +++ /dev/null @@ -1,30 +0,0 @@ -/* - Appellation: optim - Contrib: FL03 -*/ -//! # Concision Optim -//! - -extern crate concision_core as concision; -extern crate concision_neural as neural; -pub use self::{optimizer::*, primitives::*, specs::*, utils::*}; - -pub(crate) mod optimizer; -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod grad; -pub mod norm; -pub mod params; - -pub mod prelude { - pub use crate::grad::*; - pub use crate::norm::*; - pub use crate::params::*; - - pub use crate::optimizer::*; - pub use crate::primitives::*; - pub use crate::specs::*; - pub use crate::utils::*; -} diff --git a/.artifacts/archive/optim/src/norm/kinds.rs b/.artifacts/archive/optim/src/norm/kinds.rs deleted file mode 100644 index 34b97825..00000000 --- a/.artifacts/archive/optim/src/norm/kinds.rs +++ /dev/null @@ -1,72 +0,0 @@ -/* - Appellation: kinds - Contrib: FL03 -*/ -use super::Norm; -use ndarray::prelude::{Array, NdFloat}; -use ndarray::Dimension; -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum Norms { - L0 = 0, - L1 = 1, - #[default] - L2 = 2, -} - -impl Norms { - pub fn l0() -> Self { - Self::L0 - } - - pub fn l1() -> Self { - Self::L1 - } - - pub fn l2() -> Self { - Self::L2 - } - - pub fn normalize(&self, args: &S) -> T - where - S: Norm, - { - use Norms::*; - - match *self { - L0 => args.l0(), - L1 => args.l1(), - L2 => args.l2(), - } - } - - pub fn norm_and_scale(&self, args: &Array) -> Array - where - D: Dimension, - T: NdFloat, - { - args / self.normalize(args) - } -} diff --git a/.artifacts/archive/optim/src/norm/mod.rs b/.artifacts/archive/optim/src/norm/mod.rs deleted file mode 100644 index 348696f7..00000000 --- a/.artifacts/archive/optim/src/norm/mod.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* - Appellation: norm - Contrib: FL03 -*/ -//! # norm -//! -pub use self::{kinds::*, normalizer::*, utils::*}; - -pub(crate) mod kinds; -pub(crate) mod normalizer; - -use ndarray::prelude::Array; -use ndarray::Dimension; -use ndarray_stats::QuantileExt; -use num::Float; - -pub trait Normalize { - type Output; - - fn norm(&self, args: &T) -> Self::Output; -} - -pub trait Norm { - fn l0(&self) -> T; - - fn l1(&self) -> T; - - fn l2(&self) -> T; -} - -impl Norm for Array -where - D: Dimension, - T: Float, -{ - fn l0(&self) -> T { - *self.max().expect("No max value") - } - - fn l1(&self) -> T { - self.mapv(|xs| xs.abs()).sum() - } - - fn l2(&self) -> T { - self.fold(T::zero(), |b, a| b + a.powi(2)).sqrt() - } -} - -pub(crate) mod utils { - use ndarray::prelude::Array; - use ndarray::Dimension; - use ndarray_stats::QuantileExt; - use num::Float; - - pub fn l0_norm(args: &Array) -> T - where - D: Dimension, - T: Float, - { - *args.max().expect("No max value") - } - - pub fn l1_norm(args: &Array) -> T - where - D: Dimension, - T: Float, - { - args.mapv(|xs| xs.abs()).sum() - } - - pub fn l2_norm(args: &Array) -> T - where - D: Dimension, - T: Float, - { - args.mapv(|xs| xs.powi(2)).sum().sqrt() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_l0_norm() { - let args = Array::linspace(1., 3., 3).into_shape(3).unwrap(); - - assert_eq!(l0_norm(&args), 3.); - } -} diff --git a/.artifacts/archive/optim/src/norm/normalizer.rs b/.artifacts/archive/optim/src/norm/normalizer.rs deleted file mode 100644 index 706fc5dc..00000000 --- a/.artifacts/archive/optim/src/norm/normalizer.rs +++ /dev/null @@ -1,40 +0,0 @@ -/* - Appellation: normalizer - Contrib: FL03 -*/ -use super::{Norm, Norms}; -use ndarray::prelude::{Array, NdFloat}; -use ndarray::Dimension; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct Normalizer { - pub mode: Norms, -} - -impl Normalizer { - pub fn new(mode: Norms) -> Self { - Self { mode } - } - - pub fn normalize(&self, args: &S) -> T - where - S: Norm, - { - match self.mode { - Norms::L0 => args.l0(), - Norms::L1 => args.l1(), - Norms::L2 => args.l2(), - } - } - - pub fn norm_and_scale(&self, args: &Array) -> Array - where - D: Dimension, - T: NdFloat, - { - args / self.normalize(args) - } -} diff --git a/.artifacts/archive/optim/src/optimizer.rs b/.artifacts/archive/optim/src/optimizer.rs deleted file mode 100644 index f89cc4ef..00000000 --- a/.artifacts/archive/optim/src/optimizer.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: optimizer - Contrib: FL03 -*/ - -pub trait Context { - type Config; - type Params; -} - -pub trait Optimizer { - type Config; - type Dataset; - type Params; - - fn config(&self) -> &Self::Config; - - fn name(&self) -> &str; - - fn load(&mut self, dataset: &Self::Dataset); - - fn step(&mut self, params: Self::Params) -> T; -} diff --git a/.artifacts/archive/optim/src/params/mod.rs b/.artifacts/archive/optim/src/params/mod.rs deleted file mode 100644 index 2bbe6ef2..00000000 --- a/.artifacts/archive/optim/src/params/mod.rs +++ /dev/null @@ -1,46 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -//! # Parameters -//! -//! ## Overview -//! -use num::Float; - -pub trait Minimize { - fn minimize(&self, scale: T) -> Self; -} - -pub trait Dampener -where - T: Float, -{ - fn tau(&self) -> T; // Momentum Damper -} - -pub trait Decay -where - T: Float, -{ - fn lambda(&self) -> T; // Decay Rate -} - -pub trait LearningRate -where - T: Float, -{ - fn gamma(&self) -> T; -} - -pub trait Momentum -where - T: Float, -{ - fn mu(&self) -> T; // Momentum Rate - - fn nestrov(&self) -> bool; -} - -#[cfg(test)] -mod tests {} diff --git a/.artifacts/archive/optim/src/primitives.rs b/.artifacts/archive/optim/src/primitives.rs deleted file mode 100644 index d80ec4b6..00000000 --- a/.artifacts/archive/optim/src/primitives.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, types::*}; - -mod constants { - - pub const FTOL: f64 = 2.220446049250313e-09; -} - -mod statics {} - -mod types { - use ndarray::prelude::{Array1, Array2}; - - pub type ObjectiveFn = fn(&Array2, &Array1) -> Array1; -} diff --git a/.artifacts/archive/optim/src/specs.rs b/.artifacts/archive/optim/src/specs.rs deleted file mode 100644 index b1e73252..00000000 --- a/.artifacts/archive/optim/src/specs.rs +++ /dev/null @@ -1,22 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ -use ndarray::prelude::{Array, Dimension, Ix2}; -use num::Float; - -pub trait ApplyGradient -where - D: Dimension, - T: Float, -{ - fn apply_gradient(&mut self, gamma: T, gradients: &Array); -} - -pub trait Autograd -where - D: Dimension, - T: Float, -{ - fn autograd(&mut self, loss: &Array) -> Array; -} diff --git a/.artifacts/archive/optim/src/utils.rs b/.artifacts/archive/optim/src/utils.rs deleted file mode 100644 index 0ba3ba3c..00000000 --- a/.artifacts/archive/optim/src/utils.rs +++ /dev/null @@ -1,72 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use crate::prelude::FTOL; -use ndarray::prelude::{Array, Array1, Dimension}; - -use num::traits::Float; - -pub fn minimize_inner( - w: &mut Array1, - fg: F, - epsilon: T, -) -> anyhow::Result<(&mut Array1, T, Array1)> -where - F: Fn(&Array1) -> (T, Array1), - T: Float + 'static, -{ - let (mut fp, mut gp) = fg(&w); // (cost, gradient) - - loop { - w.scaled_add(-epsilon, &gp); - let (f, g) = fg(&w); - - let exp = epsilon * norm_l2(&g); - let delta = fp - f; // actual descrease; last - current - if delta < exp * T::from(2).unwrap().recip() { - return Err(anyhow::anyhow!("Not enough decrease")); - } else if delta < T::from(FTOL).unwrap() { - return Ok((w, f, g)); - } - fp = f; - gp = g; - } -} - -pub fn minimize( - w: &mut Array1, - fg: F, - epsilon: T, - max_iter: usize, -) -> anyhow::Result<(&mut Array1, T, Array1)> -where - F: Fn(&Array1) -> (T, Array1), - T: Float + 'static, -{ - let (mut fp, mut gp) = fg(&w); // (cost, gradient) - - for _ in 0..max_iter { - w.scaled_add(-epsilon, &gp); - let (f, g) = fg(&w); - - let exp = epsilon * norm_l2(&g); - let delta = fp - f; // actual descrease; last - current - if delta < exp * T::from(2).unwrap().recip() { - return Err(anyhow::anyhow!("Not enough decrease")); - } else if delta < T::from(FTOL).unwrap() { - return Ok((w, f, g)); - } - fp = f; - gp = g; - } - Ok((w, fp, gp)) -} - -pub fn norm_l2(arr: &Array) -> T -where - D: Dimension, - T: Float, -{ - arr.fold(T::zero(), |b, a| b + a.powi(2)) -} diff --git a/.artifacts/archive/optim/tests/default.rs b/.artifacts/archive/optim/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/.artifacts/archive/optim/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/.artifacts/archive/transformers/Cargo.toml b/.artifacts/archive/transformers/Cargo.toml deleted file mode 100644 index 4df37bdd..00000000 --- a/.artifacts/archive/transformers/Cargo.toml +++ /dev/null @@ -1,54 +0,0 @@ -[package] -authors.workspace = true -categories.workspace = true -description.workspace = true -edition.workspace = true -homepage.workspace = true -keywords.workspace = true -license.workspace = true -name = "transformers" -readme.workspace = true -repository.workspace = true -version.workspace = true - -[features] -default = [] - -blas = [ - "ndarray/blas", - "concision-core/blas", - "concision-neural/blas", -] - -[lib] -bench = false -crate-type = ["rlib"] -doctest = false -test = true - -[build-dependencies] - -[dependencies] -concision-core = { features = ["full"], path = "../../core", version = "0.1.12" } -concision-neural = { path = "../neural", version = "0.1.12" } - -anyhow = "1" -lazy_static = "1" -ndarray = { features = ["serde-1"], version = "0.15" } -ndarray-rand = { features = [], version = "0.14" } -ndarray-stats = "0.5" -num = { features = ["rand", "serde"], version = "0.4" } -serde = { features = ["derive"], version = "1" } -serde_json = "1" -smart-default = "0.7" -strum = { features = ["derive"], version = "0.26" } - -[dev-dependencies] - -[package.metadata.docs.rs] -all-features = true -rustc-args = ["--cfg", "docsrs"] - -[target.wasm32-unknown-unknown] - -[target.wasm32-wasi] diff --git a/.artifacts/archive/transformers/src/attention/head.rs b/.artifacts/archive/transformers/src/attention/head.rs deleted file mode 100644 index 11d3c154..00000000 --- a/.artifacts/archive/transformers/src/attention/head.rs +++ /dev/null @@ -1,162 +0,0 @@ -/* - Appellation: head - Contrib: FL03 -*/ -use super::params::{HeadShape, QKV}; -use super::Weight; -use crate::neural::func::activate::{Activate, Softmax}; -use ndarray::prelude::{Array2, NdFloat}; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; -use serde::{Deserialize, Serialize}; -use std::ops; - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[serde(rename_all = "lowercase")] -pub struct AttentionHead { - dim: HeadShape, - mask: Array2, - weights: Weight, -} - -impl AttentionHead -where - T: Float, -{ - pub fn dim(&self) -> HeadShape { - self.dim - } - - pub fn mask_mut(&mut self) -> &mut Array2 { - &mut self.mask - } - - pub fn scale(&self) -> T { - T::one() / T::from(self.dim.query_size()).unwrap().sqrt() - } - - pub fn weights(&self) -> &Weight { - &self.weights - } - - pub fn set_mask(&mut self, mask: Array2) { - self.mask = mask; - } - - pub fn with_mask(mut self, mask: Array2) -> Self { - self.mask = mask; - self - } -} - -impl AttentionHead -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn new(dim: HeadShape) -> Self { - Self { - dim, - mask: Array2::zeros((dim.sequence(), dim.sequence())), - weights: Weight::uniform(dim), - } - } -} - -impl AttentionHead -where - T: NdFloat, -{ - pub fn attention(&mut self, data: &Array2) -> Array2 { - // multiply the data by the wieghted query, key, and value matrices, respectively - let weighted = data * self.weights(); - let (q, k, v) = weighted.qkv(); - - // compute the attention score - let inner = (q.dot(&k.t()) + self.mask.clone()) * self.scale(); - Softmax::default().activate(&inner).dot(&v) - } -} - -impl std::fmt::Display for AttentionHead -where - T: Float + Serialize, -{ - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) - } -} - -impl ops::Index for AttentionHead -where - T: Float, -{ - type Output = Array2; - - fn index(&self, index: QKV) -> &Self::Output { - &self.weights[index] - } -} - -impl ops::IndexMut for AttentionHead -where - T: Float, -{ - fn index_mut(&mut self, index: QKV) -> &mut Self::Output { - &mut self.weights[index] - } -} - -impl ops::Mul> for AttentionHead -where - T: NdFloat, -{ - type Output = AttentionHead; - - fn mul(self, rhs: Array2) -> Self::Output { - let mut head = self.clone(); - head.weights = self.weights * rhs; - head - } -} - -impl ops::Mul<&Array2> for AttentionHead -where - T: NdFloat, -{ - type Output = AttentionHead; - - fn mul(self, rhs: &Array2) -> Self::Output { - let mut head = self.clone(); - head.weights = self.weights * rhs; - head - } -} - -impl ops::MulAssign> for AttentionHead -where - T: NdFloat, -{ - fn mul_assign(&mut self, rhs: Array2) { - self.weights *= rhs; - } -} - -impl ops::MulAssign<&Array2> for AttentionHead -where - T: NdFloat, -{ - fn mul_assign(&mut self, rhs: &Array2) { - self.weights *= rhs; - } -} - -impl ops::MulAssign<&Array2> for &mut AttentionHead -where - T: NdFloat, -{ - fn mul_assign(&mut self, rhs: &Array2) { - self.weights *= rhs; - } -} diff --git a/.artifacts/archive/transformers/src/attention/mod.rs b/.artifacts/archive/transformers/src/attention/mod.rs deleted file mode 100644 index b5cb610c..00000000 --- a/.artifacts/archive/transformers/src/attention/mod.rs +++ /dev/null @@ -1,119 +0,0 @@ -/* - Appellation: attention - Contrib: FL03 -*/ -//! # Attention -//! -//! The attention mechanism is a key component of the transformer architecture. -//! -//! - -pub use self::{head::*, utils::*, weights::*}; - -pub(crate) mod head; -pub(crate) mod weights; - -pub mod multi; -pub mod params; - -use params::QKV; - -use crate::core::prelude::BoxResult; -use crate::prelude::BaseDim; - -use ndarray::prelude::{Array, Array2, Ix2, NdFloat}; -use num::Float; -use std::ops; - -/// (batch, sample, seq, model) -pub type InputArray = Array; - -pub type AttentionArray = Array; - -pub trait Attention { - fn attention(&self, data: &Array2) -> BoxResult> { - // let (seq, model) = data.dim(); - - let q = self.query().dot(data); - let k = self.key().dot(data); - let v = self.value().dot(data); - - let score = scaled_dot_product_attention(&q, &k, &v, Some(self.mask().clone())); - Ok(score) - } - - fn key(&self) -> &Array2; - - fn mask(&self) -> &Array2; - - fn query(&self) -> &Array2; - - fn value(&self) -> &Array2; -} - -pub trait Head -where - T: Float, -{ - fn key(&self) -> &Array2; - - fn query(&self) -> &Array2; - - fn value(&self) -> &Array2; - - fn query_size(&self) -> usize { - self.query().dim().1 - } - - fn qkv(&self) -> (&Array2, &Array2, &Array2) { - (self.query(), self.key(), self.value()) - } - - fn scale(&self) -> T { - T::one() / (T::from(self.key().dim().1).unwrap()).sqrt() - } -} - -impl Head for S -where - S: ops::Index>, - T: Float, -{ - fn key(&self) -> &Array2 { - &self[QKV::Key] - } - - fn query(&self) -> &Array2 { - &self[QKV::Query] - } - - fn value(&self) -> &Array2 { - &self[QKV::Value] - } -} - -pub(crate) mod utils { - use crate::neural::prelude::{Activate, Softmax}; - use ndarray::prelude::{Array2, NdFloat}; - - pub fn scaled_dot_product_attention( - query: &Array2, - key: &Array2, - value: &Array2, - mask: Option>, - ) -> Array2 { - let (seq, dk) = query.dim(); - let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); - let scale = T::one() / (T::from(dk).unwrap()).sqrt(); - let score = (query.dot(&key.t()) + mask) * scale; - Softmax::new(Some(1)).activate(&score).dot(value) - } -} - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_attention() {} -} diff --git a/.artifacts/archive/transformers/src/attention/multi/attention.rs b/.artifacts/archive/transformers/src/attention/multi/attention.rs deleted file mode 100644 index 830871b1..00000000 --- a/.artifacts/archive/transformers/src/attention/multi/attention.rs +++ /dev/null @@ -1,98 +0,0 @@ -/* - Appellation: attention - Contrib: FL03 -*/ -use super::{multihead, MultiHeadParams}; -use crate::attention::Weight; -use crate::ops::Split; -use crate::Mask; -use neural::prelude::{Forward, Layer}; - -use ndarray::prelude::{Array2, NdFloat}; -use ndarray::ShapeError; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct MultiHeadAttention { - features: MultiHeadParams, - linear: Layer, - weights: Weight, -} - -impl MultiHeadAttention -where - T: Float, -{ - pub fn linear(&self) -> &Layer { - &self.linear - } - - pub fn features(&self) -> MultiHeadParams { - self.features - } - - pub fn weights(&self) -> &Weight { - &self.weights - } -} - -impl MultiHeadAttention -where - T: Default + Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn new(heads: usize, model: usize) -> Self { - let features = MultiHeadParams::new(heads, model); - let weights = Weight::uniform((model, model)); - Self { - features, - linear: Layer::from_features(model, model), - weights, - } - } -} - -impl MultiHeadAttention -where - T: NdFloat, -{ - pub fn attention(&self, data: &Array2, mask: &Mask) -> Result, ShapeError> { - let weighted = data * self.weights(); - let (q, k, v) = weighted.split(self.features().heads())?; - let score = multihead(&q, &k, &v, mask)?; - let res = self.linear().forward(&score); - Ok(res) - } -} - -// impl Attention for MultiHeadAttention { -// fn key(&self) -> &Array2 { -// self.weights.key() -// } - -// fn mask(&self) -> &Array2 { -// &self.mask -// } - -// fn query(&self) -> &Array2 { -// &self.weights.query() -// } - -// fn value(&self) -> &Array2 { -// &self.weights.value() -// } -// } - -impl Forward> for MultiHeadAttention -where - T: NdFloat, -{ - type Output = Result, ShapeError>; - - fn forward(&self, data: &Array2) -> Self::Output { - self.attention(&data, &Mask::Unmasked) - } -} diff --git a/.artifacts/archive/transformers/src/attention/multi/mod.rs b/.artifacts/archive/transformers/src/attention/multi/mod.rs deleted file mode 100644 index 423baf8e..00000000 --- a/.artifacts/archive/transformers/src/attention/multi/mod.rs +++ /dev/null @@ -1,98 +0,0 @@ -/* - Appellation: multi - Contrib: FL03 -*/ -pub use self::{attention::*, params::*, utils::*}; - -pub(crate) mod attention; -pub(crate) mod params; - -use crate::attention::Weight; -use crate::ops::Split; -use crate::prelude::{BoxResult, Mask}; -use ndarray::prelude::{Array2, NdFloat}; - -pub trait MultiHead -where - T: NdFloat, -{ - fn attention(&mut self, data: &Array2, mask: &Mask) -> BoxResult> { - let weighted = data * self.weights(); - let (q, k, v) = weighted.split(self.params().heads())?; - let score = utils::multihead(&q, &k, &v, mask)?; - Ok(score) - } - - fn params(&self) -> MultiHeadParams; - - fn weights(&self) -> &Weight; -} - -pub(crate) mod utils { - use crate::attention::scaled_dot_product_attention; - use crate::ops::Merge; - use crate::Mask; - use ndarray::{s, Array2, Array3, Array4, NdFloat, ShapeError}; - - pub fn batched_multihead( - query: &Array4, - key: &Array4, - value: &Array4, - mask: Option>, - ) -> Result, ShapeError> { - let (batch, heads, seq, _) = query.dim(); - let mask = mask.unwrap_or_else(|| Array2::::zeros((seq, seq))); - let mut score = Array4::::zeros(query.dim()); - for i in 0..batch { - for h in 0..heads { - let q = query.slice(s![i, h, .., ..]).to_owned(); - let k = key.slice(s![i, h, .., ..]).to_owned(); - let v = value.slice(s![i, h, .., ..]).to_owned(); - let head = scaled_dot_product_attention(&q, &k, &v, Some(mask.clone())); - score.slice_mut(s![i, h, .., ..]).assign(&head); - } - } - score.merge() - } - - pub fn multihead( - query: &Array3, - key: &Array3, - value: &Array3, - mask: &Mask, - ) -> Result, ShapeError> - where - T: NdFloat, - { - let (heads, _, _) = query.dim(); - let mut score = Array3::::zeros(query.dim()); - for h in 0..heads { - let pos = s![h, .., ..]; - let q = query.slice(pos).to_owned(); - let k = key.slice(pos).to_owned(); - let v = value.slice(pos).to_owned(); - let head = scaled_dot_product_attention(&q, &k, &v, mask.clone().into()); - score.slice_mut(s![h, .., ..]).assign(&head); - } - score.merge() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Mask; - - #[test] - fn test_multihead_shape() { - let (heads, seq, model) = (8, 10, 512); - let data = Array2::::zeros((seq, model)); - - let mask = Mask::::uniform(seq).into(); - let attention = MultiHeadAttention::new(heads, model); - let score = attention - .attention(&data, &mask) - .expect("Failed to compute attention"); - assert_eq!(score.dim(), (seq, model)); - } -} diff --git a/.artifacts/archive/transformers/src/attention/multi/params.rs b/.artifacts/archive/transformers/src/attention/multi/params.rs deleted file mode 100644 index b6f955b3..00000000 --- a/.artifacts/archive/transformers/src/attention/multi/params.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use crate::{HEADS, MODEL}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct MultiHeadParams { - pub heads: usize, - pub model: usize, -} - -impl MultiHeadParams { - pub fn new(heads: usize, model: usize) -> Self { - Self { heads, model } - } - - pub fn heads(&self) -> usize { - self.heads - } - - pub fn model_size(&self) -> usize { - self.model - } - - pub fn query_size(&self) -> usize { - self.model / self.heads - } -} - -impl Default for MultiHeadParams { - fn default() -> Self { - Self::new(HEADS, MODEL) - } -} - -impl From for (usize, usize) { - fn from(params: MultiHeadParams) -> Self { - (params.heads, params.model) - } -} - -impl From for [usize; 2] { - fn from(params: MultiHeadParams) -> Self { - [params.heads, params.model] - } -} diff --git a/.artifacts/archive/transformers/src/attention/params/dim.rs b/.artifacts/archive/transformers/src/attention/params/dim.rs deleted file mode 100644 index 75f30de5..00000000 --- a/.artifacts/archive/transformers/src/attention/params/dim.rs +++ /dev/null @@ -1,222 +0,0 @@ -/* - Appellation: dim - Contrib: FL03 -*/ -//! # Dimensions -//! -//! The dimensionality of the attention mechanism is defined by the following: -//! -//! - `attention`: The dimension of the key, query, and value vectors -//! - `batch`: The batch size -//! - `heads`: The number of attention heads -//! - `model`: The dimension of the model (embedding size) -use crate::{HEADS, MODEL, QUERY_SIZE}; -use ndarray::prelude::Ix2; -use ndarray::IntoDimension; -use serde::{Deserialize, Serialize}; - -pub enum Shapes { - Batched((usize, Box)), - Data(BaseShape), - Head(HeadShape), - Mask { seq: usize }, - MultiHead(MultiShape), -} - -impl Shapes { - pub fn batched(batch_size: usize, shape: impl Into) -> Self { - Shapes::Batched((batch_size, Box::new(shape.into()))) - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct BaseShape { - pub batch: usize, - pub seq: usize, - pub model: usize, -} - -impl BaseShape { - pub fn new(batch: usize, seq: usize, model: usize) -> Self { - Self { batch, seq, model } - } - - pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, seq, MODEL) - } - - pub fn batch(&self) -> usize { - self.batch - } - - pub fn model_size(&self) -> usize { - self.model - } - - pub fn seq_len(&self) -> usize { - self.seq - } -} - -impl IntoDimension for BaseShape { - type Dim = ndarray::Ix3; - - fn into_dimension(self) -> Self::Dim { - ndarray::Ix3(self.batch, self.seq, self.model) - } -} - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct MultiShape { - pub heads: usize, - pub seq: usize, - pub query: usize, -} - -impl MultiShape { - pub fn new(heads: usize, seq: usize, query: usize) -> Self { - Self { heads, seq, query } - } - - pub fn std(seq: usize) -> Self { - Self::new(HEADS, seq, *QUERY_SIZE) - } - - pub fn heads(&self) -> usize { - self.heads - } - - pub fn model_size(&self) -> usize { - self.heads() * self.query_size() - } - - pub fn query_size(&self) -> usize { - self.query - } - - pub fn seq_len(&self) -> usize { - self.seq - } -} - -impl From for Shapes { - fn from(shape: MultiShape) -> Self { - Shapes::MultiHead(shape) - } -} - -impl From for [usize; 3] { - fn from(shape: MultiShape) -> Self { - [shape.heads, shape.seq, shape.query] - } -} - -impl From for (usize, usize, usize) { - fn from(shape: MultiShape) -> Self { - (shape.heads, shape.seq, shape.query) - } -} - -impl IntoDimension for MultiShape { - type Dim = ndarray::Ix3; - - fn into_dimension(self) -> Self::Dim { - ndarray::Ix3(self.heads, self.seq, self.query) - } -} - -pub trait HeadDimension { - fn query_size(&self) -> usize; - fn sequence(&self) -> usize; - - fn as_shape(&self) -> HeadShape { - HeadShape::new(self.query_size(), self.sequence()) - } -} - -pub trait AsShape { - fn as_shape(&self) -> Sh; -} - -pub trait ShapeExt -where - D: ndarray::Dimension, -{ - fn dim(&self) -> D; -} - -impl HeadDimension for T -where - T: Clone + IntoDimension, -{ - fn query_size(&self) -> usize { - self.clone().into_dimension()[1] - } - - fn sequence(&self) -> usize { - self.clone().into_dimension()[0] - } -} - -/// -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct HeadShape { - pub query: usize, // cols - pub seq: usize, // rows -} - -impl HeadShape { - pub fn new(query: usize, seq: usize) -> Self { - Self { query, seq } - } - - pub fn query_size(&self) -> usize { - self.query - } - - pub fn sequence(&self) -> usize { - self.seq - } - - pub fn scale(&self) -> f64 { - 1.0 / (self.query as f64).sqrt() - } -} - -impl From<(usize, usize)> for HeadShape { - fn from((seq, query): (usize, usize)) -> Self { - Self::new(query, seq) - } -} - -impl From for (usize, usize) { - fn from(shape: HeadShape) -> Self { - (shape.seq, shape.query) - } -} - -impl From<[usize; 2]> for HeadShape { - fn from(dim: [usize; 2]) -> Self { - Self::new(dim[1], dim[0]) - } -} - -impl From for [usize; 2] { - fn from(shape: HeadShape) -> Self { - [shape.seq, shape.query] - } -} - -impl IntoDimension for HeadShape { - type Dim = ndarray::Ix2; - - fn into_dimension(self) -> Self::Dim { - ndarray::Ix2(self.seq, self.query) - } -} diff --git a/.artifacts/archive/transformers/src/attention/params/hyperparams.rs b/.artifacts/archive/transformers/src/attention/params/hyperparams.rs deleted file mode 100644 index 4c636d93..00000000 --- a/.artifacts/archive/transformers/src/attention/params/hyperparams.rs +++ /dev/null @@ -1,114 +0,0 @@ -/* - Appellation: hyperparams - Contrib: FL03 -*/ -//! # Hyperparameters -//! -//! Hyperparameters are one which are set before training and are not updated. -//! -//! The hyperparameters for the attention mechanism are: -//! - batch: The number of samples in a batch. -//! - heads: The number of attention heads. -//! - model: The dimension of the model (embedding size). -//! - samples: The number of samples to draw from the attention distribution. -//! -//! - -use super::dim::{BaseShape, HeadShape, MultiShape}; -use crate::{HEADS, MODEL, SAMPLES}; -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] - -pub struct AttentionParameters { - pub batch: usize, - pub heads: usize, - pub model: usize, - pub samples: usize, - pub seq: usize, -} - -impl AttentionParameters { - pub fn new(batch: usize, heads: usize, model: usize, samples: usize, seq: usize) -> Self { - Self { - batch, - heads, - model, - samples, - seq, - } - } - - pub fn std(batch: usize, seq: usize) -> Self { - Self::new(batch, HEADS, MODEL, SAMPLES, seq) - } - - pub fn batch_size(&self) -> usize { - self.batch - } - - pub fn heads(&self) -> usize { - self.heads - } - - pub fn model_size(&self) -> usize { - self.model - } - - pub fn samples(&self) -> usize { - self.samples - } - - pub fn seq_len(&self) -> usize { - self.seq - } - - pub fn query_size(&self) -> usize { - self.model / self.heads - } - - pub fn with_batch(mut self, batch: usize) -> Self { - self.batch = batch; - self - } - - pub fn with_heads(mut self, heads: usize) -> Self { - self.heads = heads; - self - } - - pub fn with_model(mut self, model: usize) -> Self { - self.model = model; - self - } - - pub fn with_samples(mut self, samples: usize) -> Self { - self.samples = samples; - self - } - - pub fn with_seq(mut self, seq: usize) -> Self { - self.seq = seq; - self - } -} - -impl From for BaseShape { - fn from(params: AttentionParameters) -> Self { - Self::new(params.batch, params.seq, params.model) - } -} - -impl From for MultiShape { - fn from(params: AttentionParameters) -> Self { - Self::new(params.heads, params.seq, params.model) - } -} - -impl From for HeadShape { - fn from(params: AttentionParameters) -> Self { - Self::new(params.seq, params.query_size()) - } -} diff --git a/.artifacts/archive/transformers/src/attention/params/mod.rs b/.artifacts/archive/transformers/src/attention/params/mod.rs deleted file mode 100644 index a629ba23..00000000 --- a/.artifacts/archive/transformers/src/attention/params/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -//! # Attention Parameters -//! -//! ### Hyperparameters -//! -//! Hyperparameters are one which are set before training and are not updated. -//! -//! The hyperparameters for the attention mechanism are: -//! - batch: The number of samples in a batch. -//! - heads: The number of attention heads. -//! - model: The dimension of the model (embedding size). -//! - samples: The number of samples to draw from the attention distribution. -//! -//! -pub use self::{dim::*, hyperparams::*, qkv::*}; - -pub(crate) mod dim; -pub(crate) mod hyperparams; -pub(crate) mod qkv; - -pub(crate) mod utils {} diff --git a/.artifacts/archive/transformers/src/attention/params/qkv.rs b/.artifacts/archive/transformers/src/attention/params/qkv.rs deleted file mode 100644 index 1a538909..00000000 --- a/.artifacts/archive/transformers/src/attention/params/qkv.rs +++ /dev/null @@ -1,56 +0,0 @@ -/* - Appellation: qkv - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; -use strum::{Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - Clone, - Copy, - Debug, - Default, - Deserialize, - Display, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - Serialize, - VariantNames, -)] -#[repr(usize)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum QKV { - #[serde(alias = "k")] - #[strum(serialize = "k", serialize = "key")] - Key, - #[default] - #[serde(alias = "q")] - #[strum(serialize = "q", serialize = "query")] - Query, - #[serde(alias = "v")] - #[strum(serialize = "v", serialize = "val", serialize = "value")] - Value, -} - -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - - #[test] - fn test_qkv() { - use QKV::Key; - let w = Key; - assert_eq!(w.to_string(), "key"); - assert_eq!(Key, QKV::from_str("key").unwrap()); - assert_eq!(Key, QKV::from_str("k").unwrap()); - } -} diff --git a/.artifacts/archive/transformers/src/attention/weights.rs b/.artifacts/archive/transformers/src/attention/weights.rs deleted file mode 100644 index 09b23a7b..00000000 --- a/.artifacts/archive/transformers/src/attention/weights.rs +++ /dev/null @@ -1,288 +0,0 @@ -/* - Appellation: weights - Contrib: FL03 -*/ -//! # Weights -//! -//! ## Overview -//! -//! The `weights` module provides a `Weight` struct that is used to -//! group the `key`, `query`, and `value` matrices leveraged by the -//! attention mechanism. -//! -//! ## Dimensionality -//! -//! Each of the `key`, `query`, and `value` weight tensors are -//! initialized as square matrices (model, model) -//! -//! - W(model, model) -//! - Q/K/V(seq, model) * W(model, model) = (seq, model) -//! - Split(Q/K/V) = (heads, seq, model/heads) = (heads, seq, query) -//! - Q(seq, model) * Key(seq, model)^T = (seq, seq) -//! - (seq, seq) + Mask(seq, seq) = (seq, seq) -//! - (seq, seq) * V(seq, model) = (seq, model) -//! -//! -//! -use super::params::QKV; -use crate::core::GenerateRandom; -use crate::ops::Split; -use ndarray::linalg::Dot; -use ndarray::prelude::{Array, Array2, Array3, Ix2}; -use ndarray::IntoDimension; -use ndarray_rand::rand_distr::uniform::SampleUniform; -use ndarray_rand::rand_distr::{Distribution, StandardNormal}; -use num::Float; -use serde::{Deserialize, Serialize}; -use std::ops; -use strum::IntoEnumIterator; - -pub type WeightTensor = Array; // (seq, model) - -pub enum AttentionTensor { - Embedding(Array2), - Multihead(Array3), -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -pub struct Weight -where - T: Float, -{ - dim: Ix2, - pub key: Array2, - pub query: Array2, - pub value: Array2, -} - -impl Weight -where - T: Float, -{ - pub fn dim(&self) -> Ix2 { - self.dim - } - - pub fn key(&self) -> &Array2 { - &self.key - } - - pub fn query(&self) -> &Array2 { - &self.query - } - - pub fn value(&self) -> &Array2 { - &self.value - } - - pub fn qkv(&self) -> (Array2, Array2, Array2) { - self.clone().into() - } -} - -impl Weight -where - T: Default + Float, -{ - pub fn new(dim: impl IntoDimension) -> Self { - let dim = dim.into_dimension(); - let arr = Array2::default(dim); - Self { - dim, - key: arr.clone(), - query: arr.clone(), - value: arr, - } - } -} - -impl Weight -where - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn uniform(dim: impl IntoDimension) -> Self { - let dim = dim.into_dimension(); - Self { - dim: dim.clone(), - key: Array2::uniform(1, dim.clone()), - query: Array2::uniform(1, dim.clone()), - value: Array2::uniform(1, dim), - } - } - pub fn init_uniform(mut self) -> Self { - self.key = Array2::uniform(1, self.dim); - self.query = Array2::uniform(1, self.dim); - self.value = Array2::uniform(1, self.dim); - self - } -} - -impl std::fmt::Display for Weight { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", serde_json::to_string(self).unwrap()) - } -} - -impl Split<(Array3, Array3, Array3)> for Weight { - type Error = ndarray::ShapeError; - - fn split(&self, heads: usize) -> Result<(Array3, Array3, Array3), Self::Error> { - let (key, query, value) = self.qkv(); - Ok((key.split(heads)?, query.split(heads)?, value.split(heads)?)) - } -} -impl From for Weight -where - D: IntoDimension, - T: Float, -{ - fn from(dim: D) -> Self { - let dim = dim.into_dimension(); - let arr = Array2::ones(dim); - Self { - dim, - key: arr.clone(), - query: arr.clone(), - value: arr, - } - } -} - -impl From> for (Array2, Array2, Array2) { - fn from(context: Weight) -> Self { - (context.key, context.query, context.value) - } -} - -impl Dot> for Weight { - type Output = Self; - - fn dot(&self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - for qkv in QKV::iter() { - ctx[qkv] = ctx[qkv].dot(rhs); - } - ctx - } -} - -impl ops::Index for Weight { - type Output = Array2; - - fn index(&self, index: QKV) -> &Self::Output { - use QKV::*; - match index { - Key => &self.key, - Query => &self.query, - Value => &self.value, - } - } -} - -impl ops::IndexMut for Weight { - fn index_mut(&mut self, index: QKV) -> &mut Self::Output { - use QKV::*; - match index { - Key => &mut self.key, - Query => &mut self.query, - Value => &mut self.value, - } - } -} - -impl ops::Mul> for Array2 { - type Output = Weight; - - fn mul(self, rhs: Weight) -> Self::Output { - let mut ctx = rhs.clone(); - for qkv in QKV::iter() { - ctx[qkv] = self.dot(&ctx[qkv]); - } - ctx - } -} - -impl ops::Mul> for &Array2 { - type Output = Weight; - - fn mul(self, rhs: Weight) -> Self::Output { - let mut ctx = rhs.clone(); - for qkv in QKV::iter() { - ctx[qkv] = self.dot(&ctx[qkv]); - } - ctx - } -} - -impl ops::Mul<&Weight> for &Array2 { - type Output = Weight; - - fn mul(self, rhs: &Weight) -> Self::Output { - let mut ctx = rhs.clone(); - for qkv in QKV::iter() { - ctx[qkv] = self.dot(&ctx[qkv]); - } - ctx - } -} - -impl ops::Mul> for Weight { - type Output = Self; - - fn mul(self, rhs: Array2) -> Self::Output { - let mut ctx = self.clone(); - for qkv in QKV::iter() { - ctx[qkv] = ctx[qkv].dot(&rhs); - } - ctx - } -} - -impl ops::Mul<&Array2> for Weight { - type Output = Self; - - fn mul(self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - for qkv in QKV::iter() { - ctx[qkv] = ctx[qkv].dot(rhs); - } - ctx - } -} - -impl ops::Mul<&Array2> for &Weight { - type Output = Weight; - - fn mul(self, rhs: &Array2) -> Self::Output { - let mut ctx = self.clone(); - for qkv in QKV::iter() { - ctx[qkv] = ctx[qkv].dot(rhs); - } - ctx - } -} - -impl ops::MulAssign> for Weight { - fn mul_assign(&mut self, rhs: Array2) { - for qkv in QKV::iter() { - self[qkv] = self[qkv].dot(&rhs); - } - } -} - -impl ops::MulAssign<&Array2> for Weight { - fn mul_assign(&mut self, rhs: &Array2) { - for qkv in QKV::iter() { - self[qkv] = self[qkv].dot(rhs); - } - } -} - -impl ops::MulAssign<&Array2> for &mut Weight { - fn mul_assign(&mut self, rhs: &Array2) { - for qkv in QKV::iter() { - self[qkv] = self[qkv].dot(rhs); - } - } -} diff --git a/.artifacts/archive/transformers/src/codec/decode/decoder.rs b/.artifacts/archive/transformers/src/codec/decode/decoder.rs deleted file mode 100644 index 1c157163..00000000 --- a/.artifacts/archive/transformers/src/codec/decode/decoder.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: decoder - Contrib: FL03 -*/ - -pub struct Decoder {} diff --git a/.artifacts/archive/transformers/src/codec/decode/mod.rs b/.artifacts/archive/transformers/src/codec/decode/mod.rs deleted file mode 100644 index 716e4454..00000000 --- a/.artifacts/archive/transformers/src/codec/decode/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -/* - Appellation: decode - Contrib: FL03 -*/ -//! # Decode -pub use self::decoder::*; - -pub(crate) mod decoder; -pub(crate) mod params; - -pub trait Decode {} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_decoder() {} -} diff --git a/.artifacts/archive/transformers/src/codec/decode/params.rs b/.artifacts/archive/transformers/src/codec/decode/params.rs deleted file mode 100644 index e1913f35..00000000 --- a/.artifacts/archive/transformers/src/codec/decode/params.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ diff --git a/.artifacts/archive/transformers/src/codec/encode/encoder.rs b/.artifacts/archive/transformers/src/codec/encode/encoder.rs deleted file mode 100644 index b57d6900..00000000 --- a/.artifacts/archive/transformers/src/codec/encode/encoder.rs +++ /dev/null @@ -1,52 +0,0 @@ -/* - Appellation: encoder - Contrib: FL03 -*/ -use super::EncoderParams; -use crate::attention::multi::MultiHeadAttention; -use crate::ffn::FFN; -use crate::Mask; -use ndarray::Array2; -use neural::prelude::{Forward, LayerNorm}; - -pub struct Encoder { - attention: MultiHeadAttention, - network: FFN, - norm_attention: LayerNorm, - norm_network: LayerNorm, - params: EncoderParams, -} - -impl Encoder { - pub fn new(params: EncoderParams) -> Self { - let attention = MultiHeadAttention::new(params.heads, params.model); - let network = FFN::new(params.model, crate::NETWORK); - Self { - attention, - network, - norm_attention: LayerNorm::new((params.model, params.model)), - norm_network: LayerNorm::new((params.model, params.model)), - params, - } - } - - fn _forward(&self, data: &Array2, mask: &Mask) -> anyhow::Result> { - let attention = data + self.attention.attention(data, mask)?; - let norm = self.norm_attention.forward(&attention); - let network = data + self.network.forward(&norm); - let norm = self.norm_network.forward(&network); - Ok(norm) - } - - pub fn forward(&mut self, data: &Array2, mask: &Mask) -> anyhow::Result> { - let norm = self.norm_attention.forward(data); - let attention = data + self.attention.attention(&norm, mask)?; - let norm = self.norm_network.forward(&attention); - let network = data + self.network.forward(&norm); - Ok(network) - } - - pub fn params(&self) -> EncoderParams { - self.params - } -} diff --git a/.artifacts/archive/transformers/src/codec/encode/mod.rs b/.artifacts/archive/transformers/src/codec/encode/mod.rs deleted file mode 100644 index f6d0bae9..00000000 --- a/.artifacts/archive/transformers/src/codec/encode/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -/* - Appellation: encode - Contrib: FL03 -*/ -//! # Encode -pub use self::{encoder::*, params::*, stack::*}; - -pub(crate) mod encoder; -pub(crate) mod params; -pub(crate) mod stack; - -pub trait Encode {} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Mask; - - use ndarray::prelude::*; - - #[test] - fn test_encoder() { - let (heads, seq, model) = (8, 10, 512); - let _data = Array2::::zeros((seq, model)); - let _mask = Mask::::uniform(seq); - let params = EncoderParams::new(heads, model); - let encoder = Encoder::new(params); - - assert_eq!(encoder.params().heads(), heads); - } -} diff --git a/.artifacts/archive/transformers/src/codec/encode/params.rs b/.artifacts/archive/transformers/src/codec/encode/params.rs deleted file mode 100644 index d7d84f35..00000000 --- a/.artifacts/archive/transformers/src/codec/encode/params.rs +++ /dev/null @@ -1,36 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use crate::{HEADS, MODEL}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct EncoderParams { - pub heads: usize, - pub model: usize, -} - -impl EncoderParams { - pub fn new(heads: usize, model: usize) -> Self { - Self { heads, model } - } - - pub fn heads(&self) -> usize { - self.heads - } - - pub fn model_size(&self) -> usize { - self.model - } - - pub fn query_size(&self) -> usize { - self.model / self.heads - } -} - -impl Default for EncoderParams { - fn default() -> Self { - Self::new(HEADS, MODEL) - } -} diff --git a/.artifacts/archive/transformers/src/codec/encode/stack.rs b/.artifacts/archive/transformers/src/codec/encode/stack.rs deleted file mode 100644 index 5b4c321e..00000000 --- a/.artifacts/archive/transformers/src/codec/encode/stack.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - Appellation: stack - Contrib: FL03 -*/ -use super::{Encoder, EncoderParams}; - -pub struct EncoderStack { - layers: usize, - params: EncoderParams, - stack: Vec, -} - -impl EncoderStack { - pub fn new(layers: usize, params: EncoderParams) -> Self { - let stack = Vec::with_capacity(layers); - - Self { - layers, - params, - stack, - } - } - - pub fn setup(&mut self) { - for _ in 0..self.layers { - self.stack.push(Encoder::new(self.params)); - } - } -} diff --git a/.artifacts/archive/transformers/src/codec/mod.rs b/.artifacts/archive/transformers/src/codec/mod.rs deleted file mode 100644 index 5a21f740..00000000 --- a/.artifacts/archive/transformers/src/codec/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - Appellation: codec - Contrib: FL03 -*/ -//! # Codec - -pub mod decode; -pub mod encode; - -pub trait Codec {} - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_codec() {} -} diff --git a/.artifacts/archive/transformers/src/ffn/mod.rs b/.artifacts/archive/transformers/src/ffn/mod.rs deleted file mode 100644 index 09fb9ba4..00000000 --- a/.artifacts/archive/transformers/src/ffn/mod.rs +++ /dev/null @@ -1,35 +0,0 @@ -/* - Appellation: decode - Contrib: FL03 -*/ -//! # Decode -pub use self::{network::*, params::*}; - -pub(crate) mod network; -pub(crate) mod params; - -pub(crate) mod utils {} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::prelude::linarr; - use crate::neural::prelude::Forward; - use ndarray::prelude::Ix2; - - #[test] - fn test_ffn() { - let samples = 20; - let (model, network) = (5, 15); - - // sample data - let x = linarr::((samples, model)).unwrap(); - let _y = linarr::((samples, model)).unwrap(); - - let ffn = FFN::new(model, network); - // assert!(network.validate_dims()); - - let pred = ffn.forward(&x); - assert_eq!(&pred.dim(), &(samples, model)); - } -} diff --git a/.artifacts/archive/transformers/src/ffn/network.rs b/.artifacts/archive/transformers/src/ffn/network.rs deleted file mode 100644 index a3cbf66d..00000000 --- a/.artifacts/archive/transformers/src/ffn/network.rs +++ /dev/null @@ -1,63 +0,0 @@ -/* - Appellation: network - Contrib: FL03 -*/ -use super::FFNParams; -use crate::neural::prelude::{Forward, Layer, ReLU}; -use crate::prelude::{MODEL, NETWORK}; -use ndarray::prelude::{Array2, NdFloat}; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -pub struct FFN -where - T: Clone, -{ - input: Layer, - output: Layer, - pub params: FFNParams, -} - -impl FFN -where - T: Clone, -{ - pub fn new(model: usize, network: usize) -> Self - where - T: Default, - { - Self { - input: Layer::from_features(model, network), - output: Layer::from_features(network, model), - params: FFNParams::new(model, network), - } - } - - pub fn input(&self) -> &Layer { - &self.input - } - - pub fn output(&self) -> &Layer { - &self.output - } -} - -impl Default for FFN -where - T: Clone + Default, -{ - fn default() -> Self { - Self::new(MODEL, NETWORK) - } -} - -impl Forward> for FFN -where - T: NdFloat, -{ - type Output = Array2; - - fn forward(&self, data: &Array2) -> Self::Output { - self.output.forward(&ReLU(&self.input.forward(data))) - } -} diff --git a/.artifacts/archive/transformers/src/ffn/params.rs b/.artifacts/archive/transformers/src/ffn/params.rs deleted file mode 100644 index da4c10a0..00000000 --- a/.artifacts/archive/transformers/src/ffn/params.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use crate::neural::prelude::Features; -use crate::prelude::{MODEL, NETWORK}; -use ndarray::prelude::Ix2; -use ndarray::IntoDimension; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct FFNParams { - pub model: usize, - pub network: usize, -} - -impl FFNParams { - pub fn new(model: usize, network: usize) -> Self { - Self { model, network } - } - - pub fn model(&self) -> usize { - self.model - } - - pub fn network(&self) -> usize { - self.network - } - - pub fn features(&self) -> usize { - self.network / self.model - } -} - -impl Default for FFNParams { - fn default() -> Self { - Self::new(MODEL, NETWORK) - } -} - -impl Features for FFNParams { - fn inputs(&self) -> usize { - self.model - } - - fn outputs(&self) -> usize { - self.network - } -} - -impl IntoDimension for FFNParams { - type Dim = Ix2; - - fn into_dimension(self) -> Ix2 { - (self.model(), self.network()).into_dimension() - } -} diff --git a/.artifacts/archive/transformers/src/lib.rs b/.artifacts/archive/transformers/src/lib.rs deleted file mode 100644 index 1587464a..00000000 --- a/.artifacts/archive/transformers/src/lib.rs +++ /dev/null @@ -1,46 +0,0 @@ -/* - Appellation: transformers - Contrib: FL03 -*/ -//! # Concision Transformers -//! -extern crate concision_core as concision; -extern crate concision_neural as neural; - -pub use self::primitives::*; - -#[allow(unused_imports)] -pub(crate) use self::base::*; - -pub(crate) mod primitives; -pub(crate) mod specs; -pub(crate) mod utils; - -pub mod attention; -pub mod codec; -pub mod ffn; -pub mod ops; -pub mod transform; - -pub(crate) use concision as core; - -#[allow(unused_imports)] -pub(crate) mod base { - - pub(crate) use neural::params::masks::Mask; - pub(crate) use neural::params::{Biased, Weighted}; - - pub(crate) type BoxResult = core::result::Result>; -} - -pub mod prelude { - pub use crate::attention::params::*; - pub use crate::codec::*; - pub use crate::ffn::*; - pub use crate::ops::*; - pub use crate::transform::*; - - pub use crate::primitives::*; - - pub(crate) use crate::base::*; -} diff --git a/.artifacts/archive/transformers/src/ops/merge.rs b/.artifacts/archive/transformers/src/ops/merge.rs deleted file mode 100644 index f7279298..00000000 --- a/.artifacts/archive/transformers/src/ops/merge.rs +++ /dev/null @@ -1,43 +0,0 @@ -/* - Appellation: merge - Contrib: FL03 -*/ -use ndarray::prelude::{Array2, Array3, Array4}; -use ndarray::{Order, ShapeError}; - -pub trait Merge { - type Error; - - fn merge(&self) -> Result; -} - -impl Merge> for Array3 -where - T: Clone, -{ - type Error = ShapeError; - - fn merge(&self) -> Result, Self::Error> { - let (heads, seq, query) = self.dim(); - let mut tmp = self.clone(); - // swap the head and sequence axes - tmp.swap_axes(0, 1); - // reshape the qkv matrix into a 2d array - let res = tmp.to_shape(((seq, heads * query), Order::ColumnMajor))?; - Ok(res.to_owned()) - } -} - -impl Merge> for Array4 { - type Error = ShapeError; - - fn merge(&self) -> Result, Self::Error> { - let (batch, heads, seq, query) = self.dim(); - let mut tmp = self.clone(); - // swap the head and sequence axes - tmp.swap_axes(1, 2); - // reshape the qkv matrix into a 2d array - let res = tmp.to_shape(((batch, seq, heads * query), Order::ColumnMajor))?; - Ok(res.to_owned()) - } -} diff --git a/.artifacts/archive/transformers/src/ops/mod.rs b/.artifacts/archive/transformers/src/ops/mod.rs deleted file mode 100644 index 65867743..00000000 --- a/.artifacts/archive/transformers/src/ops/mod.rs +++ /dev/null @@ -1,87 +0,0 @@ -/* - Appellation: ops - Contrib: FL03 -*/ -pub use self::{merge::*, split::*, utils::*}; - -pub(crate) mod merge; -pub(crate) mod split; - -pub(crate) mod utils { - use ndarray::prelude::{Array2, Array3, Array4}; - use ndarray::ShapeError; - // use num::Float; - - pub fn merge_heads(heads: &Array3) -> Result, ShapeError> - where - T: Clone, - { - let (n, seq, query) = heads.dim(); - let mut tmp = heads.clone(); - // swap the head and sequence axes - tmp.swap_axes(0, 1); - // reshape the qkv matrix into a 2d array - tmp.into_shape((seq, n * query)) - } - - pub fn split_heads(param: &Array2, num_heads: usize) -> Result, ShapeError> - where - T: Clone, - { - let dim = param.shape().last().unwrap() / num_heads; - // reshape the qkv matrix into a 3d array - let mut res = param - .clone() - .into_shape((param.shape()[0], num_heads, dim))?; - // swap the sequence and head axes - res.swap_axes(0, 1); - Ok(res) - } - - pub fn merge_batch(heads: &Array4) -> Result, ShapeError> - where - T: Clone, - { - let (batch, n, seq, query) = heads.dim(); - let mut tmp = heads.clone(); - // swap the head and sequence axes - tmp.swap_axes(1, 2); - // reshape the qkv matrix into a 2d array - tmp.into_shape((batch, seq, n * query)) - } - - pub fn split_batch(param: &Array3, num_heads: usize) -> Result, ShapeError> - where - T: Clone, - { - let dim = param.shape().last().unwrap() / num_heads; - // reshape the qkv matrix into a 3d array - let mut res = - param - .clone() - .into_shape((param.shape()[0], param.shape()[1], num_heads, dim))?; - // swap the sequence and head axes - res.swap_axes(1, 2); - Ok(res) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ndarray::Array; - - #[test] - fn reshape_ops() { - let dim_input: [usize; 3] = [2, 4, 6]; // (batch, seq, model) - let dim_split = [2, 2, 4, 3]; // (batch, heads, seq, model) - let data = Array::linspace(1., 48., 48).into_shape(dim_input).unwrap(); - - let a = split_batch(&data, 2).unwrap(); - assert_eq!(a.shape(), &dim_split); - assert_eq!(&a, &data.split(2).unwrap()); - let b = merge_batch(&a).unwrap(); - assert_eq!(b.shape(), &dim_input); - assert_eq!(&b, &data); - } -} diff --git a/.artifacts/archive/transformers/src/ops/split.rs b/.artifacts/archive/transformers/src/ops/split.rs deleted file mode 100644 index 13c367e4..00000000 --- a/.artifacts/archive/transformers/src/ops/split.rs +++ /dev/null @@ -1,50 +0,0 @@ -/* - Appellation: split - Contrib: FL03 -*/ -use ndarray::prelude::{Array2, Array3, Array4}; -use ndarray::ShapeError; - -// pub fn split(param: &Array, heads: usize) -> Result, ShapeError> { -// let mut dim = param.dim() -// let query = param.shape().last().unwrap() / heads; -// // reshape the qkv matrix into a 3d array -// let mut res = param.clone().into_shape((param.shape()[0], heads, query))?; -// // swap the sequence and head axes -// res.swap_axes(0, 1); -// Ok(res) -// } - -pub trait Split { - type Error; - - fn split(&self, heads: usize) -> Result; -} - -impl Split> for Array2 { - type Error = ShapeError; - - fn split(&self, heads: usize) -> Result, Self::Error> { - let (seq, model) = self.dim(); - let query = model / heads; - // reshape the qkv matrix into a 3d array - let mut res = self.clone().into_shape((seq, heads, query))?; - // swap the sequence and head axes - res.swap_axes(0, 1); - Ok(res) - } -} - -impl Split> for Array3 { - type Error = ShapeError; - - fn split(&self, heads: usize) -> Result, Self::Error> { - let (batch, seq, model) = self.dim(); - let query = model / heads; - // reshape the qkv matrix into a 3d array - let mut res = self.clone().into_shape((batch, seq, heads, query))?; - // swap the sequence and head axes - res.swap_axes(1, 2); - Ok(res) - } -} diff --git a/.artifacts/archive/transformers/src/primitives.rs b/.artifacts/archive/transformers/src/primitives.rs deleted file mode 100644 index d004ca57..00000000 --- a/.artifacts/archive/transformers/src/primitives.rs +++ /dev/null @@ -1,33 +0,0 @@ -/* - Appellation: primitives - Contrib: FL03 -*/ -pub use self::{constants::*, statics::*, types::*}; - -pub(crate) mod constants { - /// The default dropout rate - pub const DROPOUT: f64 = 0.1; - /// The default number of heads in the multi-head attention layer - pub const HEADS: usize = 8; - /// The default dimension of the model (embedding size) - pub const MODEL: usize = 512; - /// The default number of parameters in the feed-forward network - pub const NETWORK: usize = 2048; - /// The default number of samples to draw from the attention distribution - pub const SAMPLES: usize = 10000; -} - -pub(crate) mod statics { - use super::constants::*; - use lazy_static::lazy_static; - - lazy_static! { - /// The default dimensions of the query, key, and value tensors w/r/2 a single head - pub static ref QUERY_SIZE: usize = MODEL / HEADS; - } -} - -pub(crate) mod types { - /// The dimension of all inputs and outputs for each layer of the model (batch, seq, model) - pub type BaseDim = ndarray::Ix3; -} diff --git a/.artifacts/archive/transformers/src/specs.rs b/.artifacts/archive/transformers/src/specs.rs deleted file mode 100644 index 1d8faa71..00000000 --- a/.artifacts/archive/transformers/src/specs.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: specs - Contrib: FL03 -*/ diff --git a/.artifacts/archive/transformers/src/transform/config.rs b/.artifacts/archive/transformers/src/transform/config.rs deleted file mode 100644 index 1b66b482..00000000 --- a/.artifacts/archive/transformers/src/transform/config.rs +++ /dev/null @@ -1,16 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use serde::{Deserialize, Serialize}; - -#[derive( - Clone, Copy, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, -)] -pub struct TransformerParams { - pub batch: usize, - pub heads: usize, - pub layers: usize, - pub model: usize, - pub samples: usize, -} diff --git a/.artifacts/archive/transformers/src/transform/mod.rs b/.artifacts/archive/transformers/src/transform/mod.rs deleted file mode 100644 index c408f6eb..00000000 --- a/.artifacts/archive/transformers/src/transform/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -/* - Appellation: transform - Contrib: FL03 -*/ -//! # Transform -pub use self::{config::*, transformer::*}; - -pub(crate) mod config; -pub(crate) mod transformer; - -#[cfg(test)] -mod tests { - // use super::*; - - #[test] - fn test_transformer() {} -} diff --git a/.artifacts/archive/transformers/src/transform/transformer.rs b/.artifacts/archive/transformers/src/transform/transformer.rs deleted file mode 100644 index 4123d4fb..00000000 --- a/.artifacts/archive/transformers/src/transform/transformer.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - Appellation: transformer - Contrib: FL03 -*/ -use concision::Transform; -use ndarray::prelude::{Array2, NdFloat}; - -#[derive(Clone, Debug, Default)] -pub struct Transformer; - -impl Transform> for Transformer -where - T: NdFloat, -{ - type Output = Array2; - - fn transform(&self, args: &Array2) -> Self::Output { - args.clone() - } -} diff --git a/.artifacts/archive/transformers/src/utils.rs b/.artifacts/archive/transformers/src/utils.rs deleted file mode 100644 index 752dabaf..00000000 --- a/.artifacts/archive/transformers/src/utils.rs +++ /dev/null @@ -1,4 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ diff --git a/.artifacts/archive/transformers/tests/default.rs b/.artifacts/archive/transformers/tests/default.rs deleted file mode 100644 index 0cac1eb5..00000000 --- a/.artifacts/archive/transformers/tests/default.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[cfg(test)] -#[test] -fn compiles() { - let f = |x: usize, y: usize| x + y; - - assert_eq!(f(10, 10), 20); - assert_ne!(f(1, 1), 3); -} diff --git a/core/src/nn/model/config.rs b/core/src/nn/model/config.rs index 7596c36b..047cce43 100644 --- a/core/src/nn/model/config.rs +++ b/core/src/nn/model/config.rs @@ -4,15 +4,11 @@ */ use crate::traits::Config; -pub struct ModelConfig { +pub struct ConfigBase { + pub id: usize, pub name: String, + pub description: String, + _children: Vec>, } -impl Config for ModelConfig {} - -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] -pub struct ConfigBase { - pub id: usize, - pub name: &'static str, -} diff --git a/core/src/traits/setup.rs b/core/src/traits/setup.rs index d6f2f611..e95fbb3e 100644 --- a/core/src/traits/setup.rs +++ b/core/src/traits/setup.rs @@ -7,6 +7,14 @@ use core::borrow::{Borrow, BorrowMut}; /// A trait used to denote objects that may be used for configuring various items pub trait Config {} +pub trait CompositConfig: Config { + type Ctx: Config; + + fn children(&self) -> Vec>; + + fn context(&self) -> Self::Ctx; +} + /// [Configuration] describes composite configuration objects; /// A configuration object is allowed to inherit from another configuration object pub trait Configuration diff --git a/data/Cargo.toml b/data/Cargo.toml index 460d512a..978e9c0f 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -80,11 +80,10 @@ crate-type = ["lib"] doctest = true test = true -[[test]] -name = "default" - [build-dependencies] +[dev-dependencies] + [dependencies] itertools.workspace = true ndarray.workspace = true @@ -111,8 +110,6 @@ version = "1" optional = true version = "0.1" -[dev-dependencies] - [package.metadata.docs.rs] all-features = true rustc-args = ["--cfg", "docsrs"] diff --git a/models/kan/src/actor.rs b/models/kan/src/actor.rs new file mode 100644 index 00000000..5495d28b --- /dev/null +++ b/models/kan/src/actor.rs @@ -0,0 +1,9 @@ +/* + Appellation: actor + Contrib: FL03 +*/ +pub use self::config::ActorConfig; + +pub(crate) mod config; + +pub struct Actor; \ No newline at end of file diff --git a/models/kan/src/actor/config.rs b/models/kan/src/actor/config.rs new file mode 100644 index 00000000..c96167e3 --- /dev/null +++ b/models/kan/src/actor/config.rs @@ -0,0 +1,17 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use concision::Config; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct ActorConfig; + +impl Config for ActorConfig {} + +impl Default for ActorConfig { + fn default() -> Self { + Self + } +} \ No newline at end of file diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs index 0f618f66..bfcc81df 100644 --- a/models/kan/src/lib.rs +++ b/models/kan/src/lib.rs @@ -22,6 +22,10 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; +#[doc(inline)] +pub use self::actor::Actor; + +pub mod actor; pub mod model; pub mod prelude {} diff --git a/models/kan/src/model/mod.rs b/models/kan/src/model/mod.rs index 8b137891..d4acaf46 100644 --- a/models/kan/src/model/mod.rs +++ b/models/kan/src/model/mod.rs @@ -1 +1,5 @@ +/* + Appellation: model + Contrib: FL03 +*/ From ac749a5f6f255cabf89f47a3d1d4813d9fc95523 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 15:14:11 -0500 Subject: [PATCH 05/12] update Signed-off-by: Joe McCain III --- .github/dependabot.yml | 8 +++ .github/workflows/crates.yml | 2 +- Cargo.toml | 1 + concision/Cargo.toml | 18 +++++ concision/src/lib.rs | 21 +++--- core/src/lib.rs | 1 + core/src/nn/model/config.rs | 3 +- core/src/traits/params.rs | 9 ++- core/src/utils/tensor.rs | 27 +------- core/tests/utils.rs | 27 ++++---- models/gnn/Cargo.toml | 34 ++++------ models/kan/src/actor.rs | 56 ++++++++++++++- models/kan/src/actor/config.rs | 2 +- models/kan/src/model/mod.rs | 1 - models/s4/Cargo.toml | 120 +++++++++++++++++++++++++++++++++ models/s4/build.rs | 8 +++ models/s4/src/lib.rs | 19 ++++++ models/s4/tests/default.rs | 17 +++++ models/s4/tests/model.rs | 25 +++++++ 19 files changed, 320 insertions(+), 79 deletions(-) create mode 100644 models/s4/Cargo.toml create mode 100644 models/s4/build.rs create mode 100644 models/s4/src/lib.rs create mode 100644 models/s4/tests/default.rs create mode 100644 models/s4/tests/model.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f284eb73..d8296020 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -41,3 +41,11 @@ updates: package-ecosystem: cargo schedule: interval: weekly + - directory: /models/s4 + package-ecosystem: cargo + schedule: + interval: weekly + - directory: /models/transformer + package-ecosystem: cargo + schedule: + interval: weekly diff --git a/.github/workflows/crates.yml b/.github/workflows/crates.yml index fdef5c1a..6811b66e 100644 --- a/.github/workflows/crates.yml +++ b/.github/workflows/crates.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - features: [ kan, gnn, linear, transformer ] + features: [ kan, gnn, linear, s4, transformer ] steps: - uses: actions/checkout@v4 - name: Publish (${{ github.event.repository.name }}-${{ matrix.features }}) diff --git a/Cargo.toml b/Cargo.toml index 686ae01f..6a540ea4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ members = [ "models/linear", "models/gnn", "models/kan", + "models/s4", "models/transformers", ] diff --git a/concision/Cargo.toml b/concision/Cargo.toml index dfed699b..bcb6d11e 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -45,6 +45,7 @@ models = [ "gnn", "kan", "linear", + "s4", "transformer", ] @@ -60,6 +61,10 @@ linear = [ "dep:concision-linear", ] +s4 = [ + "dep:concision-s4", +] + transformer = [ "dep:concision-transformer", ] @@ -72,6 +77,7 @@ alloc = [ "concision-gnn?/alloc", "concision-kan?/alloc", "concision-linear?/alloc", + "concision-s4?/alloc", "concision-transformer?/alloc", ] @@ -81,6 +87,7 @@ approx = [ "concision-gnn?/approx", "concision-kan?/approx", "concision-linear?/approx", + "concision-s4?/approx", "concision-transformer?/approx", ] @@ -90,6 +97,7 @@ rand = [ "concision-gnn?/rand", "concision-kan?/rand", "concision-linear?/rand", + "concision-s4?/rand", "concision-transformer?/rand", ] @@ -99,6 +107,7 @@ serde = [ "concision-gnn?/serde", "concision-kan?/serde", "concision-linear?/serde", + "concision-s4?/serde", "concision-transformer?/serde", ] @@ -108,6 +117,7 @@ tracing = [ "concision-gnn?/tracing", "concision-kan?/tracing", "concision-linear?/tracing", + "concision-s4?/tracing", "concision-transformer?/tracing", ] @@ -119,6 +129,7 @@ std = [ "concision-gnn?/std", "concision-kan?/std", "concision-linear?/std", + "concision-s4?/std", "concision-transformer?/std", ] @@ -137,6 +148,7 @@ wasi = [ "concision-gnn?/wasi", "concision-kan?/wasi", "concision-linear?/wasi", + "concision-s4?/wasi", "concision-transformer?/wasi", ] @@ -147,6 +159,7 @@ blas = [ "concision-gnn?/blas", "concision-kan?/blas", "concision-linear?/blas", + "concision-s4?/blas", "concision-transformer?/blas", ] @@ -226,6 +239,11 @@ optional = true path = "../models/linear" version = "0.1.15" +[dependencies.concision-s4] +optional = true +path = "../models/s4" +version = "0.1.15" + [dependencies.concision-transformer] optional = true path = "../models/transformers" diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 14be53a7..2537ec36 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -27,24 +27,29 @@ pub use concision_kan as kan; pub use concision_linear as linear; #[cfg(feature = "macros")] pub use concision_macros::*; +#[cfg(feature = "s4")] +#[doc(inline)] +pub use concision_s4 as s4; #[cfg(feature = "transformer")] #[doc(inline)] pub use concision_transformer as transformer; pub mod prelude { + #[cfg(feature = "gnn")] + pub use crate::gnn::prelude::*; + #[cfg(feature = "kan")] + pub use crate::kan::prelude::*; + #[cfg(feature = "linear")] + pub use crate::linear::prelude::*; + #[cfg(feature = "s4")] + pub use crate::s4::prelude::*; + #[cfg(feature = "transformer")] + pub use crate::transformer::prelude::*; pub use concision_core::prelude::*; #[cfg(feature = "data")] pub use concision_data::prelude::*; #[cfg(feature = "derive")] pub use concision_derive::*; - #[cfg(feature = "gnn")] - pub use concision_gnn::prelude::*; - #[cfg(feature = "kan")] - pub use concision_kan::prelude::*; - #[cfg(feature = "linear")] - pub use concision_linear::prelude::*; #[cfg(feature = "macros")] pub use concision_macros::*; - #[cfg(feature = "transformer")] - pub use concision_transformer::prelude::*; } diff --git a/core/src/lib.rs b/core/src/lib.rs index a09d48e1..00284c4f 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -12,6 +12,7 @@ extern crate ndarray as nd; extern crate ndarray_rand as ndrand; pub use self::error::{Error, ErrorKind, PredictError}; +pub use self::func::Activate; pub use self::nn::Module; pub use self::{primitives::*, traits::prelude::*, types::prelude::*, utils::prelude::*}; diff --git a/core/src/nn/model/config.rs b/core/src/nn/model/config.rs index 047cce43..32734d18 100644 --- a/core/src/nn/model/config.rs +++ b/core/src/nn/model/config.rs @@ -8,7 +8,6 @@ pub struct ConfigBase { pub id: usize, pub name: String, pub description: String, - + _children: Vec>, } - diff --git a/core/src/traits/params.rs b/core/src/traits/params.rs index 1e3834b0..aa3b379c 100644 --- a/core/src/traits/params.rs +++ b/core/src/traits/params.rs @@ -4,20 +4,19 @@ */ /// A `Params` object is used to store the various learnable parameters of a model. -/// +/// /// ### Specifications -/// +/// /// - `Elem`: The type of the elements being stored -/// +/// pub trait Params { type Elem; } pub trait ParamFeatures { type Dim: nd::Dimension; - } pub trait Parameter { type Kind: 'static; -} \ No newline at end of file +} diff --git a/core/src/utils/tensor.rs b/core/src/utils/tensor.rs index 3576aec1..6956ac31 100644 --- a/core/src/utils/tensor.rs +++ b/core/src/utils/tensor.rs @@ -93,7 +93,7 @@ where pub(crate) mod gen { use nd::{Array, Array1, Dimension, IntoDimension, ShapeError}; - use num::traits::{Float, FromPrimitive, Num, NumCast}; + use num::traits::{Float, NumCast}; pub fn genspace(features: usize) -> Array1 { Array1::from_iter((0..features).map(|x| T::from(x).unwrap())) @@ -108,31 +108,6 @@ pub(crate) mod gen { let n = dim.size(); Array::linspace(A::zero(), A::from(n - 1).unwrap(), n).into_shape(dim) } - - pub fn linspace(start: T, end: T, n: usize) -> Vec - where - T: Copy + FromPrimitive + Num, - { - if n <= 1 { - panic!("linspace requires at least two points"); - } - - let step = (end - start) / T::from_usize(n - 1).unwrap(); - - (0..n) - .map(|i| start + step * T::from_usize(i).unwrap()) - .collect() - } - /// creates a matrix from the given shape filled with numerical elements [0, n) spaced evenly by 1 - pub fn rangespace(dim: impl IntoDimension) -> Array - where - A: FromPrimitive, - D: Dimension, - { - let dim = dim.into_dimension(); - let iter = (0..dim.size()).map(|i| A::from_usize(i).unwrap()).collect(); - Array::from_shape_vec(dim, iter).unwrap() - } } pub(crate) mod stack { diff --git a/core/tests/utils.rs b/core/tests/utils.rs index 698d04a1..3a426b22 100644 --- a/core/tests/utils.rs +++ b/core/tests/utils.rs @@ -4,12 +4,13 @@ */ extern crate concision_core as cnc; -use cnc::{linarr, linspace, tril, Conjugate, Inverse}; -use ndarray::*; -use num::Complex; +use cnc::linarr; +use ndarray::prelude::*; #[test] fn test_conj() { + use cnc::Conjugate; + use num::complex::Complex; let data = (1..5).map(|x| x as f64).collect::>(); let a = Array2::from_shape_vec((2, 2), data).unwrap(); let exp = array![[1.0, 2.0], [3.0, 4.0]]; @@ -30,6 +31,7 @@ fn test_conj() { #[test] fn test_inverse() { + use cnc::Inverse; let a = array![[1.0, 2.0], [3.0, 4.0]]; let b = array![[1.0, 2.0, 3.0,], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]; let exp = array![[-2.0, 1.0], [1.5, -0.5]]; @@ -39,21 +41,18 @@ fn test_inverse() { #[test] fn test_linarr() { - let args: Array2 = linarr((2, 3)).unwrap(); - assert_eq!(&args, &array![[0.0, 1.0, 2.0,], [3.0, 4.0, 5.0]]); -} - -#[test] -fn test_linspace() { - let arr = Array::linspace(1f64, 9f64, 9); - let vec = linspace(1f64, 9f64, 9); - let arr2 = Array::from_vec(vec); - assert_eq!(arr, arr2); + let shape = (2, 3); + let n = shape.0 * shape.1; + let args = linarr::(shape.clone()).unwrap(); + let exp = Array::linspace(0f64, (n - 1) as f64, n) + .into_shape(shape) + .unwrap(); + assert_eq!(args, exp); } #[test] fn test_tril() { let a = linarr::((3, 3)).unwrap(); let exp = array![[0.0, 0.0, 0.0], [3.0, 4.0, 0.0], [6.0, 7.0, 8.0,]]; - assert_eq!(exp, tril(&a)); + assert_eq!(exp, cnc::tril(&a)); } diff --git a/models/gnn/Cargo.toml b/models/gnn/Cargo.toml index fca9e052..6c16e62e 100644 --- a/models/gnn/Cargo.toml +++ b/models/gnn/Cargo.toml @@ -25,45 +25,39 @@ full = [ ] # ********* [FF] Dependencies ********* - -alloc = [ - "concision-core/alloc", -] - approx = [ "dep:approx", "concision-core/approx", "ndarray/approx-0_5", ] -blas = [ - "concision-core/blas", - "ndarray/blas", -] - rand = [ "concision-core/rand", - "num/rand" + "num/rand", ] serde = [ "dep:serde", - "serde-ext", -] - -serde-ext = [ "concision-core/serde", "ndarray/serde-1", "num/serde" ] - - tracing = [ "dep:tracing", ] # ********* [FF] Environments ********* +alloc = [ + "concision-core/alloc", + "num/alloc", + "serde?/alloc", +] + +blas = [ + "concision-core/blas", + "ndarray/blas", +] std = [ "concision-core/std", @@ -89,6 +83,9 @@ test = true [build-dependencies] +[dev-dependencies] +lazy_static.workspace = true + [dependencies] ndarray.workspace = true num.workspace = true @@ -114,9 +111,6 @@ version = "1" optional = true version = "0.1" -[dev-dependencies] -lazy_static.workspace = true - [package.metadata.docs.rs] all-features = true rustc-args = ["--cfg", "docsrs"] diff --git a/models/kan/src/actor.rs b/models/kan/src/actor.rs index 5495d28b..7fa1cb4d 100644 --- a/models/kan/src/actor.rs +++ b/models/kan/src/actor.rs @@ -6,4 +6,58 @@ pub use self::config::ActorConfig; pub(crate) mod config; -pub struct Actor; \ No newline at end of file +use concision::prelude::{Activate, Predict, PredictError}; +use core::ops::Mul; +use nd::prelude::*; +use splines::interpolate::{Interpolate, Interpolator}; +use splines::Spline; + +/// An `actor` describe the learnable activation functions +/// employed throughout KAN networks. +/// +/// The learned functions are one-dimensional, continuous +pub struct Actor { + pub(crate) bias: B, + pub(crate) omega: Array1, + pub(crate) spline: Spline, +} + +impl Actor +where + B: Activate, +{ + pub fn new(bias: B, omega: Array1, spline: Spline) -> Self { + Self { + bias, + omega, + spline, + } + } + + pub fn bias(&self) -> &B { + &self.bias + } + + pub fn omega(&self) -> &Array1 { + &self.omega + } + + pub fn spline(&self) -> &Spline { + &self.spline + } +} + +impl Predict> for Actor +where + B: Activate>, + T: Interpolator, + V: Interpolate, + Array1: Clone + Mul, +{ + type Output = as Mul>::Output; + + fn predict(&self, x: &Array1) -> Result { + let y = x.mapv(|xi| self.spline.sample(xi).unwrap()); + Ok(self.omega.clone() * self.bias.activate(y)) + } +} diff --git a/models/kan/src/actor/config.rs b/models/kan/src/actor/config.rs index c96167e3..6fbde74a 100644 --- a/models/kan/src/actor/config.rs +++ b/models/kan/src/actor/config.rs @@ -14,4 +14,4 @@ impl Default for ActorConfig { fn default() -> Self { Self } -} \ No newline at end of file +} diff --git a/models/kan/src/model/mod.rs b/models/kan/src/model/mod.rs index d4acaf46..fb253b77 100644 --- a/models/kan/src/model/mod.rs +++ b/models/kan/src/model/mod.rs @@ -2,4 +2,3 @@ Appellation: model Contrib: FL03 */ - diff --git a/models/s4/Cargo.toml b/models/s4/Cargo.toml new file mode 100644 index 00000000..ce75e910 --- /dev/null +++ b/models/s4/Cargo.toml @@ -0,0 +1,120 @@ +[package] +authors.workspace = true +build = "build.rs" +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-s4" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [ + "std", +] + +full = [ + "default", + "approx", + "rand", + "serde", +] + +# ********* [FF] Dependencies ********* +approx = [ + "dep:approx", + "concision-core/approx", + "ndarray/approx-0_5", +] + +rand = [ + "concision-core/rand", + "num/rand", +] + +serde = [ + "dep:serde", + "concision-core/serde", + "ndarray/serde-1", + "num/serde" +] + +tracing = [ + "dep:tracing", +] + +# ********* [FF] Environments ********* +alloc = [ + "concision-core/alloc", + "num/alloc", + "serde?/alloc", +] + +blas = [ + "concision-core/blas", + "ndarray/blas", +] + +std = [ + "concision-core/std", + "ndarray/std", + "num/std", + "serde?/std", + "strum/std", +] + +wasm = [ + "concision-core/wasm", +] + +wasi = [ + "concision-core/wasi", +] + +[lib] +bench = false +crate-type = ["lib"] +doctest = true +test = true + +[build-dependencies] + +[dev-dependencies] +lazy_static.workspace = true + +[dependencies] +ndarray.workspace = true +num.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dependencies.approx] +optional = true +version = "0.5" + +[dependencies.concision-core] +default-features = false +path = "../../core" +version = "0.1.15" + +[dependencies.serde] +default-features = false +features = ["derive"] +optional = true +version = "1" + +[dependencies.tracing] +optional = true +version = "0.1" + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/models/s4/build.rs b/models/s4/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/models/s4/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/models/s4/src/lib.rs b/models/s4/src/lib.rs new file mode 100644 index 00000000..67977d5f --- /dev/null +++ b/models/s4/src/lib.rs @@ -0,0 +1,19 @@ +/* + Appellation: concision-gnn + Contrib: FL03 +*/ +//! # Structured State Space Sequential Models (S4) +//! +//! +//! ## Resources +//! +#![cfg_attr(not(feature = "std"), no_std)] +#![crate_name = "concision_s4"] + +#[cfg(feature = "alloc")] +extern crate alloc; + +extern crate concision_core as concision; +extern crate ndarray as nd; + +pub mod prelude {} diff --git a/models/s4/tests/default.rs b/models/s4/tests/default.rs new file mode 100644 index 00000000..233a07af --- /dev/null +++ b/models/s4/tests/default.rs @@ -0,0 +1,17 @@ +/* + Appellation: default + Contrib: FL03 +*/ + +fn add(a: A, b: B) -> C +where + A: core::ops::Add, +{ + a + b +} + +#[test] +fn compiles() { + assert_eq!(add(10, 10), 20); + assert_ne!(add(1, 1), 3); +} diff --git a/models/s4/tests/model.rs b/models/s4/tests/model.rs new file mode 100644 index 00000000..3f6fb75a --- /dev/null +++ b/models/s4/tests/model.rs @@ -0,0 +1,25 @@ +/* + Appellation: model + Contrib: FL03 +*/ +#![allow(unused)] + +extern crate concision_core as concision; +extern crate concision_s4 as s4; + +use concision::prelude::{linarr, Forward}; + +use lazy_static::lazy_static; + +const SAMPLES: usize = 20; +const INPUTS: usize = 5; +const OUTPUT: usize = 3; + +lazy_static! { + static ref CONFIG: (usize, usize, usize) = (SAMPLES, INPUTS, OUTPUT); +} + +#[test] +fn test_gnn() { + let (samples, input, output) = CONFIG.clone(); +} From d24a116348831c2356e589c0f1932528efd67148 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 26 May 2024 15:24:13 -0500 Subject: [PATCH 06/12] update Signed-off-by: Joe McCain III --- core/src/error/err.rs | 16 ++++++++-------- core/src/error/kinds.rs | 10 +++++----- core/src/error/kinds/external.rs | 28 +++++++++++++++------------- core/src/error/mod.rs | 7 +++---- core/src/lib.rs | 2 +- core/src/types/mod.rs | 2 +- 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/core/src/error/err.rs b/core/src/error/err.rs index 9012ea7e..d475b173 100644 --- a/core/src/error/err.rs +++ b/core/src/error/err.rs @@ -2,19 +2,19 @@ Appellation: err Contrib: FL03 */ -use super::ErrorKind; +use super::Errors; use crate::uuid; #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] pub struct Error { id: String, - kind: ErrorKind, + kind: Errors, message: String, } impl Error { - pub fn new(kind: ErrorKind, message: impl ToString) -> Self { + pub fn new(kind: Errors, message: impl ToString) -> Self { Self { id: uuid().to_string(), kind, @@ -24,7 +24,7 @@ impl Error { pub fn from_kind(kind: K) -> Self where - K: Into, + K: Into, { Self::new(kind.into(), "") } @@ -33,7 +33,7 @@ impl Error { &self.id } - pub fn kind(&self) -> &ErrorKind { + pub fn kind(&self) -> &Errors { &self.kind } @@ -55,15 +55,15 @@ impl core::fmt::Display for Error { #[cfg(feature = "std")] impl std::error::Error for Error {} -impl From for Error { - fn from(kind: ErrorKind) -> Self { +impl From for Error { + fn from(kind: Errors) -> Self { Self::from_kind(kind) } } impl<'a, K> From<&'a K> for Error where - K: Clone + Into, + K: Clone + Into, { fn from(kind: &'a K) -> Self { Self::from_kind(kind.clone()) diff --git a/core/src/error/kinds.rs b/core/src/error/kinds.rs index 7764426b..7033f755 100644 --- a/core/src/error/kinds.rs +++ b/core/src/error/kinds.rs @@ -30,7 +30,7 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, VariantNames}; serde(rename_all = "lowercase", tag = "kind") )] #[strum(serialize_all = "lowercase")] -pub enum ErrorKind { +pub enum Errors { IO, External(ExternalError), Predict(PredictError), @@ -53,8 +53,8 @@ macro_rules! from_err { }; } -from_err!(ErrorKind: - ErrorKind::External(ExternalError), - ErrorKind::Model(ModelError), - ErrorKind::Predict(PredictError), +from_err!(Errors: + Errors::External(ExternalError), + Errors::Model(ModelError), + Errors::Predict(PredictError), ); diff --git a/core/src/error/kinds/external.rs b/core/src/error/kinds/external.rs index eb5d289f..41a0c2ba 100644 --- a/core/src/error/kinds/external.rs +++ b/core/src/error/kinds/external.rs @@ -2,8 +2,9 @@ Appellation: error Contrib: FL03 */ +use crate::error::ErrorKind; use smart_default::SmartDefault; -use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, VariantNames}; +use strum::{AsRefStr, EnumCount, EnumIs, VariantNames}; #[derive( AsRefStr, @@ -11,7 +12,6 @@ use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, VariantNames}; Debug, EnumCount, EnumIs, - EnumIter, Eq, Hash, Ord, @@ -26,23 +26,23 @@ use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, VariantNames}; serde(rename_all = "lowercase", untagged) )] #[strum(serialize_all = "lowercase")] -pub enum ExternalError { - Error(String), +pub enum ExternalError { + Error(E), #[default] Unknown, } -impl ExternalError { - pub fn new(err: impl ToString) -> Self { - let err = err.to_string(); - if err.is_empty() { - return Self::unknown(); +impl ExternalError { + pub fn new(err: Option) -> Self { + if let Some(err) = err { + Self::error(err) + } else { + Self::unknown() } - Self::error(err) } - pub fn error(err: impl ToString) -> Self { - ExternalError::Error(err.to_string()) + pub fn error(err: impl Into) -> Self { + ExternalError::Error(err.into()) } pub fn unknown() -> Self { @@ -50,7 +50,9 @@ impl ExternalError { } } -impl core::fmt::Display for ExternalError { +impl ErrorKind for ExternalError where E: Clone + ToString {} + +impl core::fmt::Display for ExternalError where E: ToString { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { let msg = match self { ExternalError::Error(ref err) => err.to_string(), diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index 72302066..9a00fb03 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -8,14 +8,14 @@ mod err; pub mod kinds; -pub trait ErrKind {} +pub trait ErrorKind: Clone + ToString {} macro_rules! impl_error_type { ($($ty:ty),* $(,)*) => { $(impl_error_type!(@impl $ty);)* }; (@impl $ty:ty) => { - impl ErrKind for $ty {} + impl ErrorKind for $ty {} impl_error_type!(@std $ty); }; @@ -27,8 +27,7 @@ macro_rules! impl_error_type { } impl_error_type!( - kinds::ErrorKind, - kinds::ExternalError, + kinds::Errors, kinds::PredictError, crate::nn::ModelError ); diff --git a/core/src/lib.rs b/core/src/lib.rs index 00284c4f..d3949fbf 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -11,7 +11,7 @@ extern crate ndarray as nd; #[cfg(feature = "rand")] extern crate ndarray_rand as ndrand; -pub use self::error::{Error, ErrorKind, PredictError}; +pub use self::error::{Error, Errors, PredictError}; pub use self::func::Activate; pub use self::nn::Module; pub use self::{primitives::*, traits::prelude::*, types::prelude::*, utils::prelude::*}; diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs index d6347e49..1dd40697 100644 --- a/core/src/types/mod.rs +++ b/core/src/types/mod.rs @@ -12,7 +12,7 @@ pub mod shape; pub type NdResult = core::result::Result; /// A type alias for a [Result](core::result::Result) with the crate's [Error](crate::error::Error) type. /// Defaults to `Result<(), Error>` -pub type Result = core::result::Result; +pub type Result = core::result::Result; #[cfg(feature = "std")] mod std_types { From ca662c4c3cbe7d5237f952ee622e3e042eae5ff9 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Mon, 27 May 2024 08:50:39 -0500 Subject: [PATCH 07/12] update Signed-off-by: Joe McCain III --- core/src/error/err.rs | 3 +- core/src/error/kinds/external.rs | 5 +- core/src/error/mod.rs | 6 +- core/src/func/activate/binary.rs | 16 +- core/src/func/activate/mod.rs | 22 ++- core/src/func/activate/nl.rs | 281 ---------------------------- core/src/func/activate/nonlinear.rs | 177 ++++++++++++++++++ core/src/func/activate/utils.rs | 66 +++++++ core/src/func/mod.rs | 9 + core/src/traits/arr/create.rs | 56 ++++-- 10 files changed, 321 insertions(+), 320 deletions(-) delete mode 100644 core/src/func/activate/nl.rs create mode 100644 core/src/func/activate/nonlinear.rs create mode 100644 core/src/func/activate/utils.rs diff --git a/core/src/error/err.rs b/core/src/error/err.rs index d475b173..dc9d525c 100644 --- a/core/src/error/err.rs +++ b/core/src/error/err.rs @@ -49,9 +49,10 @@ impl Error { impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - write!(f, "{}: {}", self.kind, self.message) + write!(f, "({}) Error: {}", self.kind, self.message) } } + #[cfg(feature = "std")] impl std::error::Error for Error {} diff --git a/core/src/error/kinds/external.rs b/core/src/error/kinds/external.rs index 41a0c2ba..e9de1787 100644 --- a/core/src/error/kinds/external.rs +++ b/core/src/error/kinds/external.rs @@ -52,7 +52,10 @@ impl ExternalError { impl ErrorKind for ExternalError where E: Clone + ToString {} -impl core::fmt::Display for ExternalError where E: ToString { +impl core::fmt::Display for ExternalError +where + E: ToString, +{ fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { let msg = match self { ExternalError::Error(ref err) => err.to_string(), diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index 9a00fb03..958e6d4e 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -26,11 +26,7 @@ macro_rules! impl_error_type { }; } -impl_error_type!( - kinds::Errors, - kinds::PredictError, - crate::nn::ModelError -); +impl_error_type!(kinds::Errors, kinds::PredictError, crate::nn::ModelError); pub(crate) mod prelude { pub use super::err::Error; diff --git a/core/src/func/activate/binary.rs b/core/src/func/activate/binary.rs index 5b417dc0..3fcf6458 100644 --- a/core/src/func/activate/binary.rs +++ b/core/src/func/activate/binary.rs @@ -2,22 +2,10 @@ Appellation: binary Contrib: FL03 */ +use super::utils::heavyside; use nd::{Array, ArrayBase, Data, Dimension}; -use num::{One, Zero}; -/// -pub fn heavyside(x: T) -> T -where - T: One + PartialOrd + Zero, -{ - if x > T::zero() { - T::one() - } else { - T::zero() - } -} - -unary!(Heavyside::heavyside(self),); +unary!(Heavyside::heavyside(self)); macro_rules! impl_heavyside { ($($ty:ty),* $(,)*) => { diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs index 81488f57..15d0c2b3 100644 --- a/core/src/func/activate/mod.rs +++ b/core/src/func/activate/mod.rs @@ -2,16 +2,21 @@ Appellation: activate Contrib: FL03 */ -pub use self::{binary::*, linear::*, nl::*}; +#[doc(inline)] +pub use self::utils::*; +pub use self::{binary::*, linear::*, nonlinear::*}; + +pub(crate) mod utils; pub mod binary; pub mod linear; -pub mod nl; +pub mod nonlinear; pub(crate) mod prelude { pub use super::binary::*; pub use super::linear::*; - pub use super::nl::*; + pub use super::nonlinear::*; + pub use super::utils::*; pub use super::{Activate, Evaluate}; } @@ -26,13 +31,18 @@ pub trait Activate { fn activate(&self, args: T) -> Self::Output; } -#[doc(hidden)] -pub trait Evaluate { +/// [Evaluate] is used for _lazy_, structured functions that evaluate to +/// some value. +pub trait Evaluate { type Output; - fn eval(&self, args: T) -> Self::Output; + fn eval(&self) -> Self::Output; } +/* + ************* Implementations ************* +*/ + activator!(LinearActor::(T::clone) where T: Clone); impl Activate for F diff --git a/core/src/func/activate/nl.rs b/core/src/func/activate/nl.rs deleted file mode 100644 index 2d48b659..00000000 --- a/core/src/func/activate/nl.rs +++ /dev/null @@ -1,281 +0,0 @@ -/* - Appellation: sigmoid - Contrib: FL03 -*/ -use crate::math::Exp; -use nd::*; -use num::complex::{Complex, ComplexFloat}; -use num::traits::Zero; - -pub fn relu(args: T) -> T -where - T: PartialOrd + Zero, -{ - if args > T::zero() { - return args; - } - T::zero() -} - -pub fn sigmoid(args: T) -> T -where - T: ComplexFloat, -{ - (T::one() + args.neg().exp()).recip() -} - -pub fn softmax(args: &ArrayBase) -> Array -where - A: ComplexFloat + ScalarOperand, - D: Dimension, - S: Data, -{ - let e = args.exp(); - &e / e.sum() -} - -pub fn softmax_axis(args: &ArrayBase, axis: usize) -> Array -where - A: ComplexFloat + ScalarOperand, - D: RemoveAxis, - S: Data, -{ - let axis = Axis(axis); - let e = args.exp(); - &e / &e.sum_axis(axis) -} - -pub fn tanh(args: T) -> T -where - T: ComplexFloat, -{ - args.tanh() -} - -unary!( - ReLU::relu(self), - Sigmoid::sigmoid(self), - Softmax::softmax(self), - Tanh::tanh(self), -); - -pub trait SoftmaxAxis { - type Output; - - fn softmax(self) -> Self::Output; - - fn softmax_axis(self, axis: usize) -> Self::Output; -} - -pub trait NonLinear { - type Output; - - fn relu(self) -> Self::Output; - fn sigmoid(self) -> Self::Output; - fn softmax(self) -> Self::Output; - fn softmax_axis(self, axis: usize) -> Self::Output; - fn tanh(self) -> Self::Output; -} - -/* - ********** Implementations ********** -*/ -macro_rules! nonlinear { - ($($rho:ident::$call:ident<[$($T:ty),* $(,)?]>),* $(,)? ) => { - $( - nonlinear!(@loop $rho::$call<[$($T),*]>); - )* - }; - (@loop $rho:ident::$call:ident<[$($T:ty),* $(,)?]> ) => { - $( - nonlinear!(@impl $rho::$call<$T>); - )* - - // nonlinear!(@arr $rho::$call); - }; - (@impl $rho:ident::$call:ident<$T:ty>) => { - impl $rho for $T { - type Output = $T; - - fn $call(self) -> Self::Output { - $call(self) - } - } - - impl<'a> $rho for &'a $T { - type Output = $T; - - fn $call(self) -> Self::Output { - $call(*self) - } - } - - - - }; - (@arr $name:ident::$call:ident) => { - impl $name for ArrayBase - where - A: Clone + $name, - D: Dimension, - S: Data - { - type Output = Array<::Output, D>; - - fn $call(self) -> Self::Output { - self.mapv($name::$call) - } - } - - impl<'a, A, S, D> $name for &'a ArrayBase - where - A: Clone + $name, - D: Dimension, - S: Data - { - type Output = Array<::Output, D>; - - fn $call(self) -> Self::Output { - self.mapv($name::$call) - } - } - }; -} - -macro_rules! nonlinear_rho { - ($name:ident::$call:ident where A: $($rest:tt)* ) => { - impl $name for ArrayBase - where - D: Dimension, - S: Data, - A: Clone + $($rest)* - { - type Output = Array; - - fn $call(self) -> Self::Output { - self.mapv($call) - } - } - - impl<'a, A, S, D> $name for &'a ArrayBase - where - D: Dimension, - S: Data, - A: Clone + $($rest)* - { - type Output = Array; - - fn $call(self) -> Self::Output { - self.mapv($call) - } - } - }; -} - -nonlinear!( - ReLU::relu<[ - f32, - f64, - i8, - i16, - i32, - i64, - i128, - isize, - u8, - u16, - u32, - u64, - u128, - usize - ]>, - Sigmoid::sigmoid<[ - f32, - f64, - Complex, - Complex - ]>, - Tanh::tanh<[ - f32, - f64, - Complex, - Complex - ]>, -); - -nonlinear_rho!(ReLU::relu where A: PartialOrd + Zero); -nonlinear_rho!(Sigmoid::sigmoid where A: ComplexFloat); -nonlinear_rho!(Tanh::tanh where A: ComplexFloat); - -// impl ReLU for ArrayBase -// where -// A: Clone + PartialOrd + Zero, -// D: Dimension, -// S: Data, -// { -// type Output = Array; - -// fn relu(self) -> Self::Output { -// self.mapv(_relu) -// } -// } - -impl Softmax for ArrayBase -where - A: ComplexFloat + ScalarOperand, - D: Dimension, - S: Data, -{ - type Output = Array; - - fn softmax(self) -> Self::Output { - softmax(&self) - } -} - -impl<'a, A, S, D> Softmax for &'a ArrayBase -where - A: ComplexFloat + ScalarOperand, - D: Dimension, - S: Data, -{ - type Output = Array; - - fn softmax(self) -> Self::Output { - softmax(self) - } -} - -impl SoftmaxAxis for ArrayBase -where - A: ComplexFloat + ScalarOperand, - D: RemoveAxis, - S: Data, -{ - type Output = Array; - - fn softmax(self) -> Self::Output { - softmax(&self) - } - - fn softmax_axis(self, axis: usize) -> Self::Output { - softmax_axis(&self, axis) - } -} - -impl<'a, A, S, D> SoftmaxAxis for &'a ArrayBase -where - A: ComplexFloat + ScalarOperand, - D: RemoveAxis, - S: Data, -{ - type Output = Array; - - fn softmax(self) -> Self::Output { - softmax(self) - } - - fn softmax_axis(self, axis: usize) -> Self::Output { - softmax_axis(&self, axis) - } -} diff --git a/core/src/func/activate/nonlinear.rs b/core/src/func/activate/nonlinear.rs new file mode 100644 index 00000000..df576ca7 --- /dev/null +++ b/core/src/func/activate/nonlinear.rs @@ -0,0 +1,177 @@ +/* + Appellation: sigmoid + Contrib: FL03 +*/ +use super::utils::*; +use nd::*; +use num::complex::{Complex, ComplexFloat}; +use num::traits::Zero; + +unary!( + ReLU::relu(self), + Sigmoid::sigmoid(self), + Softmax::softmax(self), + Tanh::tanh(self), +); + +pub trait SoftmaxAxis: Softmax { + fn softmax_axis(self, axis: usize) -> Self::Output; +} + +pub trait NonLinear { + type Output; + + fn relu(self) -> Self::Output; + fn sigmoid(self) -> Self::Output; + fn softmax(self) -> Self::Output; + fn softmax_axis(self, axis: usize) -> Self::Output; + fn tanh(self) -> Self::Output; +} + +/* + ********** Implementations ********** +*/ +macro_rules! nonlinear { + ($($rho:ident::$call:ident<[$($T:ty),* $(,)?]>($f:expr)),* $(,)? ) => { + $( + nonlinear!(@loop $rho::$call<[$($T),*]>($f)); + )* + }; + (@loop $rho:ident::$call:ident<[$($T:ty),* $(,)?]>($f:expr) ) => { + $( + nonlinear!(@impl $rho::$call<$T>($f)); + )* + }; + (@impl $rho:ident::$call:ident<$T:ty>($f:expr)) => { + impl $rho for $T { + type Output = $T; + + fn $call(self) -> Self::Output { + $f(self) + } + } + + impl<'a> $rho for &'a $T { + type Output = $T; + + fn $call(self) -> Self::Output { + $f(*self) + } + } + }; +} + +macro_rules! nonlinear_rho { + ($name:ident::$call:ident where A: $($rest:tt)* ) => { + impl $name for ArrayBase + where + D: Dimension, + S: Data, + A: $($rest)* + { + type Output = Array; + + fn $call(self) -> Self::Output { + self.mapv($call) + } + } + + impl<'a, A, S, D> $name for &'a ArrayBase + where + D: Dimension, + S: Data, + A: $($rest)* + { + type Output = Array; + + fn $call(self) -> Self::Output { + self.mapv($call) + } + } + }; + (alt $name:ident::$call:ident where A: $($rest:tt)* ) => { + impl $name for ArrayBase + where + D: Dimension, + S: Data, + A: $($rest)* + { + type Output = Array; + + fn $call(self) -> Self::Output { + $call(&self) + } + } + + impl<'a, A, S, D> $name for &'a ArrayBase + where + D: Dimension, + S: Data, + A: $($rest)* + { + type Output = Array; + + fn $call(self) -> Self::Output { + $call(self) + } + } + }; +} + +nonlinear!( + ReLU::relu<[ + f32, + f64, + i8, + i16, + i32, + i64, + i128, + isize, + u8, + u16, + u32, + u64, + u128, + usize + ]>(relu), + Sigmoid::sigmoid<[ + f32, + f64, + Complex, + Complex + ]>(sigmoid), + Tanh::tanh<[ + f32, + f64, + Complex, + Complex + ]>(tanh), +); + +nonlinear_rho!(ReLU::relu where A: Clone + PartialOrd + Zero); +nonlinear_rho!(Sigmoid::sigmoid where A: ComplexFloat); +nonlinear_rho!(alt Softmax::softmax where A: ComplexFloat + ScalarOperand); +nonlinear_rho!(Tanh::tanh where A: ComplexFloat); + +impl SoftmaxAxis for ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: RemoveAxis, + S: Data, +{ + fn softmax_axis(self, axis: usize) -> Self::Output { + softmax_axis(&self, axis) + } +} + +impl<'a, A, S, D> SoftmaxAxis for &'a ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: RemoveAxis, + S: Data, +{ + fn softmax_axis(self, axis: usize) -> Self::Output { + softmax_axis(&self, axis) + } +} diff --git a/core/src/func/activate/utils.rs b/core/src/func/activate/utils.rs new file mode 100644 index 00000000..9eb0cee6 --- /dev/null +++ b/core/src/func/activate/utils.rs @@ -0,0 +1,66 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use crate::math::Exp; +use nd::prelude::{Array, ArrayBase, Axis, Dimension}; +use nd::{Data, RemoveAxis, ScalarOperand}; +use num::complex::ComplexFloat; +use num::traits::{One, Zero}; + +/// +pub fn heavyside(x: T) -> T +where + T: One + PartialOrd + Zero, +{ + if x > T::zero() { + T::one() + } else { + T::zero() + } +} +/// +pub fn relu(args: T) -> T +where + T: PartialOrd + Zero, +{ + if args > T::zero() { + return args; + } + T::zero() +} +/// +pub fn sigmoid(args: T) -> T +where + T: ComplexFloat, +{ + (T::one() + args.neg().exp()).recip() +} +/// +pub fn softmax(args: &ArrayBase) -> Array +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, +{ + let e = args.exp(); + &e / e.sum() +} +/// +pub fn softmax_axis(args: &ArrayBase, axis: usize) -> Array +where + A: ComplexFloat + ScalarOperand, + D: RemoveAxis, + S: Data, +{ + let axis = Axis(axis); + let e = args.exp(); + &e / &e.sum_axis(axis) +} +/// +pub fn tanh(args: T) -> T +where + T: ComplexFloat, +{ + args.tanh() +} diff --git a/core/src/func/mod.rs b/core/src/func/mod.rs index 96513d96..005fd623 100644 --- a/core/src/func/mod.rs +++ b/core/src/func/mod.rs @@ -13,3 +13,12 @@ pub(crate) mod prelude { pub use super::activate::prelude::*; pub use super::loss::prelude::*; } + +#[doc(hidden)] +pub trait Apply { + type Output; + + fn apply(&self, f: F) -> Self::Output + where + F: Fn(T) -> U; +} diff --git a/core/src/traits/arr/create.rs b/core/src/traits/arr/create.rs index 8c45d927..3a49e674 100644 --- a/core/src/traits/arr/create.rs +++ b/core/src/traits/arr/create.rs @@ -3,15 +3,26 @@ Contrib: FL03 */ use nd::{ArrayBase, DataOwned, Dimension, Ix2, ShapeBuilder}; -use num::traits::Num; +use num::traits::{Num, One, Zero}; -pub trait NdLike +pub trait NdLike where - Self: DefaultLike - + FillLike - + OnesLike - + ZerosLike, + A: Clone, + D: Dimension, { + type Output; + + fn default_like(&self) -> Self::Output + where + A: Default; + + fn ones_like(&self) -> Self::Output + where + A: One; + + fn zeros_like(&self) -> Self::Output + where + A: Zero; } pub trait ArrayLike @@ -42,15 +53,36 @@ macro_rules! ndlike { ndlike!(DefaultLike::default_like, OnesLike::ones_like, ZerosLike::zeros_like, FillLike::::fill_like(elem: T)); /* - ******** implementations ******** + ************* Implementations ************* */ - -impl NdLike> for ArrayBase +impl NdLike for ArrayBase where - A: Clone + Default + Num, + A: Clone + Num, D: Dimension, S: DataOwned, { + type Output = ArrayBase; + + fn default_like(&self) -> Self::Output + where + A: Default, + { + ArrayBase::default(self.dim()) + } + + fn ones_like(&self) -> Self::Output + where + A: One, + { + ArrayBase::ones(self.dim()) + } + + fn zeros_like(&self) -> Self::Output + where + A: Zero, + { + ArrayBase::zeros(self.dim()) + } } impl ArrayLike for ArrayBase @@ -105,5 +137,5 @@ macro_rules! impl_ndlike { } impl_ndlike!(DefaultLike::default_like.default: Default); -impl_ndlike!(OnesLike::ones_like.ones: Clone + num::One); -impl_ndlike!(ZerosLike::zeros_like.zeros: Clone + num::Zero); +impl_ndlike!(OnesLike::ones_like.ones: Clone + One); +impl_ndlike!(ZerosLike::zeros_like.zeros: Clone + Zero); From 404f62c12ee47f99bf5ce568ee1c8e6c44629832 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Mon, 27 May 2024 10:24:04 -0500 Subject: [PATCH 08/12] update Signed-off-by: Joe McCain III --- core/src/func/activate/mod.rs | 18 ++-- core/src/func/activate/utils.rs | 7 +- core/src/nn/mod.rs | 2 + core/src/nn/model/module.rs | 4 +- core/src/nn/optim/mod.rs | 17 ++++ core/src/nn/optim/optimizer.rs | 14 +++ core/src/traits/ops.rs | 41 +++++---- core/src/traits/params.rs | 33 ++++++- core/src/traits/setup.rs | 64 ++------------ core/src/traits/train.rs | 3 + models/kan/src/actor.rs | 58 +++++++++---- models/kan/src/lib.rs | 6 +- models/kan/src/model/kan.rs | 6 ++ models/kan/src/model/mod.rs | 7 ++ models/kan/src/primitives.rs | 0 models/linear/src/impls/params/impl_params.rs | 4 +- models/linear/src/mlp/mod.rs | 20 ----- models/linear/src/mlp/perceptron.rs | 85 ++++++------------- models/linear/tests/mlp.rs | 4 +- 19 files changed, 205 insertions(+), 188 deletions(-) create mode 100644 core/src/nn/optim/mod.rs create mode 100644 core/src/nn/optim/optimizer.rs create mode 100644 models/kan/src/model/kan.rs create mode 100644 models/kan/src/primitives.rs diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs index 15d0c2b3..f20a7e32 100644 --- a/core/src/func/activate/mod.rs +++ b/core/src/func/activate/mod.rs @@ -17,7 +17,7 @@ pub(crate) mod prelude { pub use super::linear::*; pub use super::nonlinear::*; pub use super::utils::*; - pub use super::{Activate, Evaluate}; + pub use super::{Activate, NdActivate}; } /// [Activate] designates a function or structure that can be used @@ -31,14 +31,18 @@ pub trait Activate { fn activate(&self, args: T) -> Self::Output; } -/// [Evaluate] is used for _lazy_, structured functions that evaluate to -/// some value. -pub trait Evaluate { - type Output; +pub trait NdActivate { + type Data; - fn eval(&self) -> Self::Output; -} + fn activate(&self, f: F) -> nd::Array + where + F: FnMut(A) -> B; + fn activate_mut<'a, B, F>(&'a mut self, f: F) + where + A: 'a, + F: FnMut(&'a A) -> B; +} /* ************* Implementations ************* */ diff --git a/core/src/func/activate/utils.rs b/core/src/func/activate/utils.rs index 9eb0cee6..c4fa19fe 100644 --- a/core/src/func/activate/utils.rs +++ b/core/src/func/activate/utils.rs @@ -8,7 +8,7 @@ use nd::{Data, RemoveAxis, ScalarOperand}; use num::complex::ComplexFloat; use num::traits::{One, Zero}; -/// +/// Heaviside activation function pub fn heavyside(x: T) -> T where T: One + PartialOrd + Zero, @@ -25,9 +25,10 @@ where T: PartialOrd + Zero, { if args > T::zero() { - return args; + args + } else { + T::zero() } - T::zero() } /// pub fn sigmoid(args: T) -> T diff --git a/core/src/nn/mod.rs b/core/src/nn/mod.rs index 172b7e7d..45b43448 100644 --- a/core/src/nn/mod.rs +++ b/core/src/nn/mod.rs @@ -10,12 +10,14 @@ pub mod dropout; pub mod error; pub mod mask; pub mod model; +pub mod optim; pub(crate) mod prelude { pub use super::dropout::*; pub use super::error::*; pub use super::mask::prelude::*; pub use super::model::prelude::*; + pub use super::optim::prelude::*; } #[cfg(any(feature = "alloc", feature = "std"))] diff --git a/core/src/nn/model/module.rs b/core/src/nn/model/module.rs index edb450fd..bab6dd13 100644 --- a/core/src/nn/model/module.rs +++ b/core/src/nn/model/module.rs @@ -2,7 +2,7 @@ Appellation: modules Contrib: FL03 */ -use crate::{Config, Params, Predict}; +use crate::{Config, Parameters, Predict}; pub type ModuleDyn = Box>; pub type DynModuleExt = Box>; @@ -13,7 +13,7 @@ pub type Stack = Vec &Self::Config; diff --git a/core/src/nn/optim/mod.rs b/core/src/nn/optim/mod.rs new file mode 100644 index 00000000..c8c57c77 --- /dev/null +++ b/core/src/nn/optim/mod.rs @@ -0,0 +1,17 @@ +/* + Appellation: optim + Contrib: FL03 +*/ +//! # Optimizers +//! +//! This module contains various optimizers used for training neural networks. +pub use self::optimizer::*; + +pub(crate) mod optimizer; + +pub(crate) mod prelude { + pub use super::optimizer::*; + pub use super::Optimize; +} + +pub trait Optimize {} diff --git a/core/src/nn/optim/optimizer.rs b/core/src/nn/optim/optimizer.rs new file mode 100644 index 00000000..1a79eeda --- /dev/null +++ b/core/src/nn/optim/optimizer.rs @@ -0,0 +1,14 @@ +/* + Appellation: optimizer + Contrib: FL03 +*/ + +pub struct OptimizerConfig { + pub learning_rate: f32, + pub momentum: f32, + pub weight_decay: f32, +} + +pub struct OptimizerBase { + pub config: C, +} diff --git a/core/src/traits/ops.rs b/core/src/traits/ops.rs index 48e1cf3a..d936c999 100644 --- a/core/src/traits/ops.rs +++ b/core/src/traits/ops.rs @@ -9,18 +9,22 @@ pub trait Apply { fn apply(&self, f: F) -> Self::Output where F: Fn(T) -> U; - - fn apply_mut(&mut self, f: F) -> Self::Output - where - F: FnMut(T) -> U; } -pub trait ApplyOn { +/// An alternative trait for evaluating the logic of an expression; +/// [Eval] is a substitute for functional traits (Fn, FnMut, and FnOnce) as +/// implementing these traits is currently unstable. +pub trait Eval { type Output; - fn apply(self, f: F) -> Self::Output - where - F: FnMut(T) -> U; + fn eval(&self, args: T) -> Self::Output; +} +/// [EvaluateLazy] is used for _lazy_, structured functions that evaluate to +/// some value. +pub trait EvaluateLazy { + type Output; + + fn eval(&self) -> Self::Output; } pub trait Transform { @@ -32,16 +36,21 @@ pub trait Transform { /* ************* Implementations ************* */ -impl ApplyOn for S +impl Eval for F where - S: Iterator, + F: Fn(X) -> Y, { - type Output = core::iter::Map; + type Output = Y; - fn apply(self, f: F) -> Self::Output - where - F: FnMut(T) -> U, - { - self.map(f) + fn eval(&self, args: X) -> Self::Output { + self(args) + } +} + +impl Eval for Box> { + type Output = Y; + + fn eval(&self, args: X) -> Self::Output { + self.as_ref().eval(args) } } diff --git a/core/src/traits/params.rs b/core/src/traits/params.rs index aa3b379c..e788fcd2 100644 --- a/core/src/traits/params.rs +++ b/core/src/traits/params.rs @@ -2,14 +2,16 @@ Appellation: params Contrib: FL03 */ +use nd::{ArrayBase, Dimension, RawData}; -/// A `Params` object is used to store the various learnable parameters of a model. +/// [Parameters] describes any object capable of serving as a store for the +/// model's learnable parameters. /// /// ### Specifications /// /// - `Elem`: The type of the elements being stored /// -pub trait Params { +pub trait Parameters { type Elem; } @@ -17,6 +19,31 @@ pub trait ParamFeatures { type Dim: nd::Dimension; } -pub trait Parameter { +/// A `Parameter` describes learnable parameters in a model. +pub trait Parameter { + type Data; +} + +pub trait ParameterExt: Parameter { type Kind: 'static; } + +/* + ************* Implementations ************* +*/ + +impl Parameter for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Data = S; +} + +impl Parameters for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Elem = A; +} diff --git a/core/src/traits/setup.rs b/core/src/traits/setup.rs index e95fbb3e..a895534f 100644 --- a/core/src/traits/setup.rs +++ b/core/src/traits/setup.rs @@ -2,12 +2,15 @@ Appellation: setup Contrib: FL03 */ -use core::borrow::{Borrow, BorrowMut}; + +pub trait FromConfig { + fn from_config(config: Cnf) -> Self; +} /// A trait used to denote objects that may be used for configuring various items pub trait Config {} -pub trait CompositConfig: Config { +pub trait CompositeConfig: Config { type Ctx: Config; fn children(&self) -> Vec>; @@ -15,65 +18,12 @@ pub trait CompositConfig: Config { fn context(&self) -> Self::Ctx; } -/// [Configuration] describes composite configuration objects; -/// A configuration object is allowed to inherit from another configuration object -pub trait Configuration -where - C: Config, - Self::Config: Borrow, -{ +pub trait Configurable { type Config: Config; - fn root(&self) -> &C; - - fn set(&mut self, config: Self::Config); - - fn set_root(&mut self, config: C); -} - -pub trait Init { - fn init(self) -> Self; -} - -pub trait InitInplace { - fn init(&mut self); -} - -pub trait Setup { - type Config; - - fn setup(&mut self, config: Self::Config); -} - -pub trait Context -where - C: Config, -{ - type Cnf: Configuration; - - fn config(&self) -> Self::Cnf; + fn configure(config: Self::Config) -> Self; } /* ************* Implementations ************* */ - -impl Configuration for D -where - C: Config, - D: Config + BorrowMut, -{ - type Config = D; - - fn root(&self) -> &C { - self.borrow() - } - - fn set(&mut self, config: Self::Config) { - *self = config; - } - - fn set_root(&mut self, config: C) { - *self.borrow_mut() = config; - } -} diff --git a/core/src/traits/train.rs b/core/src/traits/train.rs index 4c48d4d6..5f8d82e6 100644 --- a/core/src/traits/train.rs +++ b/core/src/traits/train.rs @@ -28,6 +28,9 @@ pub trait Train: Compile { fn train(&mut self) -> Self::Output; } +/* + ************* Implementations ************* +*/ impl Backward for Option where S: Backward, diff --git a/models/kan/src/actor.rs b/models/kan/src/actor.rs index 7fa1cb4d..6ffc6c62 100644 --- a/models/kan/src/actor.rs +++ b/models/kan/src/actor.rs @@ -6,31 +6,39 @@ pub use self::config::ActorConfig; pub(crate) mod config; -use concision::prelude::{Activate, Predict, PredictError}; +use concision::prelude::{Eval, Predict, PredictError}; use core::ops::Mul; use nd::prelude::*; use splines::interpolate::{Interpolate, Interpolator}; use splines::Spline; -/// An `actor` describe the learnable activation functions -/// employed throughout KAN networks. +#[doc(hidden)] +pub type NdSpline = Spline, Array1>; + +/// An `actor` represents the learned activation functions +/// described within the KAN paper. +/// +/// The learned activation functions are univariate and continuous. +/// +/// ### Parameters +/// +/// - b(**x**): bias function +/// - spline(**x**): spline function; typically employs `Linear` interpolation +/// - **ω**: weight factor /// /// The learned functions are one-dimensional, continuous pub struct Actor { pub(crate) bias: B, - pub(crate) omega: Array1, pub(crate) spline: Spline, + pub(crate) weight: Array1, } -impl Actor -where - B: Activate, -{ - pub fn new(bias: B, omega: Array1, spline: Spline) -> Self { +impl Actor { + pub fn new(bias: B, spline: Spline, weight: Array1) -> Self { Self { bias, - omega, spline, + weight, } } @@ -38,26 +46,42 @@ where &self.bias } - pub fn omega(&self) -> &Array1 { - &self.omega + pub fn sample(&self, x: T) -> Option + where + T: Interpolator, + V: Interpolate, + { + self.spline().sample(x) } pub fn spline(&self) -> &Spline { &self.spline } + + pub fn spline_mut(&mut self) -> &mut Spline { + &mut self.spline + } + + pub fn weight(&self) -> &Array1 { + &self.weight + } + + pub fn weight_mut(&mut self) -> &mut Array1 { + &mut self.weight + } } -impl Predict> for Actor +impl Predict> for Actor where - B: Activate>, + B: Eval>, T: Interpolator, V: Interpolate, - Array1: Clone + Mul, + for<'a> &'a Array1: Mul, { - type Output = as Mul>::Output; + type Output = Z; fn predict(&self, x: &Array1) -> Result { let y = x.mapv(|xi| self.spline.sample(xi).unwrap()); - Ok(self.omega.clone() * self.bias.activate(y)) + Ok(self.weight() * self.bias().eval(y)) } } diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs index bfcc81df..819aad35 100644 --- a/models/kan/src/lib.rs +++ b/models/kan/src/lib.rs @@ -24,8 +24,12 @@ extern crate ndarray as nd; #[doc(inline)] pub use self::actor::Actor; +#[doc(inline)] +pub use self::model::Kan; pub mod actor; pub mod model; -pub mod prelude {} +pub mod prelude { + pub use super::model::prelude::*; +} diff --git a/models/kan/src/model/kan.rs b/models/kan/src/model/kan.rs new file mode 100644 index 00000000..9cce9759 --- /dev/null +++ b/models/kan/src/model/kan.rs @@ -0,0 +1,6 @@ +/* + Appellation: kan + Contrib: FL03 +*/ + +pub struct Kan; diff --git a/models/kan/src/model/mod.rs b/models/kan/src/model/mod.rs index fb253b77..ef66d5df 100644 --- a/models/kan/src/model/mod.rs +++ b/models/kan/src/model/mod.rs @@ -2,3 +2,10 @@ Appellation: model Contrib: FL03 */ +pub use self::kan::*; + +pub(crate) mod kan; + +pub(crate) mod prelude { + pub use super::kan::Kan; +} diff --git a/models/kan/src/primitives.rs b/models/kan/src/primitives.rs new file mode 100644 index 00000000..e69de29b diff --git a/models/linear/src/impls/params/impl_params.rs b/models/linear/src/impls/params/impl_params.rs index 72df65a3..a463961d 100644 --- a/models/linear/src/impls/params/impl_params.rs +++ b/models/linear/src/impls/params/impl_params.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ use crate::params::ParamsBase; -use concision::prelude::{Params, Predict, PredictError}; +use concision::prelude::{Parameters, Predict, PredictError}; use core::ops::Add; use nd::linalg::Dot; use nd::*; @@ -24,7 +24,7 @@ where } } -impl Params for ParamsBase +impl Parameters for ParamsBase where D: RemoveAxis, S: RawData, diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index 00cc601e..2bf10b43 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -15,23 +15,3 @@ pub(crate) mod perceptron; pub(crate) mod prelude { pub use super::perceptron::Perceptron; } - -pub trait MultiLayerPerceptron { - type Input; - type Hidden; - type Output; -} - -pub trait Container { - type Elem; -} - -pub trait Params { - type Data: Container; - type Dim; - type Elem; -} - -pub trait Neuron { - type Rho; -} diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index b0bf7fb8..afcf4cbf 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -2,59 +2,13 @@ Appellation: perceptron Contrib: FL03 */ -use concision::prelude::{Activate, Module, Predict, PredictError, ReLU}; -use nd::prelude::*; -use nd::Data; -use num::traits::Zero; +use concision::prelude::{Activate, Module, Predict, PredictError}; +use nd::{ArrayBase, Data, Dimension}; -pub struct Rho(T); - -impl Rho { - pub fn new(rho: T) -> Self { - Self(rho) - } - - pub fn get(&self) -> &T { - &self.0 - } - - pub fn get_mut(&mut self) -> &mut T { - &mut self.0 - } - - pub fn activate(&self) -> &T { - &self.0 - } -} -pub struct Relu; - -impl Activate> for Relu -where - A: Clone + PartialOrd + Zero, - D: Dimension, - S: Data, -{ - type Output = Array; - - fn activate(&self, args: ArrayBase) -> Self::Output { - args.relu() - } -} - -impl<'a, A, S, D> Activate<&'a ArrayBase> for Relu -where - A: Clone + PartialOrd + Zero, - D: Dimension, - S: Data, -{ - type Output = Array; - - fn activate(&self, args: &'a ArrayBase) -> Self::Output { - args.relu() - } -} - -pub struct Perceptron +/// Perceptrons are the fundamental building block of multi-layer perceptrons (MLPs). +/// They are used to model a particular layer within a neural network. Generally speaking, +/// Perceptrons consist of a linear set of parameters and an activation function. +pub struct Perceptron where M: Module, { @@ -78,15 +32,30 @@ where } } -impl Predict for Perceptron +impl Predict> for Perceptron where - F: for<'a> Activate<&'a Y, Output = Z>, - M: Module + Predict, + D: Dimension, + S: Data, + F: Activate, + M: Module + Predict>, { - type Output = Z; + type Output = F::Output; - fn predict(&self, args: &X) -> Result { + fn predict(&self, args: &ArrayBase) -> Result { let res = self.module.predict(args)?; - Ok(self.rho.activate(&res)) + Ok(self.rho.activate(res)) } } + +// impl Predict for Perceptron +// where +// F: Activate, +// M: Module + Predict, +// { +// type Output = F::Output; + +// fn predict(&self, args: &X) -> Result { +// let res = self.module.predict(args)?; +// Ok(self.rho.activate(res)) +// } +// } diff --git a/models/linear/tests/mlp.rs b/models/linear/tests/mlp.rs index 3b8d020d..3bd5b9b8 100644 --- a/models/linear/tests/mlp.rs +++ b/models/linear/tests/mlp.rs @@ -6,7 +6,7 @@ extern crate concision_core as cnc; extern crate concision_linear as linear; use cnc::prelude::{linarr, Forward, ReLU}; -use linear::mlp::{self, Perceptron}; +use linear::mlp::Perceptron; use linear::{Biased, Features, Linear}; use ndarray::prelude::*; @@ -18,6 +18,6 @@ fn test_perceptron() { let features = Features::new(1, 300); let data = linarr::((samples, features.dmodel())).unwrap(); let layer = Linear::::lecun_normal(features, 1); - let mlp = Perceptron::new(layer.clone(), mlp::Relu); + let mlp = Perceptron::new(layer.clone(), Box::new(ReLU::relu)); assert_eq!(mlp.forward(&data), layer.forward(&data).relu()); } From 9b0fa2ab4bc076e93039d36ca4ad536799a5cfde Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Mon, 27 May 2024 11:39:35 -0500 Subject: [PATCH 09/12] update Signed-off-by: Joe McCain III --- Cargo.toml | 2 +- core/src/error/mod.rs | 17 +----- core/src/macros.rs | 69 +++++++++++++---------- core/src/macros/errors.rs | 27 +++++++++ core/src/nn/dropout.rs | 36 ++++++++++-- core/src/nn/model/module.rs | 6 +- core/src/ops/pad.rs | 76 ++------------------------ core/src/ops/pad/error.rs | 19 +++++++ core/src/ops/pad/utils.rs | 68 +++++++++++++++++++++++ core/src/traits/mod.rs | 2 + core/src/traits/predict.rs | 25 +++++---- {data => core}/src/traits/shape.rs | 0 data/src/traits/build.rs | 2 +- data/src/traits/data/container.rs | 3 +- data/src/traits/mod.rs | 5 +- models/linear/src/mlp/mod.rs | 7 +++ models/linear/src/mlp/perceptron.rs | 14 +---- models/linear/src/norm/layer/config.rs | 2 +- models/linear/src/norm/layer/model.rs | 20 +------ models/linear/src/params/store.rs | 2 +- models/linear/tests/params.rs | 4 +- models/transformers/tests/ffn.rs | 7 ++- 22 files changed, 235 insertions(+), 178 deletions(-) create mode 100644 core/src/macros/errors.rs create mode 100644 core/src/ops/pad/error.rs create mode 100644 core/src/ops/pad/utils.rs rename {data => core}/src/traits/shape.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 6a540ea4..71f4271a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace.package] authors = ["FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)"] categories = [ "science" ] -description = "Concision is a complete data-science toolkit written in Rust" +description = "Concision is a toolkit for building powerful machine-learning models in Rust." edition = "2021" homepage = "https://github.com/FL03/concision/wiki" keywords = ["data-science", "scsys", "toolkit"] diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index 958e6d4e..a523fb78 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -10,23 +10,8 @@ pub mod kinds; pub trait ErrorKind: Clone + ToString {} -macro_rules! impl_error_type { - ($($ty:ty),* $(,)*) => { - $(impl_error_type!(@impl $ty);)* - }; - (@impl $ty:ty) => { - impl ErrorKind for $ty {} - impl_error_type!(@std $ty); - }; - (@std $ty:ty) => { - - #[cfg(feature = "std")] - impl std::error::Error for $ty {} - }; -} - -impl_error_type!(kinds::Errors, kinds::PredictError, crate::nn::ModelError); +impl_err!(kinds::Errors, kinds::PredictError, crate::nn::ModelError); pub(crate) mod prelude { pub use super::err::Error; diff --git a/core/src/macros.rs b/core/src/macros.rs index aaf6ceb6..4195f643 100644 --- a/core/src/macros.rs +++ b/core/src/macros.rs @@ -9,37 +9,23 @@ mod builder; #[macro_use] mod enums; #[macro_use] +mod errors; +#[macro_use] mod getters; #[macro_use] mod ops; #[macro_use] mod toggle; -/// AS +/// Generates methods for forwarding [dimensional](ndarray::Dimension) related methods inherited from +/// a particular field (or method via `$name()`) which references either an [ArrayBase](ndarray::ArrayBase) +/// or another `type` implementing #[macro_export] macro_rules! dimensional { - - (dim: $name:ident$(())?) => { - /// Returns a reference to the current dimension, as a slice. - pub fn as_slice(&self) -> &[usize] { - self.$name$(())?.shape() - } - - pub fn into_pattern(self) -> D::Pattern { - self.$name$(())?.into_pattern() - } - - pub fn ndim(&self) -> usize { - self.$name$(())?.ndim() - } - - pub fn raw_dim(&self) -> D { - self.$name$(())?.dim().clone() - } + ($name:ident$($rest:tt)*) => { + $crate::dimensional!(@impl $name$($rest)*); }; - - - ($name:ident) => { + (@impl $name:ident) => { /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension pub fn dim(&self) -> D::Pattern { self.$name.dim() @@ -48,17 +34,16 @@ macro_rules! dimensional { pub fn ndim(&self) -> usize { self.$name.ndim() } - /// Returns the raw dimension [D](ndarray::Dimension) + /// Forwards the pub fn raw_dim(&self) -> D { - self.$name.dim() + self.$name.raw_dim() } - /// Returns a reference to the current dimension, as a slice. + /// Returns a reference the shape of the dimension as a slice pub fn shape(&self) -> &[usize] { self.$name.shape() } }; - - ($name:ident()) => { + (@impl $name:ident()) => { /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension pub fn dim(&self) -> D::Pattern { self.$name().dim() @@ -67,13 +52,39 @@ macro_rules! dimensional { pub fn ndim(&self) -> usize { self.$name().ndim() } - /// Returns the raw dimension [D](ndarray::Dimension) + /// Forwards the pub fn raw_dim(&self) -> D { self.$name().raw_dim() } - /// Returns a reference to the current dimension, as a slice. + /// Returns a reference the shape of the dimension as a slice pub fn shape(&self) -> &[usize] { self.$name().shape() } }; } + +/// Implement methods native to the `ndarray` [dimension](ndarray::Dimension) +#[macro_export] +macro_rules! fwd_ndim { + ($name:ident$(())?) => { + fwd_ndim!(@impl $name$(())?); + }; + (@impl $name:ident) => { + /// Returns a reference to the current dimension, as a slice. + pub fn as_slice(&self) -> &[usize] { + self.$name$(())?.shape() + } + + pub fn into_pattern(self) -> D::Pattern { + self.$name$(())?.into_pattern() + } + + pub fn ndim(&self) -> usize { + self.$name$(())?.ndim() + } + + pub fn raw_dim(&self) -> D { + self.$name$(())?.dim().clone() + } + } +} \ No newline at end of file diff --git a/core/src/macros/errors.rs b/core/src/macros/errors.rs new file mode 100644 index 00000000..a3abf8c3 --- /dev/null +++ b/core/src/macros/errors.rs @@ -0,0 +1,27 @@ +/* + Appellation: errors + Contrib: FL03 +*/ + + +macro_rules! impl_err { + ($($ty:ty),* $(,)*) => { + $(impl_err!(@impl $ty);)* + }; + (@impl $ty:ty) => { + impl $crate::error::ErrorKind for $ty {} + + #[cfg(feature = "std")] + impl std::error::Error for $ty {} + }; +} + +#[allow(unused_macros)] +macro_rules! err_from { + ($($ty:ty),* $(,)*) => { + $(err_from!(@impl $ty);)* + }; + (@impl $ty:ty) => { + + }; +} \ No newline at end of file diff --git a/core/src/nn/dropout.rs b/core/src/nn/dropout.rs index 772d2fa9..f52df8c5 100644 --- a/core/src/nn/dropout.rs +++ b/core/src/nn/dropout.rs @@ -3,7 +3,7 @@ Contrib: FL03 */ #![allow(unused_imports)] -use crate::Forward; +use crate::{Eval, Predict, PredictError}; use nd::prelude::*; use nd::{DataOwned, ScalarOperand}; #[cfg(feature = "rand")] @@ -69,7 +69,7 @@ impl Dropout { pub fn new(p: f64) -> Self { Self { p } } - + #[cfg(feature = "rand")] pub fn apply(&self, input: &ArrayBase) -> Array where A: Num + ScalarOperand, @@ -91,7 +91,21 @@ impl Default for Dropout { } #[cfg(feature = "rand")] -impl Forward> for Dropout +impl Eval> for Dropout +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + type Output = Array; + + fn eval(&self, input: ArrayBase) -> Self::Output { + input.dropout(self.p) + } +} + +#[cfg(feature = "rand")] +impl<'a, A, S, D> Eval<&'a ArrayBase> for Dropout where A: Num + ScalarOperand, D: Dimension, @@ -99,7 +113,21 @@ where { type Output = Array; - fn forward(&self, input: &ArrayBase) -> Self::Output { + fn eval(&self, input: &'a ArrayBase) -> Self::Output { input.dropout(self.p) } } + +#[cfg(feature = "rand")] +impl Predict> for Dropout +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + type Output = Array; + + fn predict(&self, input: &ArrayBase) -> Result { + Ok(input.dropout(self.p)) + } +} diff --git a/core/src/nn/model/module.rs b/core/src/nn/model/module.rs index bab6dd13..a1ba56e0 100644 --- a/core/src/nn/model/module.rs +++ b/core/src/nn/model/module.rs @@ -2,7 +2,7 @@ Appellation: modules Contrib: FL03 */ -use crate::{Config, Parameters, Predict}; +use crate::{Config, Parameters, Forward}; pub type ModuleDyn = Box>; pub type DynModuleExt = Box>; @@ -22,10 +22,10 @@ pub trait Module { fn params_mut(&mut self) -> &mut Self::Params; } -pub trait Layer: Module + Predict {} +pub trait Layer: Module + Forward {} /* ************* Implementations ************* */ -impl Layer for M where M: Module + Predict {} +impl Layer for M where M: Module + Forward {} diff --git a/core/src/ops/pad.rs b/core/src/ops/pad.rs index e6962770..ba3554d6 100644 --- a/core/src/ops/pad.rs +++ b/core/src/ops/pad.rs @@ -2,10 +2,13 @@ Appellation: pad Contrib: FL03 */ -pub use self::{action::PadAction, mode::PadMode, utils::*}; +#[doc(inline)] +pub use self::{action::*, error::*, mode::*, utils::*}; pub(crate) mod action; +pub(crate) mod error; pub(crate) mod mode; +pub(crate) mod utils; use nd::{Array, ArrayBase, DataOwned, Dimension}; use num::traits::{FromPrimitive, Num}; @@ -25,7 +28,7 @@ where type Output = Array; fn pad(&self, mode: PadMode, pad: &[[usize; 2]]) -> Self::Output { - self::utils::pad(self, pad, mode) + utils::pad(self, pad, mode).unwrap() } } @@ -65,72 +68,3 @@ impl Padding { self } } - -mod utils { - #![cfg(any(feature = "std", feature = "alloc"))] - use super::{PadAction, PadMode}; - use crate::traits::ArrayLike; - use nd::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; - use num::{FromPrimitive, Num}; - - #[cfg(all(feature = "alloc", no_std))] - use alloc::borrow::Cow; - #[cfg(feature = "std")] - use std::borrow::Cow; - - fn reader(nb_dim: usize, pad: &[[usize; 2]]) -> Cow<[[usize; 2]]> { - if pad.len() == 1 && pad.len() < nb_dim { - // The user provided a single padding for all dimensions - Cow::from(vec![pad[0]; nb_dim]) - } else if pad.len() == nb_dim { - Cow::from(pad) - } else { - panic!("Inconsistant number of dimensions and pad arrays"); - } - } - - pub fn pad(data: &ArrayBase, pad: &[[usize; 2]], mode: PadMode) -> Array - where - A: Copy + FromPrimitive + Num, - D: Dimension, - S: DataOwned, - { - let pad = reader(data.ndim(), pad); - let mut new_dim = data.raw_dim(); - for (ax, (&ax_len, pad)) in data.shape().iter().zip(pad.iter()).enumerate() { - new_dim[ax] = ax_len + pad[0] + pad[1]; - } - - // let mut padded = array_like(&data, new_dim, mode.init()); - let mut padded = data.array_like(new_dim, mode.init()).to_owned(); - pad_to(data, &pad, mode, &mut padded); - padded - } - - pub fn pad_to( - data: &ArrayBase, - pad: &[[usize; 2]], - mode: PadMode, - output: &mut Array, - ) where - A: Copy + FromPrimitive + Num, - D: Dimension, - S: Data, - { - let pad = reader(data.ndim(), pad); - - // Select portion of padded array that needs to be copied from the original array. - output - .slice_each_axis_mut(|ad| { - let AxisDescription { axis, len, .. } = ad; - let pad = pad[axis.index()]; - Slice::from(pad[0]..len - pad[1]) - }) - .assign(data); - - match mode.action() { - PadAction::StopAfterCopy => { /* Nothing */ } - _ => unimplemented!(), - } - } -} diff --git a/core/src/ops/pad/error.rs b/core/src/ops/pad/error.rs new file mode 100644 index 00000000..3a8ab0d9 --- /dev/null +++ b/core/src/ops/pad/error.rs @@ -0,0 +1,19 @@ +/* + Appellation: error + Contrib: FL03 +*/ +use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumMessage, EnumString, EnumIter, VariantArray, VariantNames}; + +pub type PadResult = Result; + +#[derive(AsRefStr, Clone, Copy, Debug, Default, Display, EnumCount, EnumIs, EnumIter, EnumMessage, EnumString, Eq, Ord, PartialEq, PartialOrd, VariantArray, VariantNames)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(rename_all = "snake_case"))] +#[cfg_attr(feature = "std", derive(std::hash::Hash))] +#[strum(serialize_all = "snake_case")] +#[repr(u8)] +pub enum PadError { + #[default] + InconsistentDimensions, +} + +impl_err!(PadError); diff --git a/core/src/ops/pad/utils.rs b/core/src/ops/pad/utils.rs new file mode 100644 index 00000000..5b5113ab --- /dev/null +++ b/core/src/ops/pad/utils.rs @@ -0,0 +1,68 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use super::{PadAction, PadError, PadMode}; +use crate::traits::ArrayLike; +use nd::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; +use num::{FromPrimitive, Num}; + + +fn reader(nb_dim: usize, pad: &[[usize; 2]]) -> Result, PadError> { + if pad.len() == 1 && pad.len() < nb_dim { + // The user provided a single padding for all dimensions + Ok(vec![pad[0]; nb_dim]) + } else if pad.len() == nb_dim { + Ok(pad.to_vec()) + } + else { + Err(PadError::InconsistentDimensions) + } +} + +pub fn pad(data: &ArrayBase, pad: &[[usize; 2]], mode: PadMode) -> Result, PadError> +where + A: Copy + FromPrimitive + Num, + D: Dimension, + S: DataOwned, +{ + let pad = reader(data.ndim(), pad)?; + let mut dim = data.raw_dim(); + for (ax, (&ax_len, pad)) in data.shape().iter().zip(pad.iter()).enumerate() { + dim[ax] = ax_len + pad[0] + pad[1]; + } + + let mut padded = data.array_like(dim, mode.init()).to_owned(); + let _ = pad_to(data, &pad, mode, &mut padded)?; + Ok(padded) +} + +pub fn pad_to( + data: &ArrayBase, + pad: &[[usize; 2]], + mode: PadMode, + output: &mut Array, +) -> super::PadResult where + A: Copy + FromPrimitive + Num, + D: Dimension, + S: Data, +{ + let pad = reader(data.ndim(), pad)?; + + // Select portion of padded array that needs to be copied from the original array. + output + .slice_each_axis_mut(|ad| { + let AxisDescription { axis, len, .. } = ad; + let pad = pad[axis.index()]; + Slice::from(pad[0]..len - pad[1]) + }) + .assign(data); + + match mode.action() { + PadAction::StopAfterCopy => { + // Do nothing + return Ok(()) + } + _ => unimplemented!(), + } +} \ No newline at end of file diff --git a/core/src/traits/mod.rs b/core/src/traits/mod.rs index 8a59195e..ffcfd6ab 100644 --- a/core/src/traits/mod.rs +++ b/core/src/traits/mod.rs @@ -9,6 +9,7 @@ pub mod ops; pub mod params; pub mod predict; pub mod setup; +pub mod shape; pub mod train; pub mod arr { @@ -53,5 +54,6 @@ pub(crate) mod prelude { pub use super::params::*; pub use super::predict::*; pub use super::setup::*; + pub use super::shape::*; pub use super::train::*; } diff --git a/core/src/traits/predict.rs b/core/src/traits/predict.rs index 632d05b1..628e150b 100644 --- a/core/src/traits/predict.rs +++ b/core/src/traits/predict.rs @@ -3,6 +3,8 @@ Contrib: FL03 */ use crate::error::PredictError; +#[cfg(any(feature = "alloc", feature = "std"))] +use crate::rust::Box; /// [Forward] describes an object capable of forward propagation. pub trait Forward { @@ -29,18 +31,11 @@ pub trait Predict { /* ********* Implementations ********* */ -impl Forward for S -where - S: Predict, -{ - type Output = Y; +impl Forward for M where M: Predict { + type Output = M::Output; - fn forward(&self, args: &X) -> Self::Output { - if let Ok(y) = self.predict(args) { - y - } else { - panic!("Error in forward propagation") - } + fn forward(&self, args: &U) -> Self::Output { + self.predict(args).unwrap() } } @@ -57,6 +52,14 @@ where .fold(args.clone(), |acc, m| m.forward(&acc)) } } +#[cfg(any(feature = "alloc", feature = "std"))] +impl Predict for Box> { + type Output = V; + + fn predict(&self, args: &U) -> Result { + self.as_ref().predict(args) + } +} impl Predict for Option where diff --git a/data/src/traits/shape.rs b/core/src/traits/shape.rs similarity index 100% rename from data/src/traits/shape.rs rename to core/src/traits/shape.rs diff --git a/data/src/traits/build.rs b/data/src/traits/build.rs index 7944014b..0a2a1abb 100644 --- a/data/src/traits/build.rs +++ b/data/src/traits/build.rs @@ -2,7 +2,7 @@ Appellation: ndarray Contrib: FL03 */ -use crate::traits::Dimensional; +use concision::Dimensional; use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; use num::{One, Zero}; diff --git a/data/src/traits/data/container.rs b/data/src/traits/data/container.rs index 0d9e0044..5e2c883e 100644 --- a/data/src/traits/data/container.rs +++ b/data/src/traits/data/container.rs @@ -2,7 +2,8 @@ Appellation: container Contrib: FL03 */ -use crate::traits::{ContainerRepr, Dimensional}; +use crate::traits::ContainerRepr; +use concision::Dimensional; pub trait Container { type Data: ContainerRepr; diff --git a/data/src/traits/mod.rs b/data/src/traits/mod.rs index 14b24d38..1b562ac5 100644 --- a/data/src/traits/mod.rs +++ b/data/src/traits/mod.rs @@ -2,12 +2,10 @@ Appellation: traits Contrib: FL03 */ -pub use self::{data::*, ext::*, records::*, shape::*}; +pub use self::{data::*, ext::*, records::*}; pub mod build; - pub mod records; -pub mod shape; #[doc(hidden)] pub mod data { @@ -40,5 +38,4 @@ pub(crate) mod prelude { pub use super::data::prelude::*; pub use super::ext::prelude::*; pub use super::records::*; - pub use super::shape::*; } diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index 2bf10b43..655548f3 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -15,3 +15,10 @@ pub(crate) mod perceptron; pub(crate) mod prelude { pub use super::perceptron::Perceptron; } + +use concision::Forward; + +pub trait Neuron: Forward { + type Elem; + type Rho; +} diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index afcf4cbf..43c88664 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -5,6 +5,7 @@ use concision::prelude::{Activate, Module, Predict, PredictError}; use nd::{ArrayBase, Data, Dimension}; +// #91 /// Perceptrons are the fundamental building block of multi-layer perceptrons (MLPs). /// They are used to model a particular layer within a neural network. Generally speaking, /// Perceptrons consist of a linear set of parameters and an activation function. @@ -46,16 +47,3 @@ where Ok(self.rho.activate(res)) } } - -// impl Predict for Perceptron -// where -// F: Activate, -// M: Module + Predict, -// { -// type Output = F::Output; - -// fn predict(&self, args: &X) -> Result { -// let res = self.module.predict(args)?; -// Ok(self.rho.activate(res)) -// } -// } diff --git a/models/linear/src/norm/layer/config.rs b/models/linear/src/norm/layer/config.rs index ef9e7469..06cb0f9f 100644 --- a/models/linear/src/norm/layer/config.rs +++ b/models/linear/src/norm/layer/config.rs @@ -27,7 +27,7 @@ where &mut self.axis } - pub fn eps(&self) -> f64 { + pub const fn eps(&self) -> f64 { self.eps } diff --git a/models/linear/src/norm/layer/model.rs b/models/linear/src/norm/layer/model.rs index 1cca2419..c0ea6a0a 100644 --- a/models/linear/src/norm/layer/model.rs +++ b/models/linear/src/norm/layer/model.rs @@ -91,26 +91,12 @@ where pub fn params_mut(&mut self) -> &mut LinearParams { &mut self.params } - - pub fn dim(&self) -> D::Pattern { - self.config().dim() - } - - pub fn eps(&self) -> f64 { + /// Returns the epsilon value used for numerical stability. + pub const fn eps(&self) -> f64 { self.config().eps() } - pub fn ndim(&self) -> usize { - self.config().ndim() - } - - pub fn raw_dim(&self) -> D { - self.config().raw_dim() - } - - pub fn shape(&self) -> &[usize] { - self.config().shape() - } + concision::dimensional!(config()); } impl Default for LayerNorm diff --git a/models/linear/src/params/store.rs b/models/linear/src/params/store.rs index b8afc752..f62643c2 100644 --- a/models/linear/src/params/store.rs +++ b/models/linear/src/params/store.rs @@ -116,7 +116,7 @@ where pbuilder!(zeros where A: Clone + Zero, S: DataOwned); - dimensional!(weights()); + dimensional!(weight); wnbview!(into_owned::(self) where A: Clone, S: Data); diff --git a/models/linear/tests/params.rs b/models/linear/tests/params.rs index fed65fb7..56cc4628 100644 --- a/models/linear/tests/params.rs +++ b/models/linear/tests/params.rs @@ -3,10 +3,10 @@ Contrib: FL03 */ #![allow(unused_imports)] -extern crate concision_core as concision; +extern crate concision_core as cnc; extern crate concision_linear as linear; -use concision::Predict; +use cnc::Predict; use core::str::FromStr; use linear::params::{LinearParams, Param, Unbiased}; use linear::Features; diff --git a/models/transformers/tests/ffn.rs b/models/transformers/tests/ffn.rs index ad37ab83..80e3d94b 100644 --- a/models/transformers/tests/ffn.rs +++ b/models/transformers/tests/ffn.rs @@ -2,11 +2,11 @@ Appellation: ffn Contrib: FL03 */ -extern crate concision_core as concision; +extern crate concision_core as cnc; extern crate concision_linear as linear; extern crate concision_transformer as transformer; -use concision::prelude::{linarr, Predict}; +use cnc::prelude::{linarr, Predict}; use linear::Biased; use transformer::model::ffn::FeedForwardNetwork; @@ -14,8 +14,9 @@ use ndarray::prelude::*; #[test] fn test_ffn() { + let p = 0.45; let (samples, d_model, d_ff) = (100, 30, 3); - let model = FeedForwardNetwork::::std(d_model, d_ff, Some(0.1)); + let model = FeedForwardNetwork::::std(d_model, d_ff, Some(p)); let data = linarr::((samples, d_model)).unwrap(); From 0df25c1836be0918bc11153120e0fab0f2b6ea40 Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Mon, 27 May 2024 12:41:08 -0500 Subject: [PATCH 10/12] update Signed-off-by: Joe McCain III --- core/src/error/kinds.rs | 52 +++++++++++------- core/src/error/kinds/external.rs | 15 +++++- core/src/error/kinds/predict.rs | 38 -------------- core/src/error/kinds/shape.rs | 35 ------------- core/src/error/mod.rs | 1 - core/src/macros.rs | 6 +-- core/src/macros/errors.rs | 58 ++++++++++++++++++++- core/src/nn/error.rs | 21 ++++++-- core/src/nn/mod.rs | 1 + core/src/nn/model.rs | 16 +++--- core/src/nn/model/module.rs | 7 ++- core/src/ops/pad/error.rs | 31 +++++++++-- core/src/ops/pad/utils.rs | 19 ++++--- core/src/primitives.rs | 2 + core/src/traits/predict.rs | 5 +- models/linear/src/impls/model/impl_model.rs | 15 +++--- models/linear/src/mlp/mod.rs | 16 ++++-- models/linear/src/mlp/model.rs | 36 ++++++++++--- models/linear/src/mlp/perceptron.rs | 17 +++--- 19 files changed, 238 insertions(+), 153 deletions(-) delete mode 100644 core/src/error/kinds/predict.rs delete mode 100644 core/src/error/kinds/shape.rs diff --git a/core/src/error/kinds.rs b/core/src/error/kinds.rs index 7033f755..25f5d0b9 100644 --- a/core/src/error/kinds.rs +++ b/core/src/error/kinds.rs @@ -2,14 +2,30 @@ Appellation: kinds Contrib: FL03 */ -pub use self::{external::*, predict::*}; +pub use self::external::*; mod external; -mod predict; use crate::nn::ModelError; use strum::{AsRefStr, Display, EnumCount, EnumIs, VariantNames}; +err! { + PredictError { + ArithmeticError, + ShapeMismatch, + TypeError, + } +} + +err! { + ShapeError { + IncompatibleLayout, + IncompatibleRank, + ShapeMismatch, + SizeMismatch, + } +} + #[derive( AsRefStr, Clone, @@ -33,28 +49,26 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, VariantNames}; pub enum Errors { IO, External(ExternalError), - Predict(PredictError), Model(ModelError), Shape(String), } -macro_rules! from_err { - ($S:ty: $($($p:ident)::*($err:ty)),* $(,)*) => { - $( - from_err!(@impl $S: $($p)::*($err)); - )* - }; - (@impl $S:ty: $($p:ident)::*($err:ty)) => { - impl From<$err> for $S { - fn from(err: $err) -> Self { - $($p)::*(err) - } - } - }; -} - +/* + ************* Implementations ************* +*/ from_err!(Errors: Errors::External(ExternalError), Errors::Model(ModelError), - Errors::Predict(PredictError), ); + +impl From<&str> for Errors { + fn from(err: &str) -> Self { + Errors::External(err.into()) + } +} + +impl From for Errors { + fn from(err: PredictError) -> Self { + Errors::Model(ModelError::Predict(err)) + } +} diff --git a/core/src/error/kinds/external.rs b/core/src/error/kinds/external.rs index e9de1787..8a137273 100644 --- a/core/src/error/kinds/external.rs +++ b/core/src/error/kinds/external.rs @@ -3,6 +3,8 @@ Contrib: FL03 */ use crate::error::ErrorKind; +#[cfg(any(feature = "alloc", feature = "std"))] +use crate::rust::String; use smart_default::SmartDefault; use strum::{AsRefStr, EnumCount, EnumIs, VariantNames}; @@ -72,4 +74,15 @@ impl From> for ExternalError { } } -from_variant!(ExternalError::Error {<&str>.to_string(), .to_string()}); +from_variant! { + ExternalError::Error { + <&str>.to_string() + } +} + +#[cfg(any(feature = "alloc", feature = "std"))] +from_variant! { + ExternalError::Error { + .to_string(), + } +} diff --git a/core/src/error/kinds/predict.rs b/core/src/error/kinds/predict.rs deleted file mode 100644 index de39d1d8..00000000 --- a/core/src/error/kinds/predict.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - Appellation: error - Contrib: FL03 -*/ -use scsys::VariantConstructors; -use smart_default::SmartDefault; -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - SmartDefault, - VariantConstructors, - VariantNames, -)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "snake_case", untagged) -)] -#[strum(serialize_all = "snake_case")] -pub enum PredictError { - #[default] - ArithmeticError, - ShapeMismatch, - TypeError, -} diff --git a/core/src/error/kinds/shape.rs b/core/src/error/kinds/shape.rs deleted file mode 100644 index d7c2c831..00000000 --- a/core/src/error/kinds/shape.rs +++ /dev/null @@ -1,35 +0,0 @@ -/* - Appellation: shape - Contrib: FL03 -*/ -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; - -#[derive( - AsRefStr, - Clone, - Debug, - Display, - EnumCount, - EnumIs, - EnumIter, - EnumString, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - VariantNames, -)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "snake_case", untagged) -)] -#[strum(serialize_all = "snake_case")] -pub enum ShapeError { - IncompatibleLayout, - IncompatibleRank, - ShapeMismatch, - SizeMismatch, -} - diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index a523fb78..1ab69546 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -10,7 +10,6 @@ pub mod kinds; pub trait ErrorKind: Clone + ToString {} - impl_err!(kinds::Errors, kinds::PredictError, crate::nn::ModelError); pub(crate) mod prelude { diff --git a/core/src/macros.rs b/core/src/macros.rs index 4195f643..49d02509 100644 --- a/core/src/macros.rs +++ b/core/src/macros.rs @@ -34,7 +34,7 @@ macro_rules! dimensional { pub fn ndim(&self) -> usize { self.$name.ndim() } - /// Forwards the + /// Forwards the pub fn raw_dim(&self) -> D { self.$name.raw_dim() } @@ -52,7 +52,7 @@ macro_rules! dimensional { pub fn ndim(&self) -> usize { self.$name().ndim() } - /// Forwards the + /// Forwards the pub fn raw_dim(&self) -> D { self.$name().raw_dim() } @@ -87,4 +87,4 @@ macro_rules! fwd_ndim { self.$name$(())?.dim().clone() } } -} \ No newline at end of file +} diff --git a/core/src/macros/errors.rs b/core/src/macros/errors.rs index a3abf8c3..fe27ae5a 100644 --- a/core/src/macros/errors.rs +++ b/core/src/macros/errors.rs @@ -3,6 +3,27 @@ Contrib: FL03 */ +macro_rules! from_err { + ($S:ty: $($($p:ident)::*($err:ty)),* $(,)*) => { + $( + from_err!(@impl impl $S: $err => $($p)::*); + )* + }; + (@impl impl $S:ty: $err:ty => $($p:ident)::*) => { + impl From<$err> for $S { + fn from(err: $err) -> Self { + $($p)::*(err) + } + } + }; + (@impl impl $S:ty: $err:ty => $($p:ident)::*) => { + impl From<$err> for $S { + fn from(err: $err) -> Self { + $($p)::*(err) + } + } + }; +} macro_rules! impl_err { ($($ty:ty),* $(,)*) => { @@ -24,4 +45,39 @@ macro_rules! err_from { (@impl $ty:ty) => { }; -} \ No newline at end of file +} + +macro_rules! err { + ($name:ident $($rest:tt)*) => { + err!(@base $name $($rest)*); + }; + (@base $name:ident {$($rest:tt)*}) => { + #[derive( + Clone, + Copy, + Debug, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + scsys::VariantConstructors, + strum::AsRefStr, + strum::Display, + strum::EnumCount, + strum::EnumIs, + strum::EnumIter, + strum::EnumString, + strum::VariantNames, + )] + #[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "snake_case", untagged) + )] + #[strum(serialize_all = "snake_case")] + pub enum $name { + $($rest)* + } + }; +} diff --git a/core/src/nn/error.rs b/core/src/nn/error.rs index e1ece894..e613b948 100644 --- a/core/src/nn/error.rs +++ b/core/src/nn/error.rs @@ -3,17 +3,16 @@ Contrib: FL03 */ use crate::error::PredictError; -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; +use strum::{AsRefStr, Display, EnumCount, EnumIs, VariantNames}; #[derive( AsRefStr, Clone, + Copy, Debug, Display, EnumCount, EnumIs, - EnumIter, - EnumString, Eq, Hash, Ord, @@ -32,7 +31,19 @@ pub enum ModelError { } impl ModelError { - // nested_constructor!(ModelError, PredictError { + pub fn from_predict(err: PredictError) -> Self { + ModelError::Predict(err) + } - // }) + pub fn predict(&self) -> Option { + match *self { + ModelError::Predict(err) => Some(err), + } + } +} + +impl From for ModelError { + fn from(err: PredictError) -> Self { + ModelError::from_predict(err) + } } diff --git a/core/src/nn/mod.rs b/core/src/nn/mod.rs index 45b43448..78484f7f 100644 --- a/core/src/nn/mod.rs +++ b/core/src/nn/mod.rs @@ -10,6 +10,7 @@ pub mod dropout; pub mod error; pub mod mask; pub mod model; +#[doc(hidden)] pub mod optim; pub(crate) mod prelude { diff --git a/core/src/nn/model.rs b/core/src/nn/model.rs index 316d03e7..18acf765 100644 --- a/core/src/nn/model.rs +++ b/core/src/nn/model.rs @@ -4,8 +4,9 @@ */ pub use self::module::*; +pub(crate) mod module; + pub mod config; -pub mod module; #[doc(hidden)] pub mod repo; @@ -17,16 +18,11 @@ pub(crate) mod prelude { use crate::traits::Forward; -pub trait Model: Module -where - Self: Forward, -{ - type Ctx; - type Data; - - fn children(&self) -> Vec>; +pub trait Model: Forward { + type Args; + type Elem; - fn context(&self) -> Self::Ctx; + type Params; } /// This trait describes any neural networks or models that diff --git a/core/src/nn/model/module.rs b/core/src/nn/model/module.rs index a1ba56e0..99a3c0a0 100644 --- a/core/src/nn/model/module.rs +++ b/core/src/nn/model/module.rs @@ -2,17 +2,16 @@ Appellation: modules Contrib: FL03 */ -use crate::{Config, Parameters, Forward}; +use crate::{Config, Forward, Parameters}; -pub type ModuleDyn = Box>; -pub type DynModuleExt = Box>; -pub type Stack = Vec>>; +pub type ModuleDyn = Box>; /// A `Module` defines any object that may be used as a layer in a neural network. /// [Config](Module::Config) contains all of the hyperparameters for the model. /// [Params](Module::Params) refers to an object used to store the various learnable parameters. pub trait Module { type Config: Config; + type Elem; type Params: Parameters; fn config(&self) -> &Self::Config; diff --git a/core/src/ops/pad/error.rs b/core/src/ops/pad/error.rs index 3a8ab0d9..d5529c53 100644 --- a/core/src/ops/pad/error.rs +++ b/core/src/ops/pad/error.rs @@ -2,12 +2,37 @@ Appellation: error Contrib: FL03 */ -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumMessage, EnumString, EnumIter, VariantArray, VariantNames}; +use strum::{ + AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumMessage, EnumString, VariantArray, + VariantNames, +}; pub type PadResult = Result; -#[derive(AsRefStr, Clone, Copy, Debug, Default, Display, EnumCount, EnumIs, EnumIter, EnumMessage, EnumString, Eq, Ord, PartialEq, PartialOrd, VariantArray, VariantNames)] -#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(rename_all = "snake_case"))] +#[derive( + AsRefStr, + Clone, + Copy, + Debug, + Default, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumMessage, + EnumString, + Eq, + Ord, + PartialEq, + PartialOrd, + VariantArray, + VariantNames, +)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "snake_case") +)] #[cfg_attr(feature = "std", derive(std::hash::Hash))] #[strum(serialize_all = "snake_case")] #[repr(u8)] diff --git a/core/src/ops/pad/utils.rs b/core/src/ops/pad/utils.rs index 5b5113ab..1f4a5648 100644 --- a/core/src/ops/pad/utils.rs +++ b/core/src/ops/pad/utils.rs @@ -7,20 +7,22 @@ use crate::traits::ArrayLike; use nd::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; use num::{FromPrimitive, Num}; - fn reader(nb_dim: usize, pad: &[[usize; 2]]) -> Result, PadError> { if pad.len() == 1 && pad.len() < nb_dim { // The user provided a single padding for all dimensions Ok(vec![pad[0]; nb_dim]) } else if pad.len() == nb_dim { Ok(pad.to_vec()) - } - else { + } else { Err(PadError::InconsistentDimensions) } } -pub fn pad(data: &ArrayBase, pad: &[[usize; 2]], mode: PadMode) -> Result, PadError> +pub fn pad( + data: &ArrayBase, + pad: &[[usize; 2]], + mode: PadMode, +) -> Result, PadError> where A: Copy + FromPrimitive + Num, D: Dimension, @@ -42,7 +44,8 @@ pub fn pad_to( pad: &[[usize; 2]], mode: PadMode, output: &mut Array, -) -> super::PadResult where +) -> super::PadResult +where A: Copy + FromPrimitive + Num, D: Dimension, S: Data, @@ -59,10 +62,10 @@ pub fn pad_to( .assign(data); match mode.action() { - PadAction::StopAfterCopy => { + PadAction::StopAfterCopy => { // Do nothing - return Ok(()) + return Ok(()); } _ => unimplemented!(), } -} \ No newline at end of file +} diff --git a/core/src/primitives.rs b/core/src/primitives.rs index c2755083..a97e0994 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -23,6 +23,7 @@ pub(crate) mod rust { pub use alloc::borrow::Cow; pub use alloc::boxed::{self, Box}; pub use alloc::collections::{self, BTreeMap, BTreeSet, BinaryHeap, VecDeque}; + pub use alloc::string::{self, String, ToString}; pub use alloc::vec::{self, Vec}; } #[cfg(feature = "std")] @@ -30,6 +31,7 @@ pub(crate) mod rust { pub use std::borrow::Cow; pub use std::boxed::{self, Box}; pub use std::collections::{self, BTreeMap, BTreeSet, BinaryHeap, VecDeque}; + pub use std::string::{self, String, ToString}; pub use std::sync::Arc; pub use std::vec::{self, Vec}; } diff --git a/core/src/traits/predict.rs b/core/src/traits/predict.rs index 628e150b..4addf87e 100644 --- a/core/src/traits/predict.rs +++ b/core/src/traits/predict.rs @@ -31,7 +31,10 @@ pub trait Predict { /* ********* Implementations ********* */ -impl Forward for M where M: Predict { +impl Forward for M +where + M: Predict, +{ type Output = M::Output; fn forward(&self, args: &U) -> Self::Output { diff --git a/models/linear/src/impls/model/impl_model.rs b/models/linear/src/impls/model/impl_model.rs index 475c66bd..947ad513 100644 --- a/models/linear/src/impls/model/impl_model.rs +++ b/models/linear/src/impls/model/impl_model.rs @@ -2,16 +2,18 @@ Appellation: impl_model Contrib: FL03 */ -use crate::{Config, Linear, LinearParams}; +use crate::{Config, Linear, ParamsBase}; use concision::prelude::{Module, Predict, PredictError}; -use nd::RemoveAxis; +use nd::{RawData, RemoveAxis}; -impl Module for Linear +impl Module for Linear where D: RemoveAxis, + S: RawData, { type Config = Config; - type Params = LinearParams; + type Elem = A; + type Params = ParamsBase; fn config(&self) -> &Self::Config { &self.config @@ -26,10 +28,11 @@ where } } -impl Predict for Linear +impl Predict for Linear where D: RemoveAxis, - LinearParams: Predict, + S: RawData, + ParamsBase: Predict, { type Output = V; diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index 655548f3..ad07279b 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -8,17 +8,25 @@ //! //! #[doc(inline)] -pub use self::perceptron::*; +pub use self::{model::*, perceptron::*}; +pub(crate) mod model; pub(crate) mod perceptron; pub(crate) mod prelude { pub use super::perceptron::Perceptron; } -use concision::Forward; +use concision::prelude::{Activate, Module, Predict}; -pub trait Neuron: Forward { +pub trait Neuron: + Predict>::Output>>::Output> +{ + type Module: Module + Predict; + type Rho: Activate<>::Output>; +} + +pub trait Layer: Predict { type Elem; - type Rho; + type Module: Module; } diff --git a/models/linear/src/mlp/model.rs b/models/linear/src/mlp/model.rs index 8bcc66f6..9ae91e31 100644 --- a/models/linear/src/mlp/model.rs +++ b/models/linear/src/mlp/model.rs @@ -2,10 +2,34 @@ Appellation: model Contrib: FL03 */ -use super::Perceptron; +use super::{Neuron, Perceptron}; -pub struct Mlp { - input: Perceptron, - hidden: H, - output: O, -} \ No newline at end of file +pub struct Mlp +where + I: Neuron, + H: Neuron, + O: Neuron, +{ + input: Perceptron, + hidden: Vec>, + output: Perceptron, +} + +impl Mlp +where + I: Neuron, + H: Neuron, + O: Neuron, +{ + pub const fn input(&self) -> &Perceptron { + &self.input + } + + pub fn hidden(&self) -> &[Perceptron] { + &self.hidden + } + + pub const fn output(&self) -> &Perceptron { + &self.output + } +} diff --git a/models/linear/src/mlp/perceptron.rs b/models/linear/src/mlp/perceptron.rs index 43c88664..15e645e5 100644 --- a/models/linear/src/mlp/perceptron.rs +++ b/models/linear/src/mlp/perceptron.rs @@ -9,10 +9,7 @@ use nd::{ArrayBase, Data, Dimension}; /// Perceptrons are the fundamental building block of multi-layer perceptrons (MLPs). /// They are used to model a particular layer within a neural network. Generally speaking, /// Perceptrons consist of a linear set of parameters and an activation function. -pub struct Perceptron -where - M: Module, -{ +pub struct Perceptron { module: M, rho: F, } @@ -24,11 +21,15 @@ where pub fn new(module: M, rho: F) -> Self { Self { module, rho } } +} + +impl Activate for Perceptron +where + F: Activate, +{ + type Output = F::Output; - pub fn activate(&self, args: T) -> F::Output - where - F: Activate, - { + fn activate(&self, args: T) -> Self::Output { self.rho.activate(args) } } From 82dbf53c354eec56ffe5c4ed8b64392b3afbf48f Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Tue, 28 May 2024 13:59:15 -0500 Subject: [PATCH 11/12] update Signed-off-by: Joe McCain III --- core/src/func/activate/mod.rs | 98 ++++++++++++++++++- core/src/macros/toggle.rs | 20 +++- core/src/ops/fft/cmp/mode.rs | 7 +- core/src/ops/mod.rs | 2 +- core/src/traits/misc/toggle.rs | 41 +++----- models/kan/src/actor.rs | 2 +- models/linear/src/conv/mod.rs | 1 + models/linear/src/params/item.rs | 6 +- models/linear/src/params/mode.rs | 25 +---- models/linear/src/params/store.rs | 9 +- .../src/attention/multi/multi_head.rs | 1 + 11 files changed, 140 insertions(+), 72 deletions(-) diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs index f20a7e32..1d20878c 100644 --- a/core/src/func/activate/mod.rs +++ b/core/src/func/activate/mod.rs @@ -20,6 +20,11 @@ pub(crate) mod prelude { pub use super::{Activate, NdActivate}; } +use nd::prelude::*; +use nd::{Data, DataMut, RemoveAxis, ScalarOperand}; +use num::complex::ComplexFloat; +use num::traits::{One, Zero}; + /// [Activate] designates a function or structure that can be used /// as an activation function for a neural network. /// @@ -31,17 +36,75 @@ pub trait Activate { fn activate(&self, args: T) -> Self::Output; } -pub trait NdActivate { - type Data; +pub trait NdActivate +where + A: ScalarOperand, + D: Dimension, +{ + type Data: Data; - fn activate(&self, f: F) -> nd::Array + fn activate(&self, f: F) -> Array where F: FnMut(A) -> B; - fn activate_mut<'a, B, F>(&'a mut self, f: F) + fn activate_inplace<'a, F>(&'a mut self, f: F) where A: 'a, - F: FnMut(&'a A) -> B; + F: FnMut(A) -> A, + Self::Data: DataMut; + + fn linear(&self) -> Array + where + A: Clone, + { + self.activate(|x| x.clone()) + } + + fn heavyside(&self) -> Array + where + A: One + PartialOrd + Zero, + { + self.activate(heavyside) + } + + fn relu(&self) -> Array + where + A: PartialOrd + Zero, + { + self.activate(relu) + } + + fn sigmoid(&self) -> Array + where + A: ComplexFloat, + { + self.activate(sigmoid) + } + + fn softmax(&self) -> Array + where + A: ComplexFloat, + { + let exp = self.activate(ComplexFloat::exp); + &exp / exp.sum() + } + + fn softmax_axis(&self, axis: usize) -> Array + where + A: ComplexFloat, + D: RemoveAxis, + { + let exp = self.activate(ComplexFloat::exp); + let axis = Axis(axis); + &exp / &exp.sum_axis(axis) + } + + fn tanh(&self) -> Array + where + A: ComplexFloat, + { + self.activate(tanh) + } } /* ************* Implementations ************* @@ -67,3 +130,28 @@ impl Activate for Box> { self.as_ref().activate(args) } } + +impl NdActivate for ArrayBase +where + A: ScalarOperand, + D: Dimension, + S: Data, +{ + type Data = S; + + fn activate(&self, f: F) -> Array + where + F: FnMut(A) -> B, + { + self.mapv(f) + } + + fn activate_inplace<'a, F>(&'a mut self, f: F) + where + A: 'a, + S: DataMut, + F: FnMut(A) -> A, + { + self.mapv_inplace(f) + } +} diff --git a/core/src/macros/toggle.rs b/core/src/macros/toggle.rs index 908f6abf..2228552f 100644 --- a/core/src/macros/toggle.rs +++ b/core/src/macros/toggle.rs @@ -5,15 +5,25 @@ #[macro_export] macro_rules! toggle { - (enum $($name:ident),* $(,)?) => { - $(toggle!(@enum $name);)* + ($vis:vis enum {$($name:ident),* $(,)?}) => { + $(toggle!(@impl $vis enum $name);)* }; - (@enum $name:ident) => { + (@impl $vis:vis enum $name:ident) => { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] - pub enum $name {} + $vis enum $name {} - impl $crate::traits::misc::toggle::Toggle for $name {} + impl $name { + $vis fn of() -> bool { + use ::core::any::TypeId; + TypeId::of::() == TypeId::of::() + } + $vis fn phantom() -> core::marker::PhantomData { + core::marker::PhantomData + } + } + + impl $crate::traits::TypeTag for $name {} }; } diff --git a/core/src/ops/fft/cmp/mode.rs b/core/src/ops/fft/cmp/mode.rs index 7d538eae..69845f6e 100644 --- a/core/src/ops/fft/cmp/mode.rs +++ b/core/src/ops/fft/cmp/mode.rs @@ -7,7 +7,12 @@ use strum::{ AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames, }; -toggle!(enum C, R); +toggle! { + pub enum { + C, + R + } +} /// #[derive( diff --git a/core/src/ops/mod.rs b/core/src/ops/mod.rs index 96313e51..40dd5c5e 100644 --- a/core/src/ops/mod.rs +++ b/core/src/ops/mod.rs @@ -2,7 +2,7 @@ Appellation: ops Contrib: FL03 */ -//! # Operations +//! # Overloadable Operations pub use self::pad::*; pub(crate) mod pad; diff --git a/core/src/traits/misc/toggle.rs b/core/src/traits/misc/toggle.rs index 4865d33f..023d923e 100644 --- a/core/src/traits/misc/toggle.rs +++ b/core/src/traits/misc/toggle.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ -pub trait Toggle: 'static {} +pub trait TypeTag: 'static {} -pub trait OfType { +pub trait TypeOf: 'static { fn of() -> bool where T: 'static, @@ -13,36 +13,17 @@ pub trait OfType { { core::any::TypeId::of::() == core::any::TypeId::of::() } + + fn is() -> bool + where + T: 'static, + Self: 'static, + { + core::any::TypeId::of::() == core::any::TypeId::of::() + } } /* ************* Implementations ************* */ -impl OfType for T {} - -macro_rules! impl_toggle { - ($($scope:ident$(<$T:ident>)?),* $(,)?) => { - $(impl_toggle!(@impl $scope$(<$T>)?);)* - }; - (@impl $scope:ident$(<$T:ident>)?) => { - impl$(<$T>)? Toggle for $scope$(<$T> where $T: 'static)? {} - }; -} - -impl_toggle!( - bool, - char, - i8, - i16, - i32, - i64, - i128, - isize, - u8, - u16, - u32, - u64, - u128, - usize, - Option -); +impl TypeOf for T {} diff --git a/models/kan/src/actor.rs b/models/kan/src/actor.rs index 6ffc6c62..d5525acc 100644 --- a/models/kan/src/actor.rs +++ b/models/kan/src/actor.rs @@ -81,7 +81,7 @@ where type Output = Z; fn predict(&self, x: &Array1) -> Result { - let y = x.mapv(|xi| self.spline.sample(xi).unwrap()); + let y = x.mapv(|xi| self.spline().sample(xi).unwrap()); Ok(self.weight() * self.bias().eval(y)) } } diff --git a/models/linear/src/conv/mod.rs b/models/linear/src/conv/mod.rs index 73681242..915b2be5 100644 --- a/models/linear/src/conv/mod.rs +++ b/models/linear/src/conv/mod.rs @@ -2,6 +2,7 @@ Appellation: conv Contrib: FL03 */ +//! # Convolutional Neural Network pub use self::module::*; pub(crate) mod module; diff --git a/models/linear/src/params/item.rs b/models/linear/src/params/item.rs index 9ddbf28b..e839ac07 100644 --- a/models/linear/src/params/item.rs +++ b/models/linear/src/params/item.rs @@ -24,13 +24,13 @@ use strum::{AsRefStr, EnumDiscriminants, EnumIs, VariantNames}; Hash, Ord, PartialOrd, - VariantNames, strum::Display, strum::EnumCount, - EnumIs, + strum::EnumIs, strum::EnumIter, strum::EnumString, - strum::VariantArray + strum::VariantArray, + strum::VariantNames, ), strum(serialize_all = "lowercase") )] diff --git a/models/linear/src/params/mode.rs b/models/linear/src/params/mode.rs index ff153868..33f98f68 100644 --- a/models/linear/src/params/mode.rs +++ b/models/linear/src/params/mode.rs @@ -2,10 +2,9 @@ Appellation: mode Contrib: FL03 */ -use concision::Toggle; -use core::option::Option; +use concision::{toggle, TypeTag}; -pub trait ParamMode: Toggle { +pub trait ParamMode: TypeTag { const BIASED: bool = false; fn is_biased(&self) -> bool { @@ -18,30 +17,12 @@ pub trait ParamMode: Toggle { /* ************* Implementations ************* */ - -impl ParamMode for Option -where - T: 'static, -{ - const BIASED: bool = false; - - fn is_biased(&self) -> bool { - self.is_some() - } - - seal!(); -} - macro_rules! mode { {$($T:ident: $opt:expr),* $(,)?} => { $(mode!(@impl $T: $opt);)* }; (@impl $T:ident: $opt:expr) => { - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] - pub enum $T {} - - impl Toggle for $T {} + concision::toggle!(pub enum {$T}); impl ParamMode for $T { const BIASED: bool = $opt; diff --git a/models/linear/src/params/store.rs b/models/linear/src/params/store.rs index f62643c2..a74a1849 100644 --- a/models/linear/src/params/store.rs +++ b/models/linear/src/params/store.rs @@ -8,10 +8,11 @@ use core::marker::PhantomData; use nd::*; use num::{One, Zero}; -/// The [ParamsBase] struct is a generic store for linear parameters. The store mimics -/// the underlying [ArrayBase](ndarray::ArrayBase), enabling developers to specify -/// the data repr and dimension. Additionally, the store is parameterized to -/// accept a `K` type, used to designate the store as either [Biased](crate::Biased) or [Unbiased](crate::Unbiased). +/// [ParamsBase] is a flexible store for linear parameters; it is parameterized to accept +/// a `K` type, used to designate the store as either [Biased](crate::Biased) or [Unbiased](crate::Unbiased). +/// This was done in an effort to streamline the creation of new instances of the store, and to provide +/// a more ergonomic interface for the user. The store is also equipped with a number of builder methods +/// native to the [ArrayBase] from `ndarray`. pub struct ParamsBase, D = Ix2, K = Biased> where D: Dimension, diff --git a/models/transformers/src/attention/multi/multi_head.rs b/models/transformers/src/attention/multi/multi_head.rs index fc81cfc6..69e1aaec 100644 --- a/models/transformers/src/attention/multi/multi_head.rs +++ b/models/transformers/src/attention/multi/multi_head.rs @@ -7,6 +7,7 @@ use linear::{Biased, Linear}; use nd::prelude::*; use nd::{DataOwned, OwnedRepr, RawData}; +// #69 pub struct MultiHeadAttention> where D: Dimension, From ee1e6c74560c5408e5c2f158a56ce10930b489fe Mon Sep 17 00:00:00 2001 From: Joe McCain III Date: Sun, 2 Jun 2024 07:09:13 -0500 Subject: [PATCH 12/12] update Signed-off-by: Joe McCain III --- models/gnn/Cargo.toml | 1 + models/gnn/src/lib.rs | 12 +++++++++- models/gnn/src/model/base.rs | 6 +++++ models/gnn/src/model/mod.rs | 11 +++++++++ models/gnn/src/params/mod.rs | 12 ++++++++++ models/gnn/src/params/store.rs | 23 +++++++++++++++++++ models/kan/src/actor.rs | 2 +- models/kan/src/lib.rs | 6 ++++- models/kan/src/macros.rs | 4 ++++ models/kan/src/model/kan.rs | 2 +- models/kan/src/primitives.rs | 4 ++++ models/linear/src/mlp/mod.rs | 15 +++--------- models/linear/src/mlp/model.rs | 42 ++++++++++++++++++---------------- 13 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 models/gnn/src/model/base.rs create mode 100644 models/gnn/src/model/mod.rs create mode 100644 models/gnn/src/params/mod.rs create mode 100644 models/gnn/src/params/store.rs create mode 100644 models/kan/src/macros.rs diff --git a/models/gnn/Cargo.toml b/models/gnn/Cargo.toml index 6c16e62e..0ad1658a 100644 --- a/models/gnn/Cargo.toml +++ b/models/gnn/Cargo.toml @@ -89,6 +89,7 @@ lazy_static.workspace = true [dependencies] ndarray.workspace = true num.workspace = true +petgraph = { version = "0.6" } smart-default.workspace = true strum.workspace = true diff --git a/models/gnn/src/lib.rs b/models/gnn/src/lib.rs index 546bba6b..b1da8218 100644 --- a/models/gnn/src/lib.rs +++ b/models/gnn/src/lib.rs @@ -15,4 +15,14 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; -pub mod prelude {} +#[doc(inline)] +pub use self::model::*; + +pub(crate) mod model; + +pub mod params; + +pub mod prelude { + pub use crate::model::prelude::*; + pub use crate::params::prelude::*; +} diff --git a/models/gnn/src/model/base.rs b/models/gnn/src/model/base.rs new file mode 100644 index 00000000..bf0dd265 --- /dev/null +++ b/models/gnn/src/model/base.rs @@ -0,0 +1,6 @@ +/* + Appellation: base + Contrib: FL03 +*/ + +pub struct GNN; diff --git a/models/gnn/src/model/mod.rs b/models/gnn/src/model/mod.rs new file mode 100644 index 00000000..b723ceff --- /dev/null +++ b/models/gnn/src/model/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: model + Contrib: FL03 +*/ +pub use self::base::*; + +pub(crate) mod base; + +pub(crate) mod prelude { + pub use super::base::GNN; +} diff --git a/models/gnn/src/params/mod.rs b/models/gnn/src/params/mod.rs new file mode 100644 index 00000000..275f7a19 --- /dev/null +++ b/models/gnn/src/params/mod.rs @@ -0,0 +1,12 @@ +/* + Appellation: params + Contrib: FL03 +*/ + +pub use self::store::*; + +pub(crate) mod store; + +pub(crate) mod prelude { + pub use crate::store::GraphStore; +} diff --git a/models/gnn/src/params/store.rs b/models/gnn/src/params/store.rs new file mode 100644 index 00000000..105e6813 --- /dev/null +++ b/models/gnn/src/params/store.rs @@ -0,0 +1,23 @@ +/* + Appellation: store + Contrib: FL03 +*/ +use petgraph::graph::Graph; +use petgraph::{Directed, Direction}; + +pub struct GraphStore +where + Q: Direction, +{ + pub(crate) params: Graph, +} + +impl GraphStore { + pub fn new(params: Graph) -> Self { + Self { params } + } + + pub fn get(&self, key: K) -> Option<&V> { + self.params.node_weight(key) + } +} diff --git a/models/kan/src/actor.rs b/models/kan/src/actor.rs index d5525acc..49b3e660 100644 --- a/models/kan/src/actor.rs +++ b/models/kan/src/actor.rs @@ -22,7 +22,7 @@ pub type NdSpline = Spline, Array1>; /// /// ### Parameters /// -/// - b(**x**): bias function +/// - `b(**x**)`: bias function /// - spline(**x**): spline function; typically employs `Linear` interpolation /// - **ω**: weight factor /// diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs index 819aad35..7fd518e7 100644 --- a/models/kan/src/lib.rs +++ b/models/kan/src/lib.rs @@ -22,10 +22,14 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; +#[macro_use] +pub(crate) mod macros; +pub(crate) mod primitives; + #[doc(inline)] pub use self::actor::Actor; #[doc(inline)] -pub use self::model::Kan; +pub use self::model::KAN; pub mod actor; pub mod model; diff --git a/models/kan/src/macros.rs b/models/kan/src/macros.rs new file mode 100644 index 00000000..1f5eec90 --- /dev/null +++ b/models/kan/src/macros.rs @@ -0,0 +1,4 @@ +/* + Appellation: macros + Contrib: FL03 +*/ \ No newline at end of file diff --git a/models/kan/src/model/kan.rs b/models/kan/src/model/kan.rs index 9cce9759..cba0a0b7 100644 --- a/models/kan/src/model/kan.rs +++ b/models/kan/src/model/kan.rs @@ -3,4 +3,4 @@ Contrib: FL03 */ -pub struct Kan; +pub struct KAN; diff --git a/models/kan/src/primitives.rs b/models/kan/src/primitives.rs index e69de29b..238f8c7d 100644 --- a/models/kan/src/primitives.rs +++ b/models/kan/src/primitives.rs @@ -0,0 +1,4 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index ad07279b..4f8a2123 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -17,16 +17,7 @@ pub(crate) mod prelude { pub use super::perceptron::Perceptron; } -use concision::prelude::{Activate, Module, Predict}; - -pub trait Neuron: - Predict>::Output>>::Output> -{ - type Module: Module + Predict; - type Rho: Activate<>::Output>; -} - -pub trait Layer: Predict { - type Elem; - type Module: Module; +pub trait DeepNeuralNetwork { + type Input; + type Output; } diff --git a/models/linear/src/mlp/model.rs b/models/linear/src/mlp/model.rs index 9ae91e31..5ae3c862 100644 --- a/models/linear/src/mlp/model.rs +++ b/models/linear/src/mlp/model.rs @@ -2,34 +2,36 @@ Appellation: model Contrib: FL03 */ -use super::{Neuron, Perceptron}; +use concision::{Predict, PredictError}; +use core::marker::PhantomData; +// #92: Define the Multi-Layer Perceptron (MLP) model +/// A multi-layer perceptron (MLP) model. pub struct Mlp where - I: Neuron, - H: Neuron, - O: Neuron, + I: Predict, + H: Predict, + O: Predict, { - input: Perceptron, - hidden: Vec>, - output: Perceptron, + input: I, + hidden: Vec, + output: O, + _dtype: PhantomData, } -impl Mlp +impl Predict for Mlp where - I: Neuron, - H: Neuron, - O: Neuron, + I: Predict, + H: Predict, + O: Predict, { - pub const fn input(&self) -> &Perceptron { - &self.input - } - - pub fn hidden(&self) -> &[Perceptron] { - &self.hidden - } + type Output = O::Output; - pub const fn output(&self) -> &Perceptron { - &self.output + fn predict(&self, input: &A) -> Result { + let mut hidden = self.input.predict(input)?; + for layer in &self.hidden { + hidden = layer.predict(&hidden)?; + } + self.output.predict(&hidden) } }