v0.1.14 Dev (2)

.github/ISSUE_TEMPLATE/issue.md

@@ -0,0 +1,17 @@
+---
+about: A generic issue template
+assignees:
+    - FL03
+labels: []
+projects: ['@FL03/concision:features']
+name: Generic Issue
+title: ''
+---
+
+**Describe the proposal or feature that this issue is tracking.**
+
+## Issues
+
+- []
+
+## Pull Requests

.github/ISSUE_TEMPLATE/proposal.md

@@ -1,15 +1,14 @@
 ---
-name: Improvement Proposal
 about: A formal proposal discussing any new features, changes, or improvements to the project.
-title: 'CNC-0000:'
-labels: ['proposal']
-projects: ['@FL03/concision:features', '@FL03/concision:roadmap']
 assignees:
     - FL03
-
+labels: ['proposal']
+name: Improvement Proposal
+projects: ['@FL03/concision:features', '@FL03/concision:roadmap']
+title: 'CNC-0000:'
 ---
 
 
 ### Resources
 
-- [Google](https://google.com)
+- [company](https://github.com/scattered-systems)

.github/ISSUE_TEMPLATE/tracking.md

@@ -0,0 +1,17 @@
+---
+about: Create a new tracking issue to track the progress of a proposal or feature.
+assignees:
+    - FL03
+labels: ['tracking']
+projects: ['@FL03/concision:features']
+name: Tracking Issue
+title: 'Tracking Issue:'
+---
+
+**Describe the proposal or feature that this issue is tracking.**
+
+## Issues
+
+- []
+
+## Pull Requests

concision/Cargo.toml

@@ -45,7 +45,7 @@ models = [
     "gnn",
     "kan",
     "linear",
-    "transformers",
+    "transformer",
 ]
 
 gnn = [
@@ -60,7 +60,7 @@ linear = [
     "dep:concision-linear",
 ]
 
-transformers = [
+transformer = [
     "dep:concision-transformers",
 ]
 
@@ -184,6 +184,10 @@ test = true
 name = "linear"
 required-features = ["linear", "rand", "serde", "tracing"]
 
+[[example]]
+name = "transformer"
+required-features = ["transformer", "rand", "serde", "tracing"]
+
 [build-dependencies]
 
 [dependencies.concision-core]
@@ -229,6 +233,7 @@ version = "0.1.14"
 
 [dev-dependencies]
 anyhow = "1"
+approx.workspace = true
 lazy_static.workspace = true
 ndarray.workspace = true
 num = { features = ["rand", "serde"], version = "0.4" }

concision/examples/transformer.rs

@@ -0,0 +1,38 @@
+/*
+    Appellation: transformer <example>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+extern crate concision as cnc;
+
+use approx::AbsDiffEq;
+use cnc::prelude::Result;
+use cnc::transformer::AttentionHead;
+use ndarray::Array2;
+
+fn tracing() {
+    use tracing::Level;
+    use tracing_subscriber::fmt::time;
+
+    tracing_subscriber::fmt()
+        .compact()
+        .with_ansi(true)
+        .with_max_level(Level::DEBUG)
+        .with_target(false)
+        .with_timer(time::uptime())
+        .init();
+}
+
+fn main() -> Result<()> {
+    tracing();
+    tracing::info!("Starting up the transformer model example...");
+
+    let shape = (3, 3);
+    let head = AttentionHead::<f64>::ones(shape);
+    let score = head.attention();
+    assert!(score
+        .attention()
+        .abs_diff_eq(&Array2::from_elem(shape, 1f64 / 3f64), 1e-6));
+    println!("{:?}", score);
+
+    Ok(())
+}

concision/src/lib.rs

@@ -27,9 +27,9 @@ pub use concision_kan as kan;
 pub use concision_linear as linear;
 #[cfg(feature = "macros")]
 pub use concision_macros::*;
-#[cfg(feature = "transformers")]
+#[cfg(feature = "transformer")]
 #[doc(inline)]
-pub use concision_transformers as transformers;
+pub use concision_transformers as transformer;
 
 pub mod prelude {
     pub use concision_core::prelude::*;
@@ -45,4 +45,6 @@ pub mod prelude {
     pub use concision_linear::prelude::*;
     #[cfg(feature = "macros")]
     pub use concision_macros::*;
+    #[cfg(feature = "transformer")]
+    pub use concision_transformers::prelude::*;
 }

core/Cargo.toml

@@ -103,16 +103,24 @@ crate-type = ["lib"]
 doctest = false
 test = true
 
-[[test]]
-name = "random"
-required-features = ["rand"]
+
 
 [[test]]
 name = "fft"
 required-features = ["approx"]
 
+[[test]]
+name = "init"
+required-features = ["rand", "std"]
+
+[[test]]
+name = "nn"
+
 [build-dependencies]
 
+[dev-dependencies]
+lazy_static.workspace = true
+
 [dependencies]
 ndarray.workspace = true
 num.workspace = true
@@ -154,9 +162,6 @@ default-features = false
 features = ["v5", "v8"]
 version = "1"
 
-[dev-dependencies]
-lazy_static = "1"
-
 [package.metadata.docs.rs]
 all-features = true
 rustc-args = ["--cfg", "docsrs"]

core/src/func/activate/nl.rs

@@ -7,7 +7,7 @@ use ndarray::*;
 use num::complex::{Complex, ComplexFloat};
 use num::traits::Zero;
 
-pub fn relu<T>(args: T) -> T
+fn _relu<T>(args: T) -> T
 where
     T: PartialOrd + Zero,
 {
@@ -17,23 +17,33 @@ where
     T::zero()
 }
 
-pub fn sigmoid<T>(args: T) -> T
+fn _sigmoid<T>(args: T) -> T
 where
     T: ComplexFloat,
 {
     (T::one() + args.neg().exp()).recip()
 }
 
-pub fn softmax<A, S, D>(args: &ArrayBase<S, D>) -> Array<A, D>
+fn _softmax<A, S, D>(args: &ArrayBase<S, D>) -> Array<A, D>
 where
     A: ComplexFloat + ScalarOperand,
     D: Dimension,
     S: Data<Elem = A>,
 {
-    args.exp() / args.exp().sum()
+    let e = args.exp();
+    &e / e.sum()
 }
 
-pub fn tanh<T>(args: T) -> T
+// fn __softmax<T, I>(args: &I) -> I
+// where
+//     I: Clone + core::ops::Div<T, Output = I> + Exp<Output = I>, T: Exp<Output = T> + core::iter::Sum ,
+//     for<'a> I: IntoIterator<Item = &'a T>,
+// {
+//     let e = args.exp();
+//     e.clone() / e.into_iter().sum::<T>()
+// }
+
+fn _tanh<T>(args: T) -> T
 where
     T: ComplexFloat,
 {
@@ -64,22 +74,25 @@ macro_rules! nonlinear {
         nonlinear!(@arr $rho::$call);
     };
     (@impl $rho:ident::$call:ident<$T:ty>) => {
-        impl $rho for $T {
-            type Output = $T;
+        paste::paste! {
+            impl $rho for $T {
+                type Output = $T;
 
-            fn $call(self) -> Self::Output {
-                $call(self)
+                fn $call(self) -> Self::Output {
+                    [<_ $call>](self)
+                }
             }
-        }
 
-        impl<'a> $rho for &'a $T {
-            type Output = $T;
+            impl<'a> $rho for &'a $T {
+                type Output = $T;
 
-            fn $call(self) -> Self::Output {
-                $call(*self)
+                fn $call(self) -> Self::Output {
+                    [<_ $call>](*self)
+                }
             }
         }
 
+
     };
     (@arr $name:ident::$call:ident) => {
         impl<A, S, D> $name for ArrayBase<S, D>
@@ -150,7 +163,7 @@ where
     type Output = Array<A, D>;
 
     fn softmax(self) -> Self::Output {
-        softmax(&self)
+        _softmax(&self)
     }
 }
 
@@ -163,6 +176,6 @@ where
     type Output = Array<A, D>;
 
     fn softmax(self) -> Self::Output {
-        softmax(self)
+        _softmax(self)
     }
 }

core/src/func/mod.rs

@@ -7,12 +7,9 @@ pub use self::prelude::*;
 
 #[macro_use]
 pub mod activate;
-pub mod dropout;
 pub mod loss;
 
 pub(crate) mod prelude {
     pub use super::activate::prelude::*;
-    #[cfg(feature = "rand")]
-    pub use super::dropout::*;
     pub use super::loss::prelude::*;
 }

core/src/init/gen/lecun.rs → core/src/init/distr/lecun.rs

@@ -1,10 +1,11 @@
 /*
-    Appellation: lecun <module>
+    Appellation: lecun <distr>
     Contrib: FL03 <jo3mccain@icloud.com>
 */
+use crate::init::distr::TruncatedNormal;
 use num::Float;
 use rand::Rng;
-use rand_distr::{Distribution, Normal, NormalError, StandardNormal};
+use rand_distr::{Distribution, NormalError, StandardNormal};
 
 /// [LecunNormal] is a truncated [normal](rand_distr::Normal) distribution centered at 0
 /// with a standard deviation that is calculated as `σ = sqrt(1/n_in)`
@@ -18,14 +19,14 @@ impl LecunNormal {
     pub fn new(n: usize) -> Self {
         Self { n }
     }
-    /// Create a [normal](rand_distr::Normal) [distribution](Distribution) centered at 0;
+    /// Create a [truncated normal](TruncatedNormal) [distribution](Distribution) centered at 0;
     /// See [Self::std_dev] for the standard deviation calculations.
-    pub fn distr<F>(&self) -> Result<Normal<F>, NormalError>
+    pub fn distr<F>(&self) -> Result<TruncatedNormal<F>, NormalError>
     where
         F: Float,
         StandardNormal: Distribution<F>,
     {
-        Normal::new(F::zero(), self.std_dev())
+        TruncatedNormal::new(F::zero(), self.std_dev())
     }
     /// Calculate the standard deviation (`σ`) of the distribution.
     /// This is done by computing the root of the reciprocal of the number of inputs
@@ -48,6 +49,6 @@ where
     where
         R: Rng + ?Sized,
     {
-        self.distr().unwrap().sample(rng)
+        self.distr().expect("NormalError").sample(rng)
     }
 }