v0.1.14 Dev

.artifacts/archive/nn/Cargo.toml

@@ -100,17 +100,17 @@ test = false
 [build-dependencies]
 
 [dependencies]
-concision-neural = { path = "./neural", version = "0.1.12" }
-concision-nlp = { optional = true, path = "./nlp", version = "0.1.12" }
-concision-optim = { optional = true, path = "./optim", version = "0.1.12" }
-concision-s4 = { optional = true, path = "./s4", version = "0.1.12" }
-transformers = { optional = true, path = "./transformers", version = "0.1.12" }
+concision-neural = { path = "../neural", version = "0.1.12" }
+concision-nlp = { optional = true, path = "../nlp", version = "0.1.12" }
+concision-optim = { optional = true, path = "../optim", version = "0.1.12" }
+concision-s4 = { optional = true, path = "../s4", version = "0.1.12" }
+transformers = { optional = true, path = "../transformers", version = "0.1.12" }
 
 
 [dev-dependencies]
 anyhow = "1"
 approx = "0.5"
-concision = { path = "../../concision" }
+concision = { path = "../../../concision" }
 ndarray = { features = ["approx-0_5", "serde-1"], version = "0.15" }
 
 [package.metadata.docs.rs]

Cargo.toml

@@ -19,8 +19,8 @@ approx = "0.5"
 itertools = "0.12"
 lazy_static = "1"
 ndarray = { default-features = false, version = "0.15" }
-
 num = { default-features = false, version = "0.4" }
+paste = "1"
 smart-default = "0.7"
 strum = { default-features = false, features = ["derive"], version = "0.26" }
 

core/src/traits/arr/ops.rs

@@ -21,7 +21,7 @@ pub trait Inverse {
 pub trait Matmul<Rhs = Self> {
     type Output;
 
-    fn matmul(&self, rhs: Rhs) -> Self::Output;
+    fn matmul(&self, rhs: &Rhs) -> Self::Output;
 }
 
 pub trait Matpow<Rhs = Self> {
@@ -56,14 +56,14 @@ where
     }
 }
 
-impl<X, Y, S> Matmul<X> for S
+impl<S, X, Y> Matmul<X> for S
 where
     S: Dot<X, Output = Y>,
 {
     type Output = Y;
 
-    fn matmul(&self, rhs: X) -> Self::Output {
-        self.dot(&rhs)
+    fn matmul(&self, rhs: &X) -> Self::Output {
+        self.dot(rhs)
     }
 }
 

models/gnn/Cargo.toml

@@ -69,7 +69,7 @@ std = [
     "concision-core/std",
     "ndarray/std",
     "num/std",
-    "serde/std",
+    "serde?/std",
     "strum/std",
 ]
 

models/kan/Cargo.toml

@@ -66,7 +66,7 @@ std = [
     "concision-core/std",
     "ndarray/std",
     "num/std",
-    "serde/std",
+    "serde?/std",
     "strum/std",
 ]
 

models/transformers/Cargo.toml

@@ -66,7 +66,7 @@ std = [
     "concision-core/std",
     "ndarray/std",
     "num/std",
-    "serde/std",
+    "serde?/std",
     "strum/std",
 ]
 
@@ -89,6 +89,7 @@ test = true
 [dependencies]
 ndarray.workspace = true
 num.workspace = true
+paste.workspace = true
 smart-default.workspace = true
 strum.workspace = true
 

models/transformers/src/attention/head.rs

@@ -0,0 +1,45 @@
+/*
+    Appellation: head <module>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+use crate::params::QKVBase;
+use nd::*;
+
+pub struct AttentionHead<A = f64, S = OwnedRepr<A>, D = Ix2>
+where
+    D: Dimension,
+    S: RawData<Elem = A>,
+{
+    params: QKVBase<S, D>,
+}
+
+impl<A, S, D> AttentionHead<A, S, D>
+where
+    D: Dimension,
+    S: RawData<Elem = A>,
+{
+    pub fn from_params(params: QKVBase<S, D>) -> Self {
+        Self { params }
+    }
+
+    pub fn builder<Sh, F>(shape: Sh, builder: F) -> Self
+    where
+        F: Fn(D) -> ArrayBase<S, D>,
+        Sh: ShapeBuilder<Dim = D>,
+    {
+        Self::from_params(QKVBase::builder(shape, builder))
+    }
+
+    pub fn params(&self) -> &QKVBase<S, D> {
+        &self.params
+    }
+
+    pub fn params_mut(&mut self) -> &mut QKVBase<S, D> {
+        &mut self.params
+    }
+
+    access!(params::<q, k, v>);
+    fwd_builder!(new.default where A: Default, S: DataOwned);
+    fwd_builder!(ones.ones where A: Clone + num::One, S: DataOwned);
+    fwd_builder!(zeros.zeros where A: Clone + num::Zero, S: DataOwned);
+}

models/transformers/src/attention/mod.rs

@@ -2,3 +2,12 @@
     Appellation: attention <module>
     Contrib: FL03 <jo3mccain@icloud.com>
 */
+pub use self::head::AttentionHead;
+
+pub(crate) mod head;
+
+pub mod multi;
+
+pub(crate) mod prelude {
+    pub use super::head::AttentionHead;
+}

models/transformers/src/attention/multi/mod.rs

@@ -0,0 +1,10 @@
+/*
+    Appellation: multi <module>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+//! # Multi-Head Attention
+//!
+//!
+pub use self::multi_head::*;
+
+pub(crate) mod multi_head;

models/transformers/src/attention/multi/multi_head.rs

@@ -0,0 +1,6 @@
+/*
+    Appellation: multi_head <module>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+
+pub struct MultiHeadAttention;

models/transformers/src/impls/impl_head.rs

@@ -0,0 +1,19 @@
+/*
+    Appellation: impl_head <module>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+use crate::attention::AttentionHead;
+use crate::params::QKVBase;
+use nd::prelude::*;
+use nd::DataOwned;
+
+impl<A, S, D> Default for AttentionHead<A, S, D>
+where
+    A: Default,
+    D: Dimension,
+    S: DataOwned<Elem = A>,
+{
+    fn default() -> Self {
+        Self::from_params(QKVBase::default())
+    }
+}

models/transformers/src/impls/impl_linalg.rs

@@ -0,0 +1,50 @@
+/*
+    Appellation: impl_linalg <module>
+    Contrib: FL03 <jo3mccain@icloud.com>
+*/
+use crate::params::{QKVBase, QKV};
+use concision::Matmul;
+use nd::linalg::Dot;
+use nd::*;
+
+impl<A, S, T, D, E, F> Matmul<QKVBase<T, E>> for QKVBase<S, D>
+where
+    A: LinalgScalar,
+    D: Dimension,
+    E: Dimension,
+    F: Dimension,
+    S: Data<Elem = A>,
+    T: Data<Elem = A>,
+    ArrayBase<S, D>: Dot<ArrayBase<T, E>, Output = Array<A, F>>,
+{
+    type Output = QKV<A, F>;
+
+    fn matmul(&self, rhs: &QKVBase<T, E>) -> Self::Output {
+        QKVBase {
+            q: self.q().dot(rhs.q()),
+            k: self.k().dot(rhs.k()),
+            v: self.v().dot(rhs.v()),
+        }
+    }
+}
+
+impl<A, S, T, D, E, F> Matmul<ArrayBase<T, E>> for QKVBase<S, D>
+where
+    A: LinalgScalar,
+    D: Dimension,
+    E: Dimension,
+    F: Dimension,
+    S: Data<Elem = A>,
+    T: Data<Elem = A>,
+    ArrayBase<S, D>: Dot<ArrayBase<T, E>, Output = Array<A, F>>,
+{
+    type Output = QKV<A, F>;
+
+    fn matmul(&self, rhs: &ArrayBase<T, E>) -> Self::Output {
+        QKVBase {
+            q: self.q().dot(rhs),
+            k: self.k().dot(rhs),
+            v: self.v().dot(rhs),
+        }
+    }
+}