diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..a071a989 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ + +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[!src/llvm-project] +indent_style = space +indent_size = 4 + +[*.rs] +max_line_length = 100 + +[*.md] +# double whitespace at end of line +# denotes a line break in Markdown +trim_trailing_whitespace = false + +[*.yml] +indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 00000000..60bfbd9a --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,31 @@ +name: cleanup + +on: + pull_request: + types: + - closed + +jobs: + cache_cleanup: + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - + name: Cleanup + run: | + echo "Fetching list of cache keys" + cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id') + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh cache delete $cacheKey + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge \ No newline at end of file diff --git a/.github/workflows/crates.yml b/.github/workflows/crates.yml index eb5e39e6..3969f818 100644 --- a/.github/workflows/crates.yml +++ b/.github/workflows/crates.yml @@ -1,6 +1,6 @@ name: release -concurrency: +concurrency: cancel-in-progress: false group: ${{ github.workflow }}-${{ github.ref }} @@ -9,8 +9,6 @@ env: RUST_BACKTRACE: full on: - release: - types: [ published ] repository_dispatch: types: [ crates-io ] workflow_dispatch: @@ -28,6 +26,7 @@ permissions: jobs: crates-io: + if: github.event.inputs.environment == 'crates-io' || github.event_name == 'repository_dispatch' env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} environment: @@ -37,7 +36,7 @@ jobs: fail-fast: false max-parallel: 1 matrix: - package: + package: - concision-utils - concision-core - concision-data @@ -46,18 +45,20 @@ jobs: - concision-macros - concision # non-sdk packages + - concision-kan + - concision-s4 - concision-transformer - concision-ext runs-on: ubuntu-latest steps: - - + - name: Checkout uses: actions/checkout@v4 - - + - name: Setup Rust uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - + - name: Publish (${{ matrix.package }}) run: cargo publish --locked --package ${{ matrix.package }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..beb905dd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: release + +on: + release: + types: [ published ] + repository_dispatch: + types: [ release ] + workflow_dispatch: + inputs: + draft: + default: false + description: 'Create a draft release' + required: true + type: boolean + prerelease: + default: false + description: 'Create a prerelease' + required: true + type: boolean + +permissions: + contents: write + discussions: write + +jobs: + publish: + environment: + name: crates-io + url: https://crates.io/crates/${{ github.event.repository.name }} + runs-on: ubuntu-latest + steps: + - name: Trigger target workflow + uses: peter-evans/repository-dispatch@v3 + with: + event-type: crates-io + client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' + token: ${{ github.token }} + release: + continue-on-error: true + needs: publish + env: + IS_PRERELEASE: ${{ github.event.inputs.prerelease || false }} + IS_DRAFT: ${{ github.event.inputs.draft || false }} + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Create release + uses: softprops/action-gh-release@v1 + with: + append_body: false + draft: ${{ env.IS_DRAFT }} + prerelease: ${{ env.IS_PRERELEASE }} + tag_name: ${{ github.event.release.tag_name }} + body: | + ${{ github.event.release.body }} + + ## Links + + - [crates.io](https://crates.io/crates/${{ github.event.repository.name }}) + - [docs.rs](https://docs.rs/${{ github.event.repository.name }}) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f32aec32..5c022bfd 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,7 +24,7 @@ on: workflow_dispatch: inputs: benchmark: - default: true + default: false description: 'Run benchmarks' required: true type: boolean @@ -40,7 +40,7 @@ jobs: strategy: fail-fast: false matrix: - target: [x86_64-unknown-linux-gnu] # [ x86_64-unknown-linux-gnu, x86_64-apple-darwin, x86_64-pc-windows-msvc, wasm32-unknown-unknown, wasm32-wasip1, wasm32-wasip2 ] + target: [ x86_64-unknown-linux-gnu ] steps: - name: Checkout @@ -53,16 +53,20 @@ jobs: target: ${{ matrix.target }} - name: Build the workspace - run: cargo build -r --locked --workspace --all-features --target ${{ matrix.target }} - benchmark: - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') || github.event.action == 'repository_dispatch' && github.event.action == 'rust' || github.event_name == 'workflow_dispatch' && github.event.inputs.benchmark + run: cargo build --release --locked --workspace --all-features --target ${{ matrix.target }} + benchmark: + if: github.event_name == 'repository_dispatch' || github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') || github.event.inputs.benchmark == 'true' needs: build runs-on: ubuntu-latest + outputs: + results: ${{ steps.artifacts.outputs.artifact-id }} + permissions: + contents: write + checks: write strategy: fail-fast: false matrix: - package: [ concision ] - target: [ x86_64-unknown-linux-gnu ] # [ x86_64-unknown-linux-gnu, wasm32-unknown-unknown, wasm32-wasip1, wasm32-wasip2 ] + features: [ full ] steps: - name: Checkout @@ -72,18 +76,26 @@ jobs: uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - target: ${{ matrix.target }} - - name: Benchmark (${{ matrix.package }}) - run: cargo bench -v --locked --features full --package ${{ matrix.package }} --target ${{ matrix.target }} + name: Benchmark (${{ matrix.features }}) + run: cargo bench --locked --verbose --workspace --features ${{ matrix.features }} -- + - + name: Upload the benchmarks + id: artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmarks@${{ github.sha }} + if-no-files-found: error + overwrite: true + path: target/criterion/ test: needs: build runs-on: ubuntu-latest strategy: fail-fast: false matrix: - features: [ full ] - target: [ x86_64-unknown-linux-gnu ] # [ x86_64-unknown-linux-gnu, wasm32-unknown-unknown, wasm32-wasip1, wasm32-wasip2 ] + features: [ default, full ] # [ all, default, full ] + toolchain: [ stable ] # [ stable, nightly ] steps: - name: Checkout @@ -92,72 +104,18 @@ jobs: name: Setup Rust uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - target: ${{ matrix.target }} + cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + toolchain: ${{ matrix.toolchain }} + override: true - + if: matrix.features != 'default' && matrix.features != 'all' name: Test (${{ matrix.features }}) - run: cargo test -r --locked --workspace --target ${{ matrix.target}} --features ${{ matrix.features }} - test_std: - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - features: [ default ] # [ default, all ] - target: [ x86_64-unknown-linux-gnu ] - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - target: ${{ matrix.target }} - - - if: matrix.features == 'all' - name: Setup blas - run: | - sudo apt-get update -y - sudo apt-get install -y libopenblas-dev liblapack-dev + run: cargo test -r --locked --workspace --features ${{ matrix.features }} - if: matrix.features == 'default' name: Test (default) - run: cargo test -r --locked --workspace --target ${{ matrix.target}} + run: cargo test -r --locked --workspace - if: matrix.features == 'all' name: Test (all-features) - run: cargo test -r --locked --workspace --target ${{ matrix.target}} --all-features - test_no_std: - if: github.event_name == 'workflow_dispatch' && github.event.inputs.no_std || github.event_name == 'repository_dispatch' && github.event.action == 'rust' - continue-on-error: true - needs: build - runs-on: ubuntu-latest - env: - RUSTFLAGS: "-C panic=abort -Z panic_abort_tests" - strategy: - fail-fast: false - matrix: - features: [ alloc, no_std ] - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache-key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - toolchain: nightly - override: true - - - if: matrix.features != 'no_std' - continue-on-error: true - name: Test (${{ matrix.features }}) - run: cargo test -r --locked --workspace --no-default-features --features ${{ matrix.features }} - - - if: matrix.features == 'no_std' - continue-on-error: true - name: Test (no_std) - run: cargo test -r --locked --workspace --no-default-features + run: cargo test -r --locked --workspace --all-features diff --git a/Cargo.lock b/Cargo.lock index a06ee148..721ae806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,7 +257,7 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "concision" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -276,7 +276,7 @@ dependencies = [ [[package]] name = "concision-core" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -304,7 +304,7 @@ dependencies = [ [[package]] name = "concision-data" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -324,7 +324,7 @@ dependencies = [ [[package]] name = "concision-derive" -version = "0.1.23" +version = "0.2.0" dependencies = [ "proc-macro2", "quote", @@ -333,11 +333,13 @@ dependencies = [ [[package]] name = "concision-ext" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", "concision", + "concision-kan", + "concision-s4", "concision-transformer", "lazy_static", "ndarray", @@ -347,9 +349,21 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "concision-kan" +version = "0.2.0" +dependencies = [ + "anyhow", + "approx", + "concision", + "ndarray", + "num-traits", + "tracing", +] + [[package]] name = "concision-macros" -version = "0.1.23" +version = "0.2.0" dependencies = [ "proc-macro2", "quote", @@ -358,7 +372,7 @@ dependencies = [ [[package]] name = "concision-neural" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -383,9 +397,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "concision-s4" +version = "0.2.0" +dependencies = [ + "anyhow", + "approx", + "concision", + "ndarray", + "num-traits", + "tracing", +] + [[package]] name = "concision-transformer" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -397,7 +423,7 @@ dependencies = [ [[package]] name = "concision-utils" -version = "0.1.23" +version = "0.2.0" dependencies = [ "anyhow", "approx", @@ -423,6 +449,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "config" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595aae20e65c3be792d05818e8c63025294ac3cb7e200f11459063a352a6ef80" +dependencies = [ + "pathdiff", + "serde", + "serde_json", + "winnow", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1229,6 +1267,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1554,9 +1598,9 @@ dependencies = [ [[package]] name = "scsys" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611a66e87dfdb691d117bfb00cb281aea45a6cf36a0024b078e9f0354d79eb7d" +checksum = "dd97365a000c681edf2a3954250636442d7e76b2b53d32a0d516fa253c34e200" dependencies = [ "scsys-config", "scsys-core", @@ -1568,22 +1612,29 @@ dependencies = [ [[package]] name = "scsys-config" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "116a2e1d4edf29df645b60bf21f3c9e79b2d00cee4890b99632db42c2a64cbf1" +checksum = "50a68cdb65ec0a505e6a8d486755ffff8751db626d3933dae305b80192f5827b" dependencies = [ + "anyhow", + "config", "scsys-core", + "serde", + "serde_derive", + "serde_json", "smart-default", "strum", "thiserror", "tracing", + "tracing-subscriber", + "url", ] [[package]] name = "scsys-core" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd399aec2895143cefde2a71711bcb6fa46d27b072baec155c6bbbe52fe5a263" +checksum = "0939b72bac5a53728b7ff767996abd9547df2b7daf60228ee6c959761095b78b" dependencies = [ "anyhow", "chrono", @@ -1607,9 +1658,9 @@ dependencies = [ [[package]] name = "scsys-crypto" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31f0ddd5b6c6f1a5115f121c93c318979532ba7451b7d499dcb218f695298d6" +checksum = "70e266c3cbaf56d7c8fe8f47037c683c909fc85ed1cfa664cc61e35ed9f4f24b" dependencies = [ "anyhow", "bincode", @@ -1636,9 +1687,9 @@ dependencies = [ [[package]] name = "scsys-derive" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e120b90ac00014d38767588f7fe52ff34c89eff17f79f2cd5bc6ae389273ce17" +checksum = "f57af57d501f1d0b99af8ce5a7e81705fd01f9cf688da8610e42b8817c049570" dependencies = [ "proc-macro2", "quote", @@ -1647,18 +1698,18 @@ dependencies = [ [[package]] name = "scsys-traits" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "788fccc9e6f96dca5d973e66322edf445184b54afb37b07f4d5d72caa865d131" +checksum = "a5849f9fcb8ccacfaf68f287404236029a7d557fe8a875cdd4c1ec2b62b6987a" dependencies = [ "num-traits", ] [[package]] name = "scsys-util" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23e2cf19762946b932a38e04f160883b3ce0ee05f8bec9feb255aeefe922e37" +checksum = "b49c0764a02759cae0b0851ebcbab56a2ef18d5d87eb5344e1b9533478a7b82f" dependencies = [ "num-traits", "rand 0.9.1", @@ -1666,6 +1717,7 @@ dependencies = [ "serde_derive", "serde_json", "strum", + "tracing", ] [[package]] @@ -2038,6 +2090,7 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", "sharded-slab", "thread_local", "tracing", @@ -2087,6 +2140,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2406,6 +2460,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/Cargo.toml b/Cargo.toml index 7c63ea28..cd6d0cc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,22 +3,22 @@ default-members = [ "concision" ] members = [ - "concision", - "core", - "data", - "derive", - "macros", - "neural", + "concision", + "core", + "data", + "derive", + "macros", + "neural", "utils", - "ext", - "models/*", + "ext", + "models/*", ] resolver = "3" [workspace.package] authors = [ - "FL03 (https://github.com/FL03)", + "FL03 (https://github.com/FL03)", "Scattered-Systems (https://github.com/scattered-systems)" ] categories = ["science"] @@ -26,33 +26,35 @@ description = "Concision is a toolkit for designing machine-learning models in R edition = "2024" homepage = "https://github.com/FL03/concision/wiki" keywords = [ - "data-science", - "machine-learning", - "scsys", + "data-science", + "machine-learning", + "scsys", "toolkit" ] license = "Apache-2.0" readme = "README.md" repository = "https://github.com/FL03/concision.git" rust-version = "1.85.0" -version = "0.1.23" +version = "0.2.0" [workspace.dependencies] # sdk -concision = { default-features = false, path = "concision", version = "0.1.23" } -concision-core = { default-features = false, path = "core", version = "0.1.23" } -concision-data = { default-features = false, path = "data", version = "0.1.23" } -concision-derive = { default-features = false, path = "derive", version = "0.1.23" } -concision-macros = { default-features = false, path = "macros", version = "0.1.23" } -concision-neural = { default-features = false, path = "neural", version = "0.1.23" } -concision-utils = { default-features = false, path = "utils", version = "0.1.23" } +concision = { default-features = false, path = "concision", version = "0.2.0" } +concision-core = { default-features = false, path = "core", version = "0.2.0" } +concision-data = { default-features = false, path = "data", version = "0.2.0" } +concision-derive = { default-features = false, path = "derive", version = "0.2.0" } +concision-macros = { default-features = false, path = "macros", version = "0.2.0" } +concision-neural = { default-features = false, path = "neural", version = "0.2.0" } +concision-utils = { default-features = false, path = "utils", version = "0.2.0" } # extra crates -concision-ext = { default-features = false, path = "ext", version = "0.1.23" } -concision-transformer = { default-features = false, path = "models/transformer", version = "0.1.23" } +concision-ext = { default-features = false, path = "ext", version = "0.2.0" } +concision-kan = { default-features = false, path = "models/kan", version = "0.2.0" } +concision-s4 = { default-features = false, path = "models/s4", version = "0.2.0" } +concision-transformer = { default-features = false, path = "models/transformer", version = "0.2.0" } # custom -scsys = { default-features = false, features = ["derive"], version = "0.2.8" } -scsys-derive = { default-features = false, version = "0.2.8" } +scsys = { default-features = false, features = ["derive"], version = "0.3.0" } +scsys-derive = { default-features = false, version = "0.3.0" } # async futures = { default-features = false, version = "0.3" } tokio = { default-features = false, version = "1" } diff --git a/concision/Cargo.toml b/concision/Cargo.toml index 6d50005a..4a185d0f 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -18,6 +18,7 @@ version.workspace = true all-features = false features = ["full"] rustc-args = ["--cfg", "docsrs"] +version = "v{{version}}" [package.metadata.release] no-dev-version = true @@ -25,7 +26,7 @@ tag-name = "{{version}}" [lib] crate-type = [ - "cdylib", + "cdylib", "rlib" ] bench = true @@ -43,7 +44,7 @@ concision-neural = { optional = true, workspace = true } [dev-dependencies] anyhow = { workspace = true } approx = { workspace = true } -criterion = { workspace = true } +criterion = { features = ["plotters"], workspace = true } lazy_static = { workspace = true } ndarray = { workspace = true } num = { features = ["rand", "serde"], workspace = true } @@ -52,20 +53,20 @@ tracing-subscriber = { workspace = true } [features] default = [ - "data", - "neural", + "data", + "neural", "std", ] full = [ "default", "anyhow", - "approx", - "data", - "derive", - "macros", - "rand", - "serde", + "approx", + "data", + "derive", + "macros", + "rand", + "serde", "tracing" ] @@ -80,21 +81,21 @@ neural = ["dep:concision-neural"] # ************* [FF:Environments] ************* std = [ - "alloc", - "concision-core/std", + "alloc", + "concision-core/std", "concision-data?/std", "concision-neural?/std", ] wasi = [ - "concision-core/wasi", - "concision-data?/wasi", + "concision-core/wasi", + "concision-data?/wasi", "concision-neural?/wasi" ] wasm = [ - "concision-core/wasm", - "concision-data?/wasm", + "concision-core/wasm", + "concision-data?/wasm", "concision-neural?/wasm" ] @@ -105,68 +106,68 @@ alloc = [ ] anyhow = [ - "concision-core/anyhow", - "concision-data?/anyhow", + "concision-core/anyhow", + "concision-data?/anyhow", "concision-neural?/anyhow", ] approx = [ - "concision-core/approx", - "concision-data?/approx", + "concision-core/approx", + "concision-data?/approx", "concision-neural?/approx" ] complex = [ - "concision-core/complex", - "concision-data?/complex", + "concision-core/complex", + "concision-data?/complex", "concision-neural?/complex" ] json = [ - "concision-core/json", - "concision-data?/json", + "concision-core/json", + "concision-data?/json", "concision-neural?/json" ] rand = [ - "concision-core/rand", - "concision-data?/rand", + "concision-core/rand", + "concision-data?/rand", "concision-neural?/rand" ] rng = [ - "concision-core/rng", - "concision-data?/rng", + "concision-core/rng", + "concision-data?/rng", "concision-neural?/rng" ] rayon = [ - "concision-core/rayon", - "concision-data?/rayon", + "concision-core/rayon", + "concision-data?/rayon", "concision-neural?/rayon" ] rustfft = [ - "concision-core/rustfft", + "concision-core/rustfft", "concision-neural?/rustfft" ] serde = [ - "concision-core/serde", - "concision-data?/serde", + "concision-core/serde", + "concision-data?/serde", "concision-neural?/serde" ] tracing = [ - "concision-core/tracing", - "concision-data?/tracing", + "concision-core/tracing", + "concision-data?/tracing", "concision-neural?/tracing" ] # ********* [FF] Blas ********* blas = [ - "concision-core/blas", - "concision-data?/blas", + "concision-core/blas", + "concision-data?/blas", "concision-neural?/blas" ] @@ -189,7 +190,24 @@ name = "default" path = "benches/default.rs" required-features = ["std"] +[[bench]] +harness = false +name = "params" +path = "benches/params.rs" +required-features = [ + "neural", + "approx", + "rand", + "std", + "tracing", +] + # ************* [Examples] ************* [[example]] name = "basic" -required-features = ["approx", "rand", "std", "tracing"] +required-features = [ + "approx", + "rand", + "std", + "tracing", +] diff --git a/concision/benches/default.rs b/concision/benches/default.rs index 7e8fa351..e9b764e2 100644 --- a/concision/benches/default.rs +++ b/concision/benches/default.rs @@ -2,11 +2,12 @@ Appellation: default Contrib: @FL03 */ +use core::hint::black_box; use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; use lazy_static::lazy_static; -use std::hint::black_box; use std::time::Duration; +const SAMPLES: usize = 50; /// the default number of iterations to benchmark a method for const N: usize = 20; /// the default number of seconds a benchmark should complete in @@ -18,22 +19,23 @@ lazy_static! { } fn bench_fib_func(c: &mut Criterion) { - c.bench_function("fib::fibonacci", |b| { - b.iter(|| fib::fibonacci(black_box(N))) - }); + c.bench_function("fibonacci", |b| b.iter(|| fib::fibonacci(black_box(N)))); } fn bench_fib_recursive(c: &mut Criterion) { - c.bench_function("fib::recursive_fibonacci", |b| { + c.bench_function("recursive_fibonacci", |b| { b.iter(|| fib::recursive_fibonacci(black_box(N))) }); } fn bench_fib_iter(c: &mut Criterion) { let measure_for = Duration::from_secs(DEFAULT_DURATION_SECS); + // create a benchmark group for the Fibonacci iterator let mut group = c.benchmark_group("Fibonacci Iter"); + // set the measurement time for the group group.measurement_time(measure_for); - group.sample_size(50); + //set the sample size + group.sample_size(SAMPLES); for &n in &[10, 50, 100, 500, 1000] { group.bench_with_input(BenchmarkId::new("Fibonacci::compute", n), &n, |b, &x| { @@ -68,6 +70,7 @@ pub mod fib { /// a simple implementation of the fibonacci sequence for benchmarking purposes /// **Warning:** This will overflow the 128-bit unsigned integer at n=186 + #[inline] pub fn fibonacci(n: usize) -> u128 { // Use a and b to store the previous two values in the sequence let mut a = 0; @@ -82,8 +85,8 @@ pub mod fib { b } /// a recursive implementation of the fibonacci sequence - pub fn recursive_fibonacci(n: usize) -> u128 { - fn _inner(n: usize, previous: u128, current: u128) -> u128 { + pub const fn recursive_fibonacci(n: usize) -> u128 { + const fn _inner(n: usize, previous: u128, current: u128) -> u128 { if n == 0 { current } else { @@ -103,33 +106,69 @@ pub mod fib { } impl Fibonacci { - pub fn new() -> Fibonacci { + /// returns a new instance of the fibonacci sequence, with `curr` set to 0 and `next` + /// set to 1 + pub const fn new() -> Fibonacci { Fibonacci { curr: 0, next: 1 } } - + /// returns a copy of the current value + pub const fn curr(&self) -> u32 { + self.curr + } + /// returns a mutable reference to the current value + pub const fn curr_mut(&mut self) -> &mut u32 { + &mut self.curr + } + /// returns a copy of the next value + pub const fn next(&self) -> u32 { + self.next + } + /// returns a mutable reference to the next value + pub const fn next_mut(&mut self) -> &mut u32 { + &mut self.next + } + /// computes the nth value of the fibonacci sequence + #[inline] pub fn compute(&mut self, n: usize) -> u32 { if let Some(res) = self.nth(n + 1) { return res; } panic!("Unable to compute the nth value of the fibonacci sequence...") } - - pub fn set_curr(&mut self, curr: u32) -> &mut Self { + /// reset the instance to its default state, with `curr` set to 0 and `next` set to 1 + pub const fn reset(&mut self) -> &mut Self { + self.set_curr(0).set_next(1) + } + /// compute the next value in the fibonacci sequence, using the current and next values + #[inline] + const fn compute_next(&self) -> u32 { + self.curr() + self.next() + } + /// [`replace`](core::mem::replace) the current value with the given value, returning the + /// previous value + const fn replace_curr(&mut self, curr: u32) -> u32 { + core::mem::replace(self.curr_mut(), curr) + } + /// [`replace`](core::mem::replace) the next value with the given value, returning the + /// previous value + const fn replace_next(&mut self, next: u32) -> u32 { + core::mem::replace(self.next_mut(), next) + } + /// update the current value and return a mutable reference to the instance + const fn set_curr(&mut self, curr: u32) -> &mut Self { self.curr = curr; self } - - pub fn set_next(&mut self, next: u32) -> &mut Self { + /// update the next value and return a mutable reference to the instance + const fn set_next(&mut self, next: u32) -> &mut Self { self.next = next; self } - - pub fn reset(&mut self) -> &mut Self { - self.set_curr(0).set_next(1) - } - - fn compute_next(&self) -> u32 { - self.curr + self.next + /// replace the next value with the given, using the previous next as the new current + /// value, and returning the previous current value + const fn update(&mut self, next: u32) -> u32 { + let new = self.replace_next(next); + self.replace_curr(new) } } @@ -143,11 +182,13 @@ pub mod fib { type Item = u32; fn next(&mut self) -> Option { - use core::mem::replace; + // compute the new next value let new_next = self.compute_next(); - let new_curr = replace(&mut self.next, new_next); - - Some(replace(&mut self.curr, new_curr)) + // replaces the current next with the new value and replaces the current value with + // the previous next + let prev = self.update(new_next); + // return the previous current value + Some(prev) } } } diff --git a/concision/benches/params.rs b/concision/benches/params.rs new file mode 100644 index 00000000..ef271f27 --- /dev/null +++ b/concision/benches/params.rs @@ -0,0 +1,54 @@ +/* + appellation: params + authors: @FL03 +*/ +extern crate concision as cnc; +use cnc::init::Initialize; + +use core::hint::black_box; +use criterion::{BatchSize, BenchmarkId, Criterion}; +use ndarray::Array1; + +const SAMPLES: usize = 50; + +const DEFAULT_DURATION_SECS: u64 = 10; + +fn bench_params_forward(c: &mut Criterion) { + // create a benchmark group for the Fibonacci iterator + let mut group = c.benchmark_group("Params"); + // set the measurement time for the group + group.measurement_time(std::time::Duration::from_secs(DEFAULT_DURATION_SECS)); + //set the sample size + group.sample_size(SAMPLES); + + for &n in &[10, 50, 100, 500, 1000] { + group.bench_with_input(BenchmarkId::new("Params::forward", n), &n, |b, &x| { + b.iter_batched( + || { + let params = cnc::Params::::glorot_normal((n, 64)); + // return the configured parameters + params + }, + |params| { + let input = Array1::::linspace(0.0, 1.0, x); + let y = params + .forward(black_box(&input)) + .expect("Forward pass failed"); + y + }, + BatchSize::SmallInput, + ); + }); + } + + group.finish(); +} + +criterion::criterion_group! { + benches, + bench_params_forward, +} + +criterion::criterion_main! { + benches +} diff --git a/concision/src/lib.rs b/concision/src/lib.rs index 5219588f..4a738d9d 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -2,16 +2,35 @@ Appellation: concision Contrib: FL03 */ +//! # concision (cnc) +//! +//! [![crates.io](https://img.shields.io/crates/v/concision?style=for-the-badge&logo=rust)](https://crates.io/crates/concision) +//! [![docs.rs](https://img.shields.io/docsrs/concision?style=for-the-badge&logo=docs.rs)](https://docs.rs/concision) +//! [![GitHub License](https://img.shields.io/github/license/FL03/concision?style=for-the-badge&logo=github)](https://github.com/FL03/concision/blob/main/LICENSE) +//! +//! *** +//! //! `concision` aims to be a complete machine-learning toolkit written in Rust. The framework //! is designed to be performant, extensible, and easy to use while offering a wide range of //! features for building and training machine learning models. //! +//! The framework relies heavily on the [`ndarray`](https://docs.rs/ndarray) crate for its +//! n-dimensional arrays, which are essential for efficient data manipulation and mathematical +//! operations. +//! //! ## Features //! -//! - `ndarray`: extensive support for multi-dimensional arrays, enabling efficient data -//! manipulation. +//! - `data`: Provides utilities for data loading, preprocessing, and augmentation. +//! - `derive`: Custom derive macros for automatic implementation of traits +//! - `macros`: Procedural macros for simplifying common tasks in machine learning. +//! - `neural`: A neural network module that includes layers, optimizers, and training +//! utilities. //! -//! ### Long term goals +//! ### Dependency Features +//! +//! - `rayon`: Enables parallel processing for data loading and training. +//! +//! ## Roadmap //! //! - **DSL**: Create a pseudo-DSL for defining machine learning models and training processes. //! - **GPU**: Support for GPU acceleration to speed up training and inference. @@ -19,19 +38,24 @@ //! - **Visualization**: Utilities for visualizing model architectures and training progress //! - **WASM**: Native support for WebAssembly enabling models to be run in web browsers. //! +#![allow(clippy::module_inception, clippy::needless_doctest_main)] +#![cfg_attr(not(feature = "std"), no_std)] #![crate_name = "concision"] #[doc(inline)] pub use concision_core::*; #[doc(inline)] -#[cfg(feature = "data")] -pub use concision_data as data; -#[doc(inline)] #[cfg(feature = "derive")] pub use concision_derive::*; #[doc(inline)] #[cfg(feature = "macros")] pub use concision_macros::*; + +/// this module contains various data loaders, preprocessors, and augmenters +#[doc(inline)] +#[cfg(feature = "data")] +pub use concision_data as data; +/// this module defines various neural network abstractions, layers, and training utilities #[doc(inline)] #[cfg(feature = "neural")] pub use concision_neural as nn; diff --git a/core/Cargo.toml b/core/Cargo.toml index 25edb371..bbdbca02 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -65,10 +65,10 @@ rand = { optional = true, workspace = true } rand_distr = { optional = true, workspace = true } [dev-dependencies] -lazy_static.workspace = true +lazy_static = { workspace = true } [features] -default = [ +default = [ "std", ] @@ -125,7 +125,7 @@ alloc = [ ] anyhow = [ - "dep:anyhow", + "dep:anyhow", "scsys/anyhow", ] diff --git a/core/src/activate/mod.rs b/core/src/activate/mod.rs index dac3e884..32707432 100644 --- a/core/src/activate/mod.rs +++ b/core/src/activate/mod.rs @@ -26,13 +26,7 @@ mod impls { pub(crate) mod prelude { pub use super::traits::*; - pub use super::{Activate, ActivateGradient, BinaryAction}; -} - -pub trait BinaryAction { - type Output; - - fn activate(&self, lhs: A, rhs: B) -> Self::Output; + pub use super::{Activate, ActivateGradient}; } /// The [Activate] trait enables the definition of new activation functions often implemented @@ -50,6 +44,10 @@ pub trait ActivateGradient: Activate { fn activate_gradient(&self, rhs: Rhs) -> Self::Delta; } +/* + ************* Implementations ************* +*/ + impl Activate for Box> { type Output = Y; diff --git a/core/src/error.rs b/core/src/error.rs index 5d5314cb..3e1d21b2 100644 --- a/core/src/error.rs +++ b/core/src/error.rs @@ -2,50 +2,62 @@ Appellation: error Contrib: @FL03 */ +//! This module implements the core [`Error`] type for the framework and provides a [`Result`] +//! type alias for convenience. +#[cfg(feature = "alloc")] +use alloc::{boxed::Box, string::String}; /// a type alias for a [Result] with a [Error] pub type Result = core::result::Result; +/// The [`Error`] type enumerates various errors that can occur within the framework. #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("Invalid Shape: {0}")] - InvalidShape(&'static str), - #[error("Unknown Error: {0}")] - Unknown(&'static str), - #[error(transparent)] - MathError(#[from] concision_utils::UtilityError), + #[error("Invalid Shape")] + InvalidShape, #[error(transparent)] PadError(#[from] crate::ops::pad::error::PadError), #[error(transparent)] ParamError(#[from] crate::params::error::ParamsError), + #[cfg(feature = "anyhow")] #[error(transparent)] - ShapeError(#[from] ndarray::ShapeError), + AnyError(#[from] anyhow::Error), #[cfg(feature = "alloc")] #[error(transparent)] - BoxError(#[from] alloc::boxed::Box), - #[cfg(feature = "anyhow")] - #[error(transparent)] - Other(#[from] anyhow::Error), + BoxError(#[from] Box), #[cfg(feature = "serde")] #[error(transparent)] DeserializeError(#[from] serde::de::value::Error), + #[error(transparent)] + FmtError(#[from] core::fmt::Error), + #[cfg(feature = "serde_json")] + #[error(transparent)] + JsonError(#[from] serde_json::Error), #[cfg(feature = "std")] #[error(transparent)] IoError(#[from] std::io::Error), + #[error(transparent)] + ShapeError(#[from] ndarray::ShapeError), #[cfg(feature = "rand")] #[error(transparent)] UniformError(#[from] rand_distr::uniform::Error), + #[cfg(feature = "alloc")] + #[error("Unknown Error: {0}")] + Unknown(String), + #[error(transparent)] + UtilError(#[from] concision_utils::UtilityError), } #[cfg(feature = "alloc")] -impl From for Error { - fn from(value: alloc::string::String) -> Self { - Self::Unknown(alloc::boxed::Box::leak(value.into_boxed_str())) +impl From for Error { + fn from(value: String) -> Self { + Self::Unknown(value) } } -impl From<&'static str> for Error { - fn from(value: &'static str) -> Self { - Self::Unknown(value) +#[cfg(feature = "alloc")] +impl From<&str> for Error { + fn from(value: &str) -> Self { + Self::Unknown(String::from(value)) } } diff --git a/core/src/init/mod.rs b/core/src/init/mod.rs index 00171b62..43fc1fa6 100644 --- a/core/src/init/mod.rs +++ b/core/src/init/mod.rs @@ -35,12 +35,6 @@ pub mod distr { } } -#[doc(hidden)] -#[doc(no_inline)] -pub use rand; -#[doc(no_inline)] -pub use rand_distr; - pub(crate) mod prelude { pub use super::UniformResult; pub use super::distr::prelude::*; diff --git a/core/src/lib.rs b/core/src/lib.rs index dab7d7a5..dfac659b 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -2,22 +2,35 @@ Appellation: concision-core Contrib: @FL03 */ -//! This library provides the core abstractions and utilities for the Concision framework. +//! # concision-core +//! +//! This library provides the core abstractions and utilities for the concision (cnc) machine +//! learning framework. //! //! ## Features //! -//! - [ParamsBase]: A structure for defining the parameters within a neural network. -//! - [Backward]: This trait denotes a single backward pass through a layer of a neural network. -//! - [Forward]: This trait denotes a single forward pass through a layer of a neural network. +//! - [`ParamsBase`]: A structure for defining the parameters within a neural network. +//! - [`Backward`]: This trait establishes a common interface for backward propagation. +//! - [`Forward`]: This trait denotes a single forward pass through a layer of a neural network //! #![allow( clippy::module_inception, + clippy::needless_doctest_main, + clippy::upper_case_acronyms )] #![cfg_attr(not(feature = "std"), no_std)] +#![crate_type = "lib"] #[cfg(feature = "alloc")] extern crate alloc; +#[cfg(feature = "rand")] +#[doc(no_inline)] +pub use rand; +#[cfg(feature = "rand")] +#[doc(no_inline)] +pub use rand_distr; + #[doc(inline)] pub use concision_utils as utils; @@ -31,17 +44,23 @@ pub use self::{ pub(crate) mod macros { #[macro_use] pub mod seal; - #[macro_use] - pub mod unary; } - +/// this module is dedicated to activation function pub mod activate; +/// this module provides the base [`Error`] type for the library pub mod error; +/// this module establishes generic random initialization routines for models, params, and +/// tensors. pub mod init; +/// this module focuses on the loss functions used in training neural networks. pub mod loss; +/// this module provides the [`ParamsBase`] type for the library, which is used to define the +/// parameters of a neural network. pub mod params; pub mod ops { + //! This module provides the core operations for tensors, including filling, padding, + //! reshaping, and tensor manipulation. #[doc(inline)] pub use self::prelude::*; @@ -62,9 +81,12 @@ pub mod ops { } } pub mod traits { + //! This module provides the core traits for the library, such as [`Backward`] and + //! [`Forward`] #[doc(inline)] pub use self::prelude::*; + pub mod apply; pub mod clip; pub mod codex; pub mod gradient; @@ -78,6 +100,8 @@ pub mod traits { pub mod wnb; pub(crate) mod prelude { + #[doc(inline)] + pub use super::apply::*; #[doc(inline)] pub use super::clip::*; #[doc(inline)] diff --git a/core/src/macros/unary.rs b/core/src/macros/unary.rs deleted file mode 100644 index 377f90c7..00000000 --- a/core/src/macros/unary.rs +++ /dev/null @@ -1,67 +0,0 @@ -/* - Appellation: unary - Contrib: @FL03 -*/ -#[allow(unused_macros)] -macro_rules! unary_op_trait { - ($($name:ident::$call:ident($($rest:tt)*)),* $(,)?) => { - $( - unary_op_trait!(@impl $name::$call($($rest)*)); - )* - }; - - (@impl $name:ident::$call:ident(self)) => { - paste::paste! { - pub trait $name { - type Output; - - fn $call(self) -> Self::Output; - - fn [<$call _derivative>](self) -> Self::Output; - } - } - - }; - (@impl $name:ident::$call:ident(&self)) => { - paste::paste! { - pub trait $name { - type Output; - - fn $call(&self) -> Self::Output; - - fn [<$call _derivative>](&self) -> Self::Output; - } - } - }; -} -#[allow(unused_macros)] -macro_rules! impl_unary_op_trait { - ($name:ident::<$T:ty, Output = $out:ty>::$call:ident(self) => { - f => $func:expr, - df => $df:expr, - }) => { - paste::paste! { - impl $name for $T { - type Output = $out; - - fn $call(self) -> Self::Output { - $func(self) - } - - fn [<$call _derivative>](self) -> Self::Output { - $df(self) - } - } - } - }; - - ($name:ident::$call:ident(&self) => { - f: $func:expr, - df: $df:expr, - }) => { - impl_unary_op_trait!($name::::$call(self) => { - f: $func, - df: $df, - }); - }; -} diff --git a/core/src/ops/pad/mode.rs b/core/src/ops/pad/mode.rs index a6aa0d76..8d8e43a8 100644 --- a/core/src/ops/pad/mode.rs +++ b/core/src/ops/pad/mode.rs @@ -76,7 +76,6 @@ pub enum PadMode { Wrap, } - impl From for PadMode { fn from(value: T) -> Self { PadMode::Constant(value) diff --git a/core/src/params/params.rs b/core/src/params/params.rs index c423de33..7757d931 100644 --- a/core/src/params/params.rs +++ b/core/src/params/params.rs @@ -5,7 +5,10 @@ use ndarray::prelude::*; use ndarray::{Data, DataMut, DataOwned, Dimension, RawData, RemoveAxis, ShapeBuilder}; -/// this structure extends the `ArrayBase` type to include bias +/// The [`ParamsBase`] struct is a generic container for a set of weights and biases for a +/// model. The implementation is designed around the [`ArrayBase`] type from the +/// `ndarray` crate, which allows for flexible and efficient storage of multi-dimensional +/// arrays. pub struct ParamsBase where D: Dimension, @@ -20,14 +23,11 @@ where D: Dimension, S: RawData, { - pub fn new(bias: ArrayBase, weights: ArrayBase) -> Self - where - A: Clone, - S: DataOwned, - { + /// create a new instance of the [`ParamsBase`] with the given bias and weights + pub const fn new(bias: ArrayBase, weights: ArrayBase) -> Self { Self { bias, weights } } - + /// create a new instance of the [`ModelParams`] from the given shape and element; pub fn from_elems(shape: Sh, elem: A) -> Self where A: Clone, @@ -38,7 +38,7 @@ where let weights = ArrayBase::from_elem(shape, elem.clone()); let dim = weights.raw_dim(); let bias = ArrayBase::from_elem(dim.remove_axis(Axis(0)), elem); - Self { bias, weights } + Self::new(bias, weights) } /// create an instance of the parameters with all values set to the default value pub fn default(shape: Sh) -> Self @@ -48,10 +48,7 @@ where S: DataOwned, Sh: ShapeBuilder, { - let weights = ArrayBase::default(shape); - let dim = weights.raw_dim(); - let bias = ArrayBase::default(dim.remove_axis(Axis(0))); - Self { bias, weights } + Self::from_elems(shape, A::default()) } /// initialize the parameters with all values set to zero pub fn ones(shape: Sh) -> Self @@ -61,10 +58,7 @@ where S: DataOwned, Sh: ShapeBuilder, { - let weights = ArrayBase::ones(shape); - let dim = weights.raw_dim(); - let bias = ArrayBase::ones(dim.remove_axis(Axis(0))); - Self { bias, weights } + Self::from_elems(shape, A::one()) } /// create an instance of the parameters with all values set to zero pub fn zeros(shape: Sh) -> Self @@ -74,18 +68,14 @@ where S: DataOwned, Sh: ShapeBuilder, { - let weights = ArrayBase::zeros(shape); - let dim = weights.raw_dim(); - let bias = ArrayBase::zeros(dim.remove_axis(Axis(0))); - Self { bias, weights } + Self::from_elems(shape, A::zero()) } /// returns an immutable reference to the bias pub const fn bias(&self) -> &ArrayBase { &self.bias } /// returns a mutable reference to the bias - #[inline] - pub fn bias_mut(&mut self) -> &mut ArrayBase { + pub const fn bias_mut(&mut self) -> &mut ArrayBase { &mut self.bias } /// returns an immutable reference to the weights @@ -93,11 +83,9 @@ where &self.weights } /// returns a mutable reference to the weights - #[inline] - pub fn weights_mut(&mut self) -> &mut ArrayBase { + pub const fn weights_mut(&mut self) -> &mut ArrayBase { &mut self.weights } - /// assign the bias pub fn assign_bias(&mut self, bias: &ArrayBase) -> &mut Self where @@ -224,9 +212,13 @@ where self.bias().is_empty() } /// the total number of elements within the weight tensor - pub fn len(&self) -> usize { + pub fn count_weight(&self) -> usize { self.weights().len() } + /// the total number of elements within the bias tensor + pub fn count_bias(&self) -> usize { + self.bias().len() + } /// returns the raw dimensions of the weights; pub fn raw_dim(&self) -> D { self.weights().raw_dim() @@ -240,16 +232,17 @@ where pub fn shape_bias(&self) -> &[usize] { self.bias().shape() } + /// returns the total number of parameters within the layer + pub fn size(&self) -> usize { + self.weights().len() + self.bias().len() + } /// returns an owned instance of the parameters pub fn to_owned(&self) -> ParamsBase, D> where A: Clone, S: DataOwned, { - ParamsBase { - bias: self.bias().to_owned(), - weights: self.weights().to_owned(), - } + ParamsBase::new(self.bias().to_owned(), self.weights().to_owned()) } /// change the shape of the parameters; the shape of the bias parameters is determined by /// removing the "zero-th" axis of the given shape @@ -267,27 +260,21 @@ where let dim = shape.raw_dim().clone(); let bias = self.bias().to_shape(dim.remove_axis(Axis(0)))?; let weights = self.weights().to_shape(dim)?; - Ok(ParamsBase { bias, weights }) + Ok(ParamsBase::new(bias, weights)) } /// returns a "view" of the parameters; see [view](ArrayBase::view) for more information pub fn view(&self) -> ParamsBase, D> where S: Data, { - ParamsBase { - bias: self.bias().view(), - weights: self.weights().view(), - } + ParamsBase::new(self.bias().view(), self.weights().view()) } /// returns mutable view of the parameters; see [view_mut](ArrayBase::view_mut) for more information pub fn view_mut(&mut self) -> ParamsBase, D> where S: ndarray::DataMut, { - ParamsBase { - bias: self.bias.view_mut(), - weights: self.weights.view_mut(), - } + ParamsBase::new(self.bias.view_mut(), self.weights.view_mut()) } } @@ -331,10 +318,7 @@ where A: Clone, { fn clone(&self) -> Self { - Self { - bias: self.bias.clone(), - weights: self.weights.clone(), - } + Self::new(self.bias().clone(), self.weights().clone()) } } @@ -354,7 +338,7 @@ where A: PartialEq, { fn eq(&self, other: &Self) -> bool { - self.bias == other.bias && self.weights == other.weights + self.bias() == other.bias() && self.weights() == other.weights() } } @@ -374,8 +358,8 @@ where { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { f.debug_struct("ModelParams") - .field("bias", &self.bias) - .field("weights", &self.weights) + .field("bias", self.bias()) + .field("weights", self.weights()) .finish() } } diff --git a/core/src/traits/apply.rs b/core/src/traits/apply.rs new file mode 100644 index 00000000..a390bd54 --- /dev/null +++ b/core/src/traits/apply.rs @@ -0,0 +1,142 @@ +/* + Appellation: gradient + Contrib: @FL03 +*/ +/// A trait declaring basic gradient-related routines for a neural network +pub trait ApplyGradient { + type Output; + + fn apply_gradient(&mut self, grad: &Delta, lr: T) -> crate::Result; + + fn apply_gradient_with_decay( + &mut self, + grad: &Delta, + lr: T, + decay: T, + ) -> crate::Result; +} + +/// This trait extends the [ApplyGradient] trait by allowing for momentum-based optimization +pub trait ApplyGradientExt: ApplyGradient { + type Velocity; + + fn apply_gradient_with_momentum( + &mut self, + grad: &Delta, + lr: T, + momentum: T, + velocity: &mut Self::Velocity, + ) -> crate::Result; + + fn apply_gradient_with_decay_and_momentum( + &mut self, + grad: &Delta, + lr: T, + decay: T, + momentum: T, + velocity: &mut Self::Velocity, + ) -> crate::Result; +} + +/* + ************* Implementations ************* +*/ + +use ndarray::{Array, ArrayBase, Data, DataMut, Dimension, ScalarOperand, ShapeError}; +use num_traits::{Float, FromPrimitive}; + +impl ApplyGradient, A> for ArrayBase +where + A: Float + FromPrimitive + ScalarOperand, + S: DataMut, + T: Data, + D: Dimension, +{ + type Output = (); + + fn apply_gradient(&mut self, grad: &ArrayBase, lr: A) -> crate::Result { + if self.shape() != grad.shape() { + return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); + } + let batch_size = if !grad.shape().is_empty() { + A::from_usize(self.shape()[0]).unwrap() + } else { + A::one() + }; + self.scaled_add(lr / batch_size, grad); + Ok(()) + } + + fn apply_gradient_with_decay( + &mut self, + grad: &ArrayBase, + lr: A, + decay: A, + ) -> crate::Result { + if self.shape() != grad.shape() { + return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); + } + let batch_size = if !grad.shape().is_empty() { + A::from_usize(self.shape()[0]).unwrap() + } else { + A::one() + }; + self.scaled_add(lr / batch_size, &(grad + &*self * decay)); + Ok(()) + } +} + +impl ApplyGradientExt, A> for ArrayBase +where + A: Float + FromPrimitive + ScalarOperand, + S: DataMut, + T: Data, + D: Dimension, +{ + type Velocity = Array; + + fn apply_gradient_with_momentum( + &mut self, + grad: &ArrayBase, + lr: A, + momentum: A, + velocity: &mut Self::Velocity, + ) -> crate::Result { + if self.shape() != grad.shape() { + return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); + } + let batch_size = if !grad.shape().is_empty() { + A::from_usize(self.shape()[0]).unwrap() + } else { + A::one() + }; + *velocity = &*velocity * momentum + grad * (A::one() - momentum); + self.scaled_add(lr / batch_size, velocity); + Ok(()) + } + + fn apply_gradient_with_decay_and_momentum( + &mut self, + grad: &ArrayBase, + lr: A, + decay: A, + momentum: A, + velocity: &mut Self::Velocity, + ) -> crate::Result { + if self.shape() != grad.shape() { + return Err( + ndarray::ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into(), + ); + } + let batch_size = if !grad.shape().is_empty() { + A::from_usize(self.shape()[0]).unwrap() + } else { + A::one() + }; + + let adjusted_grad = grad + &*self * decay; + *velocity = &*velocity * momentum + adjusted_grad * (A::one() - momentum); + self.scaled_add(lr / batch_size, velocity); + Ok(()) + } +} diff --git a/core/src/traits/gradient.rs b/core/src/traits/gradient.rs index 036f4b75..b53a10a8 100644 --- a/core/src/traits/gradient.rs +++ b/core/src/traits/gradient.rs @@ -1,142 +1,12 @@ /* - Appellation: train - Contrib: @FL03 + appellation: gradient + authors: @FL03 */ -/// A trait declaring basic gradient-related routines for a neural network -pub trait ApplyGradient { - type Output; - - fn apply_gradient(&mut self, grad: &Delta, lr: T) -> crate::Result; - - fn apply_gradient_with_decay( - &mut self, - grad: &Delta, - lr: T, - decay: T, - ) -> crate::Result; -} - -/// This trait extends the [ApplyGradient] trait by allowing for momentum-based optimization -pub trait ApplyGradientExt: ApplyGradient { - type Velocity; - fn apply_gradient_with_momentum( - &mut self, - grad: &Delta, - lr: T, - momentum: T, - velocity: &mut Self::Velocity, - ) -> crate::Result; - - fn apply_gradient_with_decay_and_momentum( - &mut self, - grad: &Delta, - lr: T, - decay: T, - momentum: T, - velocity: &mut Self::Velocity, - ) -> crate::Result; -} - -/* - ************* Implementations ************* -*/ - -use ndarray::{Array, ArrayBase, Data, DataMut, Dimension, ScalarOperand, ShapeError}; -use num_traits::{Float, FromPrimitive}; - -impl ApplyGradient, A> for ArrayBase -where - A: Float + FromPrimitive + ScalarOperand, - S: DataMut, - T: Data, - D: Dimension, -{ - type Output = (); - - fn apply_gradient(&mut self, grad: &ArrayBase, lr: A) -> crate::Result { - if self.shape() != grad.shape() { - return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); - } - let batch_size = if !grad.shape().is_empty() { - A::from_usize(self.shape()[0]).unwrap() - } else { - A::one() - }; - self.scaled_add(lr / batch_size, grad); - Ok(()) - } - - fn apply_gradient_with_decay( - &mut self, - grad: &ArrayBase, - lr: A, - decay: A, - ) -> crate::Result { - if self.shape() != grad.shape() { - return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); - } - let batch_size = if !grad.shape().is_empty() { - A::from_usize(self.shape()[0]).unwrap() - } else { - A::one() - }; - self.scaled_add(lr / batch_size, &(grad + &*self * decay)); - Ok(()) - } -} - -impl ApplyGradientExt, A> for ArrayBase -where - A: Float + FromPrimitive + ScalarOperand, - S: DataMut, - T: Data, - D: Dimension, -{ - type Velocity = Array; - - fn apply_gradient_with_momentum( - &mut self, - grad: &ArrayBase, - lr: A, - momentum: A, - velocity: &mut Self::Velocity, - ) -> crate::Result { - if self.shape() != grad.shape() { - return Err(ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into()); - } - let batch_size = if !grad.shape().is_empty() { - A::from_usize(self.shape()[0]).unwrap() - } else { - A::one() - }; - *velocity = &*velocity * momentum + grad * (A::one() - momentum); - self.scaled_add(lr / batch_size, velocity); - Ok(()) - } - - fn apply_gradient_with_decay_and_momentum( - &mut self, - grad: &ArrayBase, - lr: A, - decay: A, - momentum: A, - velocity: &mut Self::Velocity, - ) -> crate::Result { - if self.shape() != grad.shape() { - return Err( - ndarray::ShapeError::from_kind(ndarray::ErrorKind::IncompatibleShape).into(), - ); - } - let batch_size = if !grad.shape().is_empty() { - A::from_usize(self.shape()[0]).unwrap() - } else { - A::one() - }; +/// The [`Gradient`] trait defines a common interface for all gradients +pub trait Gradient { + type Delta<_T>; + type Output; - let adjusted_grad = grad + &*self * decay; - *velocity = &*velocity * momentum + adjusted_grad * (A::one() - momentum); - self.scaled_add(lr / batch_size, velocity); - Ok(()) - } + fn grad(&self, rhs: &Rhs) -> Self::Delta; } diff --git a/ext/Cargo.toml b/ext/Cargo.toml index ff19e520..83bef4a4 100644 --- a/ext/Cargo.toml +++ b/ext/Cargo.toml @@ -33,6 +33,8 @@ doctest = true test = true [dependencies] +concision-kan = { optional = true, workspace = true } +concision-s4 = { optional = true, workspace = true } concision-transformer = { optional = true, workspace = true } # local concision = { features = ["neural"], workspace = true } @@ -60,26 +62,32 @@ default = [ full = [ "anyhow", - "default", + "default", "models", - "rand", - "serde", + "rand", + "serde", "tracing" ] # ************* [FF:Features] ************* models = [ - "simple", + "simple", "transformer" ] simple = [] +kan = ["dep:concision-kan"] + +s4 = ["dep:concision-s4"] + transformer = ["dep:concision-transformer"] # ************* [FF:Environments] ************* std = [ - "concision/std", + "concision/std", + "concision-kan?/std", + "concision-s4?/std", "concision-transformer?/std", "scsys/std", "ndarray/std", @@ -91,51 +99,77 @@ std = [ anyhow = [ "dep:anyhow", "concision/anyhow", + "concision-kan?/anyhow", + "concision-s4?/anyhow", "concision-transformer?/anyhow", "scsys/anyhow", ] approx = [ "dep:approx", - "concision/approx", - "ndarray/approx", + "concision/approx", + "concision-kan?/approx", + "concision-s4?/approx", "concision-transformer?/approx", + "ndarray/approx", ] blas = [ - "concision/blas", - "ndarray/blas", + "concision/blas", + "concision-kan?/blas", + "concision-s4?/blas", "concision-transformer?/blas", + "ndarray/blas", ] rand = [ - "concision/rand", + "concision/rand", + "concision-kan?/rand", + "concision-s4?/rand", "concision-transformer?/rand", ] rayon = [ - "concision/rayon", - "ndarray/rayon", + "concision/rayon", + "concision-kan?/rayon", + "concision-s4?/rayon", "concision-transformer?/rayon", + "ndarray/rayon", ] serde = [ - "concision/serde", + "concision/serde", + "concision-kan?/serde", + "concision-s4?/serde", "concision-transformer?/serde", + "ndarray/serde", + "scsys/serde", ] tracing = [ - "concision/tracing", "dep:tracing", + "concision/tracing", + "concision-kan?/tracing", + "concision-s4?/tracing", "concision-transformer?/tracing", + "scsys/tracing", ] # ************* [Examples] ************* [[example]] name = "simple" -required-features = ["anyhow", "approx", "rand", "tracing"] +required-features = [ + "anyhow", + "approx", + "rand", + "tracing", +] # ************* [FF:Dependencies] ************* [[test]] name = "simple" -required-features = ["anyhow", "approx", "rand"] +required-features = [ + "anyhow", + "approx", + "rand", +] diff --git a/ext/src/lib.rs b/ext/src/lib.rs index 5f0bf8d1..fa048da6 100644 --- a/ext/src/lib.rs +++ b/ext/src/lib.rs @@ -2,7 +2,12 @@ Appellation: concision-models Contrib: @FL03 */ -//! This crate extends the `concision` framework to provide a set of models and layers +//! # concision-ext +//! +//! This library uses the [`concision`](https://docs.rs/concision) framework to implement a +//! variety of additional machine learning models and layers. +//! +#![allow(clippy::module_inception, clippy::needless_doctest_main)] #![cfg_attr(not(feature = "std"), no_std)] // #[cfg(feature = "alloc")] @@ -12,12 +17,20 @@ extern crate concision as cnc; #[cfg(feature = "simple")] pub mod simple; +#[cfg(feature = "kan")] +pub use concision_kan as kan; +#[cfg(feature = "s4")] +pub use concision_s4 as s4; #[cfg(feature = "transformer")] pub use concision_transformer as transformer; pub mod prelude { #[cfg(feature = "simple")] pub use crate::simple::SimpleModel; + #[cfg(feature = "kan")] + pub use concision_kan::prelude::*; + #[cfg(feature = "s4")] + pub use concision_s4::prelude::*; #[cfg(feature = "transformer")] - pub use concision_transformer::TransformerModel; + pub use concision_transformer::prelude::*; } diff --git a/ext/src/simple.rs b/ext/src/simple.rs index 067031f9..b5693821 100644 --- a/ext/src/simple.rs +++ b/ext/src/simple.rs @@ -9,6 +9,7 @@ use ndarray::prelude::*; use ndarray::{Data, ScalarOperand}; use num_traits::{Float, FromPrimitive, NumAssign}; +#[derive(Clone, Debug)] pub struct SimpleModel { pub config: StandardModelConfig, pub features: ModelFeatures, @@ -27,35 +28,67 @@ impl SimpleModel { params, } } - #[cfg(feature = "rand")] - pub fn init(self) -> Self - where - T: Float + FromPrimitive, - cnc::init::rand_distr::StandardNormal: cnc::init::rand_distr::Distribution, - { - let params = ModelParams::glorot_normal(self.features); - SimpleModel { params, ..self } - } - + /// returns a reference to the model configuration pub const fn config(&self) -> &StandardModelConfig { &self.config } - - pub fn config_mut(&mut self) -> &mut StandardModelConfig { + /// returns a mutable reference to the model configuration + pub const fn config_mut(&mut self) -> &mut StandardModelConfig { &mut self.config } - + /// returns the model features pub const fn features(&self) -> ModelFeatures { self.features } - + /// returns a mutable reference to the model features + pub const fn features_mut(&mut self) -> &mut ModelFeatures { + &mut self.features + } + /// returns a reference to the model parameters pub const fn params(&self) -> &ModelParams { &self.params } - - pub fn params_mut(&mut self) -> &mut ModelParams { + /// returns a mutable reference to the model parameters + pub const fn params_mut(&mut self) -> &mut ModelParams { &mut self.params } + /// set the current configuration and return a mutable reference to the model + pub fn set_config(&mut self, config: StandardModelConfig) -> &mut Self { + self.config = config; + self + } + /// set the current features and return a mutable reference to the model + pub fn set_features(&mut self, features: ModelFeatures) -> &mut Self { + self.features = features; + self + } + /// set the current parameters and return a mutable reference to the model + pub fn set_params(&mut self, params: ModelParams) -> &mut Self { + self.params = params; + self + } + /// consumes the current instance to create another with the given configuration + pub fn with_config(self, config: StandardModelConfig) -> Self { + Self { config, ..self } + } + /// consumes the current instance to create another with the given features + pub fn with_features(self, features: ModelFeatures) -> Self { + Self { features, ..self } + } + /// consumes the current instance to create another with the given parameters + pub fn with_params(self, params: ModelParams) -> Self { + Self { params, ..self } + } + /// initializes the model with Glorot normal distribution + #[cfg(feature = "rand")] + pub fn init(self) -> Self + where + T: Float + FromPrimitive, + cnc::rand_distr::StandardNormal: cnc::rand_distr::Distribution, + { + let params = ModelParams::glorot_normal(self.features()); + SimpleModel { params, ..self } + } } impl Model for SimpleModel { diff --git a/models/.gitkeep b/models/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/models/kan/Cargo.toml b/models/kan/Cargo.toml new file mode 100644 index 00000000..f901e27d --- /dev/null +++ b/models/kan/Cargo.toml @@ -0,0 +1,107 @@ +[package] +build = "build.rs" +description = "this crate implements the kan model using the concision framework" +name = "concision-kan" + +authors.workspace = true +categories.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = false +features = ["full"] +rustc-args = [ + "--cfg", + "docsrs", +] + +[lib] +crate-type = [ + "cdylib", + "rlib", +] +bench = false +doc = true +doctest = true +test = true + +[dependencies] +# sdk +concision = { features = ["neural"], workspace = true } +# error +anyhow = { optional = true, workspace = true } +# mathematics +approx = { optional = true, workspace = true } +ndarray = { workspace = true } +num-traits = { workspace = true } +# logging +tracing = { optional = true, workspace = true } + +[features] +default = [ + "std", +] + +full = [ + "default", + "rand", + "serde", + "tracing" +] + +# ************* [FF:Environments] ************* +std = [ + "concision/std", + "ndarray/std", + "num-traits/std", + "tracing/std", +] + +wasi = [ + "concision/wasi", +] + +wasm = [ + "concision/wasm", +] + +# ************* [FF:Dependencies] ************* +anyhow = [ + "dep:anyhow", + "concision/anyhow", +] +approx = [ + "dep:approx", + "concision/approx", + "ndarray/approx", +] + +blas = [ + "concision/blas", + "ndarray/blas" +] + +rand = [ + "concision/rand", +] + +rayon = [ + "concision/rayon", + "ndarray/rayon" +] + +serde = [ + "concision/serde", +] + +tracing = [ + "dep:tracing", + "concision/tracing", +] diff --git a/models/kan/build.rs b/models/kan/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/models/kan/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs new file mode 100644 index 00000000..a94e4319 --- /dev/null +++ b/models/kan/src/lib.rs @@ -0,0 +1,29 @@ +/* + appellation: concision-kan + authors: @FL03 +*/ +//! # `concision-kan` +//! +//! This library provides an implementation of the Kolmogorov–Arnold Networks (kan) model using +//! the [`concision`](https://docs.rs/concision) framework. +//! +//! ## References +//! +//! - [KAN: Kolmogorov–Arnold Networks](https://arxiv.org/html/2404.19756v1) +//! +#![crate_name = "concision_kan"] +#![crate_type = "lib"] +#![cfg_attr(not(feature = "std"), no_std)] +#![allow(clippy::module_inception)] + +extern crate concision as cnc; + +#[doc(inline)] +pub use self::model::*; + +pub mod model; + +pub mod prelude { + #[doc(inline)] + pub use super::model::*; +} diff --git a/models/kan/src/model.rs b/models/kan/src/model.rs new file mode 100644 index 00000000..00479737 --- /dev/null +++ b/models/kan/src/model.rs @@ -0,0 +1,118 @@ +/* + appellation: model + authors: @FL03 +*/ + +use cnc::nn::{Model, ModelFeatures, ModelParams, StandardModelConfig}; +#[cfg(feature = "rand")] +use cnc::rand_distr; + +use num_traits::{Float, FromPrimitive}; + +#[derive(Clone, Debug)] +pub struct KanModel { + pub config: StandardModelConfig, + pub features: ModelFeatures, + pub params: ModelParams, +} + +impl KanModel +where + T: Float + FromPrimitive, +{ + pub fn new(config: StandardModelConfig, features: ModelFeatures) -> Self + where + T: Clone + Default, + { + let params = ModelParams::default(features); + KanModel { + config, + features, + params, + } + } + #[cfg(feature = "rand")] + pub fn init(self) -> Self + where + rand_distr::StandardNormal: rand_distr::Distribution, + { + let params = ModelParams::glorot_normal(self.features()); + KanModel { params, ..self } + } + /// returns a reference to the model configuration + pub const fn config(&self) -> &StandardModelConfig { + &self.config + } + /// returns a mutable reference to the model configuration + pub const fn config_mut(&mut self) -> &mut StandardModelConfig { + &mut self.config + } + /// returns the model features + pub const fn features(&self) -> ModelFeatures { + self.features + } + /// returns a mutable reference to the model features + pub const fn features_mut(&mut self) -> &mut ModelFeatures { + &mut self.features + } + /// returns a reference to the model parameters + pub const fn params(&self) -> &ModelParams { + &self.params + } + /// returns a mutable reference to the model parameters + pub const fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params + } + /// set the current configuration and return a mutable reference to the model + pub fn set_config(&mut self, config: StandardModelConfig) -> &mut Self { + self.config = config; + self + } + /// set the current features and return a mutable reference to the model + pub fn set_features(&mut self, features: ModelFeatures) -> &mut Self { + self.features = features; + self + } + /// set the current parameters and return a mutable reference to the model + pub fn set_params(&mut self, params: ModelParams) -> &mut Self { + self.params = params; + self + } + /// consumes the current instance to create another with the given configuration + pub fn with_config(self, config: StandardModelConfig) -> Self { + Self { config, ..self } + } + /// consumes the current instance to create another with the given features + pub fn with_features(self, features: ModelFeatures) -> Self { + Self { features, ..self } + } + /// consumes the current instance to create another with the given parameters + pub fn with_params(self, params: ModelParams) -> Self { + Self { params, ..self } + } +} + +impl Model for KanModel { + type Config = StandardModelConfig; + type Layout = ModelFeatures; + + fn config(&self) -> &StandardModelConfig { + &self.config + } + + fn config_mut(&mut self) -> &mut StandardModelConfig { + &mut self.config + } + + fn layout(&self) -> ModelFeatures { + self.features + } + + fn params(&self) -> &ModelParams { + &self.params + } + + fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params + } +} diff --git a/models/s4/Cargo.toml b/models/s4/Cargo.toml new file mode 100644 index 00000000..2fc0b3c1 --- /dev/null +++ b/models/s4/Cargo.toml @@ -0,0 +1,107 @@ +[package] +build = "build.rs" +description = "this crate implements the s4 model using the concision framework" +name = "concision-s4" + +authors.workspace = true +categories.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = false +features = ["full"] +rustc-args = [ + "--cfg", + "docsrs", +] + +[lib] +crate-type = [ + "cdylib", + "rlib", +] +bench = false +doc = true +doctest = true +test = true + +[dependencies] +# sdk +concision = { features = ["neural"], workspace = true } +# error +anyhow = { optional = true, workspace = true } +# mathematics +approx = { optional = true, workspace = true } +ndarray = { workspace = true } +num-traits = { workspace = true } +# logging +tracing = { optional = true, workspace = true } + +[features] +default = [ + "std", +] + +full = [ + "default", + "rand", + "serde", + "tracing" +] + +# ************* [FF:Environments] ************* +std = [ + "concision/std", + "ndarray/std", + "num-traits/std", + "tracing/std", +] + +wasi = [ + "concision/wasi", +] + +wasm = [ + "concision/wasm", +] + +# ************* [FF:Dependencies] ************* +anyhow = [ + "dep:anyhow", + "concision/anyhow", +] +approx = [ + "dep:approx", + "concision/approx", + "ndarray/approx", +] + +blas = [ + "concision/blas", + "ndarray/blas" +] + +rand = [ + "concision/rand", +] + +rayon = [ + "concision/rayon", + "ndarray/rayon" +] + +serde = [ + "concision/serde", +] + +tracing = [ + "dep:tracing", + "concision/tracing", +] diff --git a/models/s4/build.rs b/models/s4/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/models/s4/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/models/s4/src/lib.rs b/models/s4/src/lib.rs new file mode 100644 index 00000000..36a7354c --- /dev/null +++ b/models/s4/src/lib.rs @@ -0,0 +1,29 @@ +/* + appellation: concision-s4 + authors: @FL03 +*/ +//! # `concision-s4` +//! +//! This library provides the sequential, structured state-space (S4) model implementation for the Concision framework. +//! +//! ## References +//! +//! - [Structured State Spaces for Sequence Modeling](https://arxiv.org/abs/2106.08084) +//! - [Efficiently Modeling Long Sequences with Structured State Spaces](https://arxiv.org/abs/2111.00396) +//! +#![crate_name = "concision_s4"] +#![crate_type = "lib"] +#![cfg_attr(not(feature = "std"), no_std)] +#![allow(clippy::module_inception)] + +extern crate concision as cnc; + +#[doc(inline)] +pub use self::model::*; + +pub mod model; + +pub mod prelude { + #[doc(inline)] + pub use super::model::*; +} diff --git a/models/s4/src/model.rs b/models/s4/src/model.rs new file mode 100644 index 00000000..6eb3f6b2 --- /dev/null +++ b/models/s4/src/model.rs @@ -0,0 +1,120 @@ +/* + appellation: model + authors: @FL03 +*/ + +use cnc::nn::{Model, ModelFeatures, ModelParams, StandardModelConfig}; +#[cfg(feature = "rand")] +use cnc::rand_distr; + +use num_traits::{Float, FromPrimitive}; + +#[derive(Clone, Debug)] +pub struct S4Model { + pub config: StandardModelConfig, + pub features: ModelFeatures, + pub params: ModelParams, +} + +impl S4Model +where + T: Float + FromPrimitive, +{ + pub fn new(config: StandardModelConfig, features: ModelFeatures) -> Self + where + T: Clone + Default, + { + let params = ModelParams::default(features); + S4Model { + config, + features, + params, + } + } + #[cfg(feature = "rand")] + pub fn init(self) -> Self + where + T: Float + FromPrimitive, + rand_distr::StandardNormal: rand_distr::Distribution, + { + let params = ModelParams::glorot_normal(self.features()); + S4Model { params, ..self } + } + /// returns a reference to the model configuration + pub const fn config(&self) -> &StandardModelConfig { + &self.config + } + /// returns a mutable reference to the model configuration + pub const fn config_mut(&mut self) -> &mut StandardModelConfig { + &mut self.config + } + /// returns the model features + pub const fn features(&self) -> ModelFeatures { + self.features + } + /// returns a mutable reference to the model features + pub const fn features_mut(&mut self) -> &mut ModelFeatures { + &mut self.features + } + /// returns a reference to the model parameters + pub const fn params(&self) -> &ModelParams { + &self.params + } + /// returns a mutable reference to the model parameters + pub const fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params + } + /// set the current configuration and return a mutable reference to the model + pub fn set_config(&mut self, config: StandardModelConfig) -> &mut Self { + self.config = config; + self + } + /// set the current features and return a mutable reference to the model + pub fn set_features(&mut self, features: ModelFeatures) -> &mut Self { + self.features = features; + self + } + /// set the current parameters and return a mutable reference to the model + pub fn set_params(&mut self, params: ModelParams) -> &mut Self { + self.params = params; + self + } + /// consumes the current instance to create another with the given configuration + pub fn with_config(self, config: StandardModelConfig) -> Self { + Self { config, ..self } + } + /// consumes the current instance to create another with the given features + pub fn with_features(self, features: ModelFeatures) -> Self { + Self { features, ..self } + } + /// consumes the current instance to create another with the given parameters + pub fn with_params(self, params: ModelParams) -> Self { + Self { params, ..self } + } +} + +impl Model for S4Model { + type Config = StandardModelConfig; + + type Layout = ModelFeatures; + + fn config(&self) -> &StandardModelConfig { + &self.config + } + + fn config_mut(&mut self) -> &mut StandardModelConfig { + &mut self.config + } + + fn layout(&self) -> ModelFeatures { + self.features + } + + fn params(&self) -> &ModelParams { + &self.params + } + + fn params_mut(&mut self) -> &mut ModelParams { + &mut self.params + } +} diff --git a/models/transformer/src/model.rs b/models/transformer/src/model.rs index 1855b7e6..a19265a0 100644 --- a/models/transformer/src/model.rs +++ b/models/transformer/src/model.rs @@ -3,15 +3,16 @@ Contrib: @FL03 */ -#[cfg(feature = "rand")] -use cnc::init::rand_distr; use cnc::nn::{Model, ModelFeatures, ModelParams, NeuralError, StandardModelConfig, Train}; +#[cfg(feature = "rand")] +use cnc::rand_distr; use cnc::{Forward, Norm, Params, ReLU, Sigmoid}; use ndarray::prelude::*; use ndarray::{Data, ScalarOperand}; use num_traits::{Float, FromPrimitive, NumAssign}; +#[derive(Clone, Debug)] pub struct TransformerModel { pub config: StandardModelConfig, pub features: ModelFeatures, @@ -36,34 +37,64 @@ impl TransformerModel { T: Float + FromPrimitive, rand_distr::StandardNormal: rand_distr::Distribution, { - let params = ModelParams::glorot_normal(self.features); + let params = ModelParams::glorot_normal(self.features()); TransformerModel { params, ..self } } - + /// returns a reference to the model configuration pub const fn config(&self) -> &StandardModelConfig { &self.config } - - pub fn config_mut(&mut self) -> &mut StandardModelConfig { + /// returns a mutable reference to the model configuration + pub const fn config_mut(&mut self) -> &mut StandardModelConfig { &mut self.config } - + /// returns the model features pub const fn features(&self) -> ModelFeatures { self.features } - + /// returns a mutable reference to the model features + pub const fn features_mut(&mut self) -> &mut ModelFeatures { + &mut self.features + } + /// returns a reference to the model parameters pub const fn params(&self) -> &ModelParams { &self.params } - - pub fn params_mut(&mut self) -> &mut ModelParams { + /// returns a mutable reference to the model parameters + pub const fn params_mut(&mut self) -> &mut ModelParams { &mut self.params } + /// set the current configuration and return a mutable reference to the model + pub fn set_config(&mut self, config: StandardModelConfig) -> &mut Self { + self.config = config; + self + } + /// set the current features and return a mutable reference to the model + pub fn set_features(&mut self, features: ModelFeatures) -> &mut Self { + self.features = features; + self + } + /// set the current parameters and return a mutable reference to the model + pub fn set_params(&mut self, params: ModelParams) -> &mut Self { + self.params = params; + self + } + /// consumes the current instance to create another with the given configuration + pub fn with_config(self, config: StandardModelConfig) -> Self { + Self { config, ..self } + } + /// consumes the current instance to create another with the given features + pub fn with_features(self, features: ModelFeatures) -> Self { + Self { features, ..self } + } + /// consumes the current instance to create another with the given parameters + pub fn with_params(self, params: ModelParams) -> Self { + Self { params, ..self } + } } impl Model for TransformerModel { type Config = StandardModelConfig; - type Layout = ModelFeatures; fn config(&self) -> &StandardModelConfig { diff --git a/neural/Cargo.toml b/neural/Cargo.toml index 2ba7ef00..3e86e3c5 100644 --- a/neural/Cargo.toml +++ b/neural/Cargo.toml @@ -112,6 +112,13 @@ wasm = [ ] # ************* [FF:Dependencies] ************* +anyhow = [ + "dep:anyhow", + "concision-core/anyhow", + "concision-data/anyhow", + "scsys/anyhow", +] + approx = [ "concision-core/approx", "concision-data/approx", diff --git a/neural/src/error.rs b/neural/src/error.rs index fedda6c2..7aafb7c8 100644 --- a/neural/src/error.rs +++ b/neural/src/error.rs @@ -14,14 +14,29 @@ pub enum NeuralError { InvalidInputShape, #[error("Invalid Output Shape")] InvalidOutputShape, + #[cfg(feature = "std")] #[error("Parameter Error")] ParameterError(String), #[error("Training Failed")] TrainingFailed, #[error(transparent)] TrainingError(#[from] TrainingError), + #[cfg(feature = "anyhow")] + #[error(transparent)] + AnyError(#[from] anyhow::Error), + #[cfg(feature = "std")] + #[error(transparent)] + BoxError(#[from] Box), #[error(transparent)] CoreError(#[from] concision_core::error::Error), + #[error(transparent)] + FmtError(#[from] core::fmt::Error), + #[cfg(feature = "std")] + #[error(transparent)] + IOError(#[from] std::io::Error), + #[cfg(feature = "std")] + #[error("Unknown Error: {0}")] + UnknownError(String), } #[derive(Debug, scsys_derive::VariantConstructors, thiserror::Error)] diff --git a/neural/src/layers/attention/mod.rs b/neural/src/layers/attention/mod.rs index 677721ef..2419ce04 100644 --- a/neural/src/layers/attention/mod.rs +++ b/neural/src/layers/attention/mod.rs @@ -19,10 +19,10 @@ //! multiple times with different learned linear projections, and then concatenating the //! results. //! - **FFT Attention**: This is a more advanced attention mechanism that uses the Fast Fourier -//! Transform (FFT) to compute the attention scores more efficiently. It is particularly +//! Transform (FFT) to compute the attention scores more efficiently. It is particularly //! useful for long sequences where the standard attention mechanism can be computationally //! expensive. -//! +//! #[doc(inline)] pub use self::{multi_head::MultiHeadAttention, qkv::*, scaled::ScaledDotProductAttention}; diff --git a/neural/src/layers/attention/scaled.rs b/neural/src/layers/attention/scaled.rs index 2c5e0fd1..6a3b95cc 100644 --- a/neural/src/layers/attention/scaled.rs +++ b/neural/src/layers/attention/scaled.rs @@ -118,7 +118,6 @@ where let attention_weights = scaled_scores.mapv(|x| x.exp()) / scaled_scores.sum(); // Compute the final attention output by multiplying the attention weights with the value vectors - attention_weights.dot(value) } diff --git a/neural/src/layers/layer.rs b/neural/src/layers/layer.rs index 09738d17..9af362f5 100644 --- a/neural/src/layers/layer.rs +++ b/neural/src/layers/layer.rs @@ -2,8 +2,8 @@ Appellation: layer Contrib: @FL03 */ -use super::Layer; -use cnc::{Activate, ActivateGradient, Forward, ParamsBase}; +use super::{Activate, ActivateGradient, Layer}; +use cnc::{Forward, ParamsBase}; use ndarray::{Dimension, Ix2, RawData}; pub type LayerDyn = LayerBase + 'static>, S, D>; @@ -162,7 +162,7 @@ where type Input = F::Input; type Delta = F::Delta; - fn activate_gradient(&self, inputs: U) -> F::Delta { + fn activate_gradient(&self, inputs: F::Input) -> F::Delta { self.rho().activate_gradient(inputs) } } diff --git a/neural/src/layers/mod.rs b/neural/src/layers/mod.rs index 4cf7a808..99f9868f 100644 --- a/neural/src/layers/mod.rs +++ b/neural/src/layers/mod.rs @@ -29,7 +29,7 @@ pub trait Activate { /// Applies the activation function to the input tensor. fn activate(&self, input: T) -> Self::Output; } -/// The [`ActivateGradient`] trait extends the [`Activate`] trait to include a method for +/// The [`ActivateGradient`] trait extends the [`Activate`] trait to include a method for /// computing the gradient of the activation function. pub trait ActivateGradient: Activate { type Input; diff --git a/neural/src/lib.rs b/neural/src/lib.rs index f24cff65..53903321 100644 --- a/neural/src/lib.rs +++ b/neural/src/lib.rs @@ -2,15 +2,17 @@ Appellation: concision-neural Contrib: @FL03 */ +//! # concision-neural (cnc::neural) +//! //! The neural network abstractions used to create and train models. //! //! ## Features //! -//! - [Model]: A trait for defining a neural network model. -//! - [ModelParams]: A structure for storing the parameters of a neural network model. -//! - [StandardModelConfig]: A standard configuration for the models -//! - [Predict]: A trait extending the basic [Forward](cnc::Forward) pass -//! - [Train]: A trait for training a neural network model. +//! - [`Model`]: A trait for defining a neural network model. +//! - [`ModelParams`]: A structure for storing the parameters of a neural network model. +//! - [`StandardModelConfig`]: A standard configuration for the models +//! - [`Predict`]: A trait extending the basic [`Forward`](cnc::Forward) pass +//! - [`Train`]: A trait for training a neural network model. //! //! ### _Work in Progress_ //! diff --git a/neural/src/model/impls/impl_model_params.rs b/neural/src/model/impls/impl_model_params.rs new file mode 100644 index 00000000..2eaf8a48 --- /dev/null +++ b/neural/src/model/impls/impl_model_params.rs @@ -0,0 +1,93 @@ +/* + appellation: impl_model_params + authors: @FL03 +*/ +use crate::model::{ModelFeatures, ModelParamsBase}; + +use cnc::params::ParamsBase; +use ndarray::{DataOwned, Dimension, RawData}; +use num_traits::{One, Zero}; + +impl ModelParamsBase +where + S: RawData, +{ + /// create a new instance of the model; + /// all parameters are initialized to their defaults (i.e., zero) + pub fn default(features: ModelFeatures) -> Self + where + A: Clone + Default, + S: DataOwned, + { + let input = ParamsBase::default(features.dim_input()); + let hidden = (0..features.layers()) + .map(|_| ParamsBase::default(features.dim_hidden())) + .collect::>(); + let output = ParamsBase::default(features.dim_output()); + Self::new(input, hidden, output) + } + /// create a new instance of the model; + /// all parameters are initialized to zero + pub fn ones(features: ModelFeatures) -> Self + where + A: Clone + One, + S: DataOwned, + { + let input = ParamsBase::ones(features.dim_input()); + let hidden = (0..features.layers()) + .map(|_| ParamsBase::ones(features.dim_hidden())) + .collect::>(); + let output = ParamsBase::ones(features.dim_output()); + Self::new(input, hidden, output) + } + /// create a new instance of the model; + /// all parameters are initialized to zero + pub fn zeros(features: ModelFeatures) -> Self + where + A: Clone + Zero, + S: DataOwned, + { + let input = ParamsBase::zeros(features.dim_input()); + let hidden = (0..features.layers()) + .map(|_| ParamsBase::zeros(features.dim_hidden())) + .collect::>(); + let output = ParamsBase::zeros(features.dim_output()); + Self::new(input, hidden, output) + } +} + +impl core::ops::Index for ModelParamsBase +where + A: Clone, + D: Dimension, + S: ndarray::Data, +{ + type Output = ParamsBase; + + fn index(&self, index: usize) -> &Self::Output { + if index == 0 { + &self.input + } else if index == self.count_hidden() + 1 { + &self.output + } else { + &self.hidden[index - 1] + } + } +} + +impl core::ops::IndexMut for ModelParamsBase +where + A: Clone, + D: Dimension, + S: ndarray::Data, +{ + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index == 0 { + &mut self.input + } else if index == self.count_hidden() + 1 { + &mut self.output + } else { + &mut self.hidden[index - 1] + } + } +} diff --git a/neural/src/model/impls/impl_model_params_rand.rs b/neural/src/model/impls/impl_model_params_rand.rs new file mode 100644 index 00000000..b63e76ec --- /dev/null +++ b/neural/src/model/impls/impl_model_params_rand.rs @@ -0,0 +1,59 @@ +/* + appellation: impl_model_params_rand + authors: @FL03 +*/ + +use crate::model::{ModelFeatures, ModelParamsBase}; + +use cnc::init::{self, Initialize}; +use cnc::params::ParamsBase; +use cnc::rand_distr; + +use ndarray::DataOwned; +use num_traits::{Float, FromPrimitive}; +use rand_distr::uniform::{SampleUniform, Uniform}; +use rand_distr::{Distribution, StandardNormal}; + +impl ModelParamsBase +where + S: DataOwned, +{ + /// returns a new instance of the model initialized with the given features and random + /// distribution + pub fn init_rand(features: ModelFeatures, distr: G) -> Self + where + G: Fn((usize, usize)) -> Ds, + Ds: Clone + Distribution, + S: DataOwned, + { + let input = ParamsBase::rand(features.dim_input(), distr(features.dim_input())); + let hidden = (0..features.layers()) + .map(|_| ParamsBase::rand(features.dim_hidden(), distr(features.dim_hidden()))) + .collect::>(); + + let output = ParamsBase::rand(features.dim_output(), distr(features.dim_output())); + + Self::new(input, hidden, output) + } + /// initialize the model parameters using a glorot normal distribution + pub fn glorot_normal(features: ModelFeatures) -> Self + where + A: Float + FromPrimitive, + StandardNormal: Distribution, + { + Self::init_rand(features, |(rows, cols)| { + cnc::init::XavierNormal::new(rows, cols) + }) + } + /// initialize the model parameters using a glorot uniform distribution + pub fn glorot_uniform(features: ModelFeatures) -> Self + where + A: Clone + Float + FromPrimitive + SampleUniform, + ::Sampler: Clone, + Uniform: Distribution, + { + Self::init_rand(features, |(rows, cols)| { + init::XavierUniform::new(rows, cols).expect("failed to create distribution") + }) + } +} diff --git a/neural/src/model/mod.rs b/neural/src/model/mod.rs index 7e4f505d..7f1bf0ad 100644 --- a/neural/src/model/mod.rs +++ b/neural/src/model/mod.rs @@ -5,21 +5,30 @@ //! This module provides the scaffolding for creating models and layers in a neural network. #[doc(inline)] -pub use self::{ - config::StandardModelConfig, layout::*, model_params::ModelParams, trainer::Trainer, -}; +pub use self::{config::StandardModelConfig, layout::*, model_params::*, trainer::Trainer}; pub mod config; pub mod layout; pub mod model_params; pub mod trainer; +mod impls { + pub mod impl_model_params; + #[cfg(feature = "rand")] + pub mod impl_model_params_rand; +} + pub(crate) mod prelude { - pub use super::Model; + #[doc(inline)] pub use super::config::*; + #[doc(inline)] pub use super::layout::*; + #[doc(inline)] pub use super::model_params::*; + #[doc(inline)] pub use super::trainer::*; + #[doc(inline)] + pub use super::{Model, ModelExt}; } use crate::{NetworkConfig, Predict, Train}; @@ -79,20 +88,20 @@ pub trait ModelExt: Model where Self::Layout: ModelLayout, { - /// replaces the current configuration and returns the old one; + /// [`replace`](core::mem::replace) the current configuration and returns the old one; fn replace_config(&mut self, config: Self::Config) -> Self::Config { core::mem::replace(self.config_mut(), config) } - /// replaces the current model parameters and returns the old one; + /// [`replace`](core::mem::replace) the current model parameters and returns the old one fn replace_params(&mut self, params: ModelParams) -> ModelParams { core::mem::replace(self.params_mut(), params) } - /// overrides the current configuration and returns a mutable reference to the model; + /// overrides the current configuration and returns a mutable reference to the model fn set_config(&mut self, config: Self::Config) -> &mut Self { *self.config_mut() = config; self } - /// overrides the current model parameters and returns a mutable reference to the model; + /// overrides the current model parameters and returns a mutable reference to the model fn set_params(&mut self, params: ModelParams) -> &mut Self { *self.params_mut() = params; self diff --git a/neural/src/model/model_params.rs b/neural/src/model/model_params.rs index 17722286..70df7371 100644 --- a/neural/src/model/model_params.rs +++ b/neural/src/model/model_params.rs @@ -2,10 +2,8 @@ Appellation: store Contrib: @FL03 */ -use super::layout::ModelFeatures; use cnc::params::ParamsBase; -use ndarray::{Data, DataOwned, Dimension, Ix2, RawData}; -use num_traits::{One, Zero}; +use ndarray::{Data, Dimension, Ix2, RawData}; pub type ModelParams = ModelParamsBase, D>; @@ -22,11 +20,11 @@ where S: RawData, { /// the input layer of the model - pub input: ParamsBase, + pub(crate) input: ParamsBase, /// a sequential stack of params for the model's hidden layers - pub hidden: Vec>, + pub(crate) hidden: Vec>, /// the output layer of the model - pub output: ParamsBase, + pub(crate) output: ParamsBase, } impl ModelParamsBase @@ -34,7 +32,9 @@ where D: Dimension, S: RawData, { - pub fn new( + /// returns a new instance of the [`ModelParamsBase`] with the specified input, hidden, and + /// output layers. + pub const fn new( input: ParamsBase, hidden: Vec>, output: ParamsBase, @@ -45,17 +45,12 @@ where output, } } - /// returns true if the stack is shallow - pub fn is_shallow(&self) -> bool { - self.hidden.is_empty() || self.hidden.len() == 1 - } /// returns an immutable reference to the input layer of the model pub const fn input(&self) -> &ParamsBase { &self.input } /// returns a mutable reference to the input layer of the model - #[inline] - pub fn input_mut(&mut self) -> &mut ParamsBase { + pub const fn input_mut(&mut self) -> &mut ParamsBase { &mut self.input } /// returns an immutable reference to the hidden layers of the model @@ -68,8 +63,7 @@ where self.hidden.as_slice() } /// returns a mutable reference to the hidden layers of the model - #[inline] - pub fn hidden_mut(&mut self) -> &mut Vec> { + pub const fn hidden_mut(&mut self) -> &mut Vec> { &mut self.hidden } /// returns an immutable reference to the output layer of the model @@ -77,30 +71,52 @@ where &self.output } /// returns a mutable reference to the output layer of the model - #[inline] - pub fn output_mut(&mut self) -> &mut ParamsBase { + pub const fn output_mut(&mut self) -> &mut ParamsBase { &mut self.output } /// set the input layer of the model - pub fn set_input(&mut self, input: ParamsBase) { + #[inline] + pub fn set_input(&mut self, input: ParamsBase) -> &mut Self { *self.input_mut() = input; + self } /// set the hidden layers of the model - pub fn set_hidden(&mut self, iter: I) - where - I: IntoIterator>, - { - *self.hidden_mut() = Vec::from_iter(iter); + #[inline] + pub fn set_hidden(&mut self, hidden: Vec>) -> &mut Self { + *self.hidden_mut() = hidden; + self + } + /// set the layer at the specified index in the hidden layers of the model + /// + /// ## Panics + /// + /// Panics if the index is out of bounds or if the dimension of the provided layer is + /// inconsistent with the others in the stack. + #[inline] + pub fn set_hidden_layer(&mut self, idx: usize, layer: ParamsBase) -> &mut Self { + if layer.dim() != self.dim_hidden() { + panic!( + "the dimension of the layer ({:?}) does not match the dimension of the hidden layers ({:?})", + layer.dim(), + self.dim_hidden() + ); + } + self.hidden_mut()[idx] = layer; + self } /// set the output layer of the model - pub fn set_output(&mut self, output: ParamsBase) { - self.output = output; + #[inline] + pub fn set_output(&mut self, output: ParamsBase) -> &mut Self { + *self.output_mut() = output; + self } /// consumes the current instance and returns another with the specified input layer + #[inline] pub fn with_input(self, input: ParamsBase) -> Self { Self { input, ..self } } /// consumes the current instance and returns another with the specified hidden layers + #[inline] pub fn with_hidden(self, iter: I) -> Self where I: IntoIterator>, @@ -111,129 +127,97 @@ where } } /// consumes the current instance and returns another with the specified output layer + #[inline] pub fn with_output(self, output: ParamsBase) -> Self { Self { output, ..self } } /// returns the dimension of the input layer + #[inline] pub fn dim_input(&self) -> ::Pattern { self.input().dim() } /// returns the dimension of the hidden layers + #[inline] pub fn dim_hidden(&self) -> ::Pattern { - assert!(self.hidden.iter().all(|p| p.dim() == self.hidden[0].dim())); + // verify that all hidden layers have the same dimension + assert!( + self.hidden() + .iter() + .all(|p| p.dim() == self.hidden()[0].dim()) + ); + // use the first hidden layer's dimension as the representative + // dimension for all hidden layers self.hidden()[0].dim() } /// returns the dimension of the output layer + #[inline] pub fn dim_output(&self) -> ::Pattern { - self.output.dim() + self.output().dim() + } + /// returns the hidden layer associated with the given index + #[inline] + pub fn get_hidden_layer(&self, idx: I) -> Option<&I::Output> + where + I: core::slice::SliceIndex<[ParamsBase]>, + { + self.hidden().get(idx) + } + /// returns a mutable reference to the hidden layer associated with the given index + #[inline] + pub fn get_hidden_layer_mut(&mut self, idx: I) -> Option<&mut I::Output> + where + I: core::slice::SliceIndex<[ParamsBase]>, + { + self.hidden_mut().get_mut(idx) } /// sequentially forwards the input through the model without any activations or other /// complexities in-between. not overly usefuly, but it is here for completeness + #[inline] pub fn forward(&self, input: &X) -> cnc::Result where A: Clone, S: Data, ParamsBase: cnc::Forward + cnc::Forward, { + // forward the input through the input layer let mut output = self.input().forward(input)?; + // forward the input through each of the hidden layers for layer in self.hidden() { output = layer.forward(&output)?; } + // finally, forward the output through the output layer self.output().forward(&output) } -} - -impl ModelParamsBase -where - S: RawData, -{ - /// create a new instance of the model; - /// all parameters are initialized to their defaults (i.e., zero) - pub fn default(features: ModelFeatures) -> Self - where - A: Clone + Default, - S: DataOwned, - { - let input = ParamsBase::default(features.dim_input()); - let hidden = (0..features.layers()) - .map(|_| ParamsBase::default(features.dim_hidden())) - .collect::>(); - let output = ParamsBase::default(features.dim_output()); - Self::new(input, hidden, output) - } - /// create a new instance of the model; - /// all parameters are initialized to zero - pub fn ones(features: ModelFeatures) -> Self - where - A: Clone + One, - S: DataOwned, - { - let input = ParamsBase::ones(features.dim_input()); - let hidden = (0..features.layers()) - .map(|_| ParamsBase::ones(features.dim_hidden())) - .collect::>(); - let output = ParamsBase::ones(features.dim_output()); - Self::new(input, hidden, output) + /// returns true if the stack is shallow; a neural network is considered to be _shallow_ if + /// it has at most one hidden layer (`n <= 1`). + #[inline] + pub fn is_shallow(&self) -> bool { + self.count_hidden() <= 1 || self.hidden().is_empty() } - /// create a new instance of the model; - /// all parameters are initialized to zero - pub fn zeros(features: ModelFeatures) -> Self - where - A: Clone + Zero, - S: DataOwned, - { - let input = ParamsBase::zeros(features.dim_input()); - let hidden = (0..features.layers()) - .map(|_| ParamsBase::zeros(features.dim_hidden())) - .collect::>(); - let output = ParamsBase::zeros(features.dim_output()); - Self::new(input, hidden, output) + /// returns true if the model stack of parameters is considered to be _deep_, meaning that + /// there the number of hidden layers is greater than one. + #[inline] + pub fn is_deep(&self) -> bool { + self.count_hidden() > 1 } - - #[cfg(feature = "rand")] - pub fn init_rand(features: ModelFeatures, distr: G) -> Self - where - G: Fn((usize, usize)) -> Ds, - Ds: Clone + cnc::init::rand_distr::Distribution, - S: DataOwned, - { - use cnc::init::Initialize; - let input = ParamsBase::rand(features.dim_input(), distr(features.dim_input())); - let hidden = (0..features.layers()) - .map(|_| ParamsBase::rand(features.dim_hidden(), distr(features.dim_hidden()))) - .collect::>(); - - let output = ParamsBase::rand(features.dim_output(), distr(features.dim_output())); - - Self::new(input, hidden, output) + /// returns the total number of hidden layers within the model + #[inline] + pub fn count_hidden(&self) -> usize { + self.hidden().len() } - /// initialize the model parameters using a glorot normal distribution - #[cfg(feature = "rand")] - pub fn glorot_normal(features: ModelFeatures) -> Self - where - S: DataOwned, - A: num_traits::Float + num_traits::FromPrimitive, - cnc::init::rand_distr::StandardNormal: cnc::init::rand_distr::Distribution, - { - Self::init_rand(features, |(rows, cols)| { - cnc::init::XavierNormal::new(rows, cols) - }) + /// returns the total number of layers within the model, including the input and output layers + #[inline] + pub fn len(&self) -> usize { + self.count_hidden() + 2 // +2 for input and output layers } - /// initialize the model parameters using a glorot uniform distribution - #[cfg(feature = "rand")] - pub fn glorot_uniform(features: ModelFeatures) -> Self - where - S: ndarray::DataOwned, - A: Clone - + num_traits::Float - + num_traits::FromPrimitive - + cnc::init::rand_distr::uniform::SampleUniform, - ::Sampler: Clone, - cnc::init::rand_distr::Uniform: cnc::init::rand_distr::Distribution, - { - Self::init_rand(features, |(rows, cols)| { - cnc::init::XavierUniform::new(rows, cols).expect("failed to create distribution") - }) + /// returns the total number parameters within the model, including the input and output layers + #[inline] + pub fn size(&self) -> usize { + let mut size = self.input().count_weight(); + for layer in self.hidden() { + size += layer.count_weight(); + } + size + self.output().count_weight() } } @@ -245,9 +229,9 @@ where { fn clone(&self) -> Self { Self { - input: self.input.clone(), - hidden: self.hidden.to_vec(), - output: self.output.clone(), + input: self.input().clone(), + hidden: self.hidden().to_vec(), + output: self.output().clone(), } } } @@ -281,39 +265,3 @@ where ) } } - -impl core::ops::Index for ModelParamsBase -where - A: Clone, - D: Dimension, - S: ndarray::Data, -{ - type Output = ParamsBase; - - fn index(&self, index: usize) -> &Self::Output { - if index == 0 { - &self.input - } else if index == self.hidden.len() + 1 { - &self.output - } else { - &self.hidden[index - 1] - } - } -} - -impl core::ops::IndexMut for ModelParamsBase -where - A: Clone, - D: Dimension, - S: ndarray::Data, -{ - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - if index == 0 { - &mut self.input - } else if index == self.hidden.len() + 1 { - &mut self.output - } else { - &mut self.hidden[index - 1] - } - } -} diff --git a/neural/tests/default.rs b/neural/tests/default.rs index 233a07af..62e1f28a 100644 --- a/neural/tests/default.rs +++ b/neural/tests/default.rs @@ -3,15 +3,14 @@ Contrib: FL03 */ -fn add(a: A, b: B) -> C -where - A: core::ops::Add, -{ - a + b -} - #[test] -fn compiles() { +fn lib_compiles() { + fn add(a: A, b: B) -> C + where + A: core::ops::Add, + { + a + b + } assert_eq!(add(10, 10), 20); assert_ne!(add(1, 1), 3); } diff --git a/neural/tests/simple/main.rs b/neural/tests/simple/main.rs index 1a10b878..c9bc62c2 100644 --- a/neural/tests/simple/main.rs +++ b/neural/tests/simple/main.rs @@ -1,8 +1,8 @@ extern crate concision_core as cnc; extern crate concision_neural as neural; -use concision_neural::ModelFeatures; use concision_neural::model::{Model, StandardModelConfig}; +use concision_neural::{ModelFeatures, NeuralResult}; use ndarray::prelude::*; use model::SimpleModel; @@ -28,7 +28,7 @@ fn test_standard_model_config() { } #[test] -fn test_simple_model() -> anyhow::Result<()> { +fn test_simple_model() -> NeuralResult<()> { let mut config = StandardModelConfig::new() .with_epochs(1000) .with_batch_size(32); diff --git a/utils/src/stats/summary.rs b/utils/src/stats/summary.rs index 46e71b87..2e148653 100644 --- a/utils/src/stats/summary.rs +++ b/utils/src/stats/summary.rs @@ -17,7 +17,7 @@ where type Item; type Output; /// returns the number of elements in the iterator - fn len(&self) -> usize; + fn len(&self) -> usize; /// returns the number of elements in the iterator as an [`Item`](Self::Item) type. fn product(&self) -> Self::Output; /// returns the sum of the iterator @@ -26,7 +26,7 @@ where fn std(&self) -> Self::Output; /// returns the variance of the iterator fn var(&self) -> Self::Output; - + /// returns the number of elements in the iterator as an [`Item`](Self::Item) type. fn elems(&self) -> Self::Item { Self::Item::from_usize(self.len()).unwrap() diff --git a/utils/src/utils/norm.rs b/utils/src/utils/norm.rs index bb464a2e..5eb28896 100644 --- a/utils/src/utils/norm.rs +++ b/utils/src/utils/norm.rs @@ -30,6 +30,6 @@ where let mean = x.mean_axis(axis).unwrap(); let var = x.var_axis(axis, A::zero()); let inv_std = var.mapv(|v| (v + eps).recip().sqrt()); - + (x - &mean) * &inv_std }