From a5776da0e4624bff80bda60afef88e12ed85d779 Mon Sep 17 00:00:00 2001 From: Tony Arcieri Date: Fri, 22 Jan 2021 10:23:43 -0800 Subject: [PATCH] [WIP] simd-buffers Implements the following SIMD types, as proposed in RustCrypto/traits#444: - `U128` (portable) - `U256` (x86/x86_64 only) - `U128x8` (portable) These types are largely "storage only" and don't implement arithmetic (if we needed that, `stdsimd`/`packed_simd` would be a better choice) The implementation *does* expose optimized XOR intrinsics, however, which seems to be the main thing useful in a portable cryptographic context, at least as far as our current usages of SIMD go. The `x86` backend exposes unsafe `target_feature(enable = "...")` functions as part of its API, intended to be used/inlined within SIMD backends for particular algorithms. --- .github/workflows/simd-buffers.yml | 55 +++++ Cargo.lock | 13 ++ Cargo.toml | 3 +- simd-buffers/CHANGELOG.md | 5 + simd-buffers/Cargo.toml | 20 ++ simd-buffers/LICENSE-APACHE | 201 +++++++++++++++++ simd-buffers/LICENSE-MIT | 25 +++ simd-buffers/README.md | 40 ++++ simd-buffers/src/lib.rs | 85 ++++++++ simd-buffers/src/portable.rs | 170 +++++++++++++++ simd-buffers/src/temp_traits.rs | 16 ++ simd-buffers/src/x86.rs | 337 +++++++++++++++++++++++++++++ simd-buffers/tests/lib.rs | 1 + 13 files changed, 970 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/simd-buffers.yml create mode 100644 simd-buffers/CHANGELOG.md create mode 100644 simd-buffers/Cargo.toml create mode 100644 simd-buffers/LICENSE-APACHE create mode 100644 simd-buffers/LICENSE-MIT create mode 100644 simd-buffers/README.md create mode 100644 simd-buffers/src/lib.rs create mode 100644 simd-buffers/src/portable.rs create mode 100644 simd-buffers/src/temp_traits.rs create mode 100644 simd-buffers/src/x86.rs create mode 100644 simd-buffers/tests/lib.rs diff --git a/.github/workflows/simd-buffers.yml b/.github/workflows/simd-buffers.yml new file mode 100644 index 00000000..42a4d97f --- /dev/null +++ b/.github/workflows/simd-buffers.yml @@ -0,0 +1,55 @@ +name: simd-buffers + +on: + pull_request: + paths: + - "simd-buffers/**" + - "Cargo.*" + push: + branches: master + +defaults: + run: + working-directory: simd-buffers + +env: + CARGO_INCREMENTAL: 0 + RUSTFLAGS: "-Dwarnings" + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - 1.47.0 # MSRV + - stable + target: + - thumbv7em-none-eabi + - wasm32-unknown-unknown + steps: + - uses: actions/checkout@v1 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + target: ${{ matrix.target }} + override: true + - run: cargo build --target ${{ matrix.target }} --release + + test: + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - 1.47.0 # MSRV + - stable + steps: + - uses: actions/checkout@v1 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + - run: cargo test --release + - run: cargo test --release --features force-soft diff --git a/Cargo.lock b/Cargo.lock index 991a70da..fa1813fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "collectable" version = "0.0.2" @@ -122,6 +128,13 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "simd-buffers" +version = "0.0.0" +dependencies = [ + "cfg-if", +] + [[package]] name = "syn" version = "1.0.60" diff --git a/Cargo.toml b/Cargo.toml index cec001a1..31a10ddd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,5 +12,6 @@ members = [ "der/derive", "hex-literal", "opaque-debug", - "pkcs8" + "pkcs8", + "simd-buffers" ] diff --git a/simd-buffers/CHANGELOG.md b/simd-buffers/CHANGELOG.md new file mode 100644 index 00000000..d6637e04 --- /dev/null +++ b/simd-buffers/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/simd-buffers/Cargo.toml b/simd-buffers/Cargo.toml new file mode 100644 index 00000000..bef91bb8 --- /dev/null +++ b/simd-buffers/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "simd-buffers" +version = "0.0.0" # Also update html_root_url in lib.rs when bumping this +description = """ +SIMD buffer types +""" +authors = ["RustCrypto Developers"] +license = "Apache-2.0 OR MIT" +edition = "2018" +documentation = "https://docs.rs/simd-buffers" +repository = "https://github.com/RustCrypto/utils/tree/master/simd-buffers" +categories = ["cryptography", "data-structures"] +keywords = ["simd"] +readme = "README.md" + +[dependencies] +cfg-if = "1" + +[features] +force-soft = [] diff --git a/simd-buffers/LICENSE-APACHE b/simd-buffers/LICENSE-APACHE new file mode 100644 index 00000000..78173fa2 --- /dev/null +++ b/simd-buffers/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/simd-buffers/LICENSE-MIT b/simd-buffers/LICENSE-MIT new file mode 100644 index 00000000..2726e14a --- /dev/null +++ b/simd-buffers/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2020 The RustCrypto Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/simd-buffers/README.md b/simd-buffers/README.md new file mode 100644 index 00000000..731df430 --- /dev/null +++ b/simd-buffers/README.md @@ -0,0 +1,40 @@ +# RustCrypto: SIMD buffers + +[![crate][crate-image]][crate-link] +[![Docs][docs-image]][docs-link] +![Apache2/MIT licensed][license-image] +![Rust Version][rustc-image] +[![Project Chat][chat-image]][chat-link] +[![Build Status][build-image]][build-link] + +SIMD buffer types. + +[Documentation][docs-link] + +## License + +Licensed under-buffers either of: + + * [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + * [MIT license](http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. + +[//]: # (badges) + +[crate-image]: https://img.shields.io/crates/v/simd-buffers.svg +[crate-link]: https://crates.io/crates/simd-buffers +[docs-image]: https://docs.rs/simd-buffers/badge.svg +[docs-link]: https://docs.rs/simd-buffers/ +[license-image]: https://img.shields.io/badge/license-Apache2.0/MIT-blue.svg +[rustc-image]: https://img.shields.io/badge/rustc-1.47+-blue.svg +[chat-image]: https://img.shields.io/badge/zulip-join_chat-blue.svg +[chat-link]: https://rustcrypto.zulipchat.com/#narrow/stream/260052-utils +[build-image]: https://github.com/RustCrypto/utils/workflows/simd-buffers/badge.svg?branch=master&event=push +[build-link]: https://github.com/RustCrypto/utils/actions?query=workflow:simd-buffers diff --git a/simd-buffers/src/lib.rs b/simd-buffers/src/lib.rs new file mode 100644 index 00000000..bb93e42d --- /dev/null +++ b/simd-buffers/src/lib.rs @@ -0,0 +1,85 @@ +//! SIMD buffer types. + +#![no_std] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc( + html_logo_url = "https://raw.githubusercontent.com/RustCrypto/meta/master/logo_small.png", + html_root_url = "https://docs.rs/simd-buffers/0.0.0" +)] +#![warn(rust_2018_idioms)] // TODO: missing_docs + +pub mod temp_traits; + +use cfg_if::cfg_if; +use core::{convert::TryFrom, fmt::Debug, ops, slice}; + +cfg_if! { + if #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), not(feature = "force-soft")))] { + mod x86; + pub use x86::{U128x8, U128, U256}; + } else { + mod portable; + pub use portable::{U128x8, U128}; + } +} + +/// Length error +pub struct LengthError; + +/// SIMD register trait. +pub trait SimdRegister: + Copy + + Clone + + Debug + + Default + + Sized + + ops::BitXor + + ops::BitXorAssign + + for<'a> TryFrom<&'a [u8], Error = LengthError> +{ + /// Byte representation (i.e. a byte array) + type Repr: Default + AsRef<[u8]> + AsMut<[u8]>; + + /// Return the zero value. + #[inline(always)] + fn zero() -> Self { + Self::default() + } + + /// Load a register from its (potentially unaligned) byte representation. + fn load(bytes: &Self::Repr) -> Self; + + /// Store a register into the given byte representation. + fn store(self, bytes: &mut Self::Repr); +} + +/// SIMD buffer trait +pub trait SimdBuffer: + Default + + Sized + + ops::BitXorAssign + + ops::Index + + ops::IndexMut + + for<'a> TryFrom<&'a [u8], Error = LengthError> +{ + /// SIMD register type used by this buffer. + type Register: SimdRegister; + + /// Borrow all registers in this buffer as a slice. + fn as_slice(&self) -> &[Self::Register]; + + /// Borrow all registers in this buffer as a mut slice. + fn as_mut_slice(&mut self) -> &mut [Self::Register]; + + /// Iterate over the registers. + #[inline(always)] + fn iter(&self) -> slice::Iter<'_, Self::Register> { + self.as_slice().iter() + } + + /// Iterate over the registers mutably. + #[inline(always)] + fn iter_mut(&mut self) -> slice::IterMut<'_, Self::Register> { + self.as_mut_slice().iter_mut() + } +} diff --git a/simd-buffers/src/portable.rs b/simd-buffers/src/portable.rs new file mode 100644 index 00000000..9a0d6308 --- /dev/null +++ b/simd-buffers/src/portable.rs @@ -0,0 +1,170 @@ +//! Portable "SIMD" buffers. +//! +//! These allow these buffer types to be used on platforms where there is +//! no native SIMD implementation for them to map to. + +use crate::{LengthError, SimdBuffer, SimdRegister}; +use core::{ + convert::TryFrom, + mem, + ops::{BitXor, BitXorAssign, Index, IndexMut}, +}; + +/// 128-bit SIMD integer +#[derive(Copy, Clone, Default, Debug)] +#[repr(transparent)] +pub struct U128(u128); + +impl SimdRegister for U128 { + type Repr = [u8; 16]; + + fn load(bytes: &[u8; 16]) -> Self { + Self::from(bytes) + } + + fn store(self, bytes: &mut [u8; 16]) { + bytes.copy_from_slice(&Self::Repr::from(self)) + } +} + +impl From for U128 { + #[inline(always)] + fn from(x: u128) -> U128 { + Self(x) + } +} + +impl From for u128 { + #[inline(always)] + fn from(x: U128) -> u128 { + x.0 + } +} + +impl From<[u8; 16]> for U128 { + #[inline(always)] + fn from(bytes: [u8; 16]) -> U128 { + Self(u128::from_ne_bytes(bytes)) + } +} + +impl From<&[u8; 16]> for U128 { + #[inline(always)] + fn from(bytes: &[u8; 16]) -> U128 { + Self(u128::from_ne_bytes(*bytes)) + } +} + +impl From for [u8; 16] { + #[inline(always)] + fn from(x: U128) -> [u8; 16] { + x.0.to_ne_bytes() + } +} + +impl BitXor for U128 { + type Output = Self; + + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0) + } +} + +impl BitXorAssign for U128 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 ^= rhs.0; + } +} + +impl TryFrom<&[u8]> for U128 { + type Error = LengthError; + + fn try_from(bytes: &[u8]) -> Result { + if bytes.len() <= 16 { + let mut buf = [0u8; 16]; + buf[..bytes.len()].copy_from_slice(bytes); + Ok(buf.into()) + } else { + Err(LengthError) + } + } +} + +/// 1024-bit SIMD buffer decomposed into 8 x [`U128`] values. +#[derive(Clone, Debug, Default)] +#[repr(align(128))] +pub struct U128x8([U128; 8]); + +impl TryFrom<&[u8]> for U128x8 { + type Error = LengthError; + + fn try_from(bytes: &[u8]) -> Result { + let mut chunks = bytes.chunks_exact(16); + let mut result: [U128; 8] = unsafe { mem::zeroed() }; + let mut i = 0; + + for chunk in &mut chunks { + if i == 8 { + return Err(LengthError); + } + + result[i] = U128::try_from(chunk)?; + i += 1; + } + + let rem = chunks.remainder(); + + if !rem.is_empty() { + if i < 8 { + result[i] = U128::try_from(rem)?; + i += 1; + } else { + return Err(LengthError); + } + } + + while i < 8 { + result[i] = U128::zero(); + i += 1; + } + + Ok(Self(result)) + } +} + +impl BitXorAssign for U128x8 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + for (x, y) in self.0.iter_mut().zip(&rhs.0) { + *x ^= *y + } + } +} + +impl Index for U128x8 { + type Output = U128; + + fn index(&self, idx: usize) -> &U128 { + self.0.index(idx) + } +} + +impl IndexMut for U128x8 { + fn index_mut(&mut self, idx: usize) -> &mut Self::Output { + self.0.index_mut(idx) + } +} + +impl SimdBuffer for U128x8 { + type Register = U128; + + fn as_slice(&self) -> &[U128] { + &self.0 + } + + fn as_mut_slice(&mut self) -> &mut [U128] { + &mut self.0 + } +} diff --git a/simd-buffers/src/temp_traits.rs b/simd-buffers/src/temp_traits.rs new file mode 100644 index 00000000..cba41f16 --- /dev/null +++ b/simd-buffers/src/temp_traits.rs @@ -0,0 +1,16 @@ +//! Traits that probably shouldn't live in this crate, but are here temporarily +//! because it's a convenient place for prototyping. + +use crate::SimdBuffer; + +pub trait BlockEncryptPar { + fn encrypt_par(&self, buffer: &mut B); +} + +pub trait StreamCipherPar { + fn try_apply_keystream_par(&mut self, buffer: &mut B); +} + +pub trait UniversalHashPar { + fn update_par(&mut self, blocks: &B); +} diff --git a/simd-buffers/src/x86.rs b/simd-buffers/src/x86.rs new file mode 100644 index 00000000..66b380b7 --- /dev/null +++ b/simd-buffers/src/x86.rs @@ -0,0 +1,337 @@ +//! x86 SIMD buffers + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{LengthError, SimdBuffer, SimdRegister}; +use core::{ + convert::TryFrom, + mem, + ops::{BitXor, BitXorAssign, Index, IndexMut}, +}; + +macro_rules! simd_int { + ( + $name:ident, + $repr:ty, + $inner:ty, + $zero:ident, + $loadu:ident, + $storeu:ident, + $xor:ident, + $doc:expr + ) => { + #[doc=$doc] + #[derive(Copy, Clone, Debug)] + #[repr(transparent)] + pub struct $name($inner); + + impl SimdRegister for $name { + type Repr = $repr; + + #[inline(always)] + fn load(bytes: &Self::Repr) -> Self { + Self(unsafe { $loadu(bytes.as_ptr() as *const $inner) }) + } + + #[inline(always)] + fn store(self, bytes: &mut Self::Repr) { + unsafe { $storeu(bytes.as_mut_ptr() as *mut $inner, self.0); } + } + } + + impl Default for $name { + #[inline(always)] + fn default() -> Self { + Self(unsafe { $zero() }) + } + } + + impl From<$inner> for $name { + #[inline(always)] + fn from(x: $inner) -> $name { + Self(x) + } + } + + impl From<$name> for $inner { + #[inline(always)] + fn from(x: $name) -> $inner { + x.0 + } + } + + impl From<$repr> for $name { + #[inline(always)] + fn from(bytes: $repr) -> $name { + Self::load(&bytes) + } + } + + impl From<&$repr> for $name { + #[inline(always)] + fn from(bytes: &$repr) -> $name { + Self::load(&bytes) + } + } + + impl From<$name> for $repr { + #[inline(always)] + fn from(x: $name) -> $repr { + let mut bytes: $repr = unsafe { mem::zeroed() }; + x.store(&mut bytes); + bytes + } + } + + impl BitXor for $name { + type Output = Self; + + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + // TODO(tarcieri): runtime CPU feature detection + unsafe { self.$xor(rhs) } + } + } + + impl BitXorAssign for $name { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } + } + }; +} + +simd_int!( + U128, + [u8; 16], + __m128i, + _mm_setzero_si128, + _mm_loadu_si128, + _mm_storeu_si128, + bitxor_sse2, + "128-bit SIMD integer" +); + +impl U128 { + #[target_feature(enable = "sse2")] + pub unsafe fn bitxor_sse2(self, rhs: Self) -> Self { + Self(_mm_xor_si128(self.0, rhs.0)) + } +} + +impl From for U128 { + #[inline(always)] + fn from(x: u128) -> U128 { + Self(unsafe { *(&x as *const _ as *const __m128i) }) + } +} + +impl From for u128 { + #[inline(always)] + fn from(x: U128) -> u128 { + unsafe { *(&x.0 as *const _ as *const u128) } + } +} + +impl TryFrom<&[u8]> for U128 { + type Error = LengthError; + + #[inline(always)] + fn try_from(bytes: &[u8]) -> Result { + match bytes.len() { + 0..=15 => { + let mut buf = [0u8; 16]; + buf[..bytes.len()].copy_from_slice(bytes); + Ok(buf.into()) + } + 16 => Ok(Self(unsafe { + _mm_loadu_si128(bytes.as_ptr() as *const __m128i) + })), + _ => Err(LengthError), + } + } +} + +simd_int!( + U256, + [u8; 32], + __m256i, + _mm256_setzero_si256, + _mm256_loadu_si256, + _mm256_storeu_si256, + bitxor_avx2, + "256-bit SIMD integer" +); + +impl U256 { + #[target_feature(enable = "avx")] + pub unsafe fn from_u128x2_avx(lo: U128, hi: U128) -> Self { + // TODO(tarcieri): compare lo/hi ordering and properties vs `_mm256_set_m128i` + Self(_mm256_setr_m128i(lo.0, hi.0)) + } + + #[target_feature(enable = "avx")] + #[target_feature(enable = "sse2")] + pub unsafe fn load_u128x2_avx(lo: &U128, hi: &U128) -> Self { + Self(_mm256_loadu2_m128i(&lo.0, &hi.0)) + } + + #[target_feature(enable = "avx")] + #[target_feature(enable = "sse2")] + pub unsafe fn store_u128x2_avx(self, lo: &mut U128, hi: &mut U128) { + _mm256_storeu2_m128i(&mut hi.0, &mut lo.0, self.0); + } + + #[target_feature(enable = "avx2")] + pub unsafe fn bitxor_avx2(self, rhs: Self) -> Self { + Self(_mm256_xor_si256(self.0, rhs.0)) + } +} + +impl From<(U128, U128)> for U256 { + #[inline(always)] + fn from(pair: (U128, U128)) -> U256 { + // TODO(tarcieri): runtime AVX feature detection + unsafe { U256::from_u128x2_avx(pair.0, pair.1) } + } +} + +impl From for (U128, U128) { + #[inline(always)] + fn from(x: U256) -> (U128, U128) { + let mut lo = U128::default(); + let mut hi = U128::default(); + + // TODO(tarcieri): runtime AVX feature detection + unsafe { x.store_u128x2_avx(&mut lo, &mut hi) } + (lo, hi) + } +} + +impl TryFrom<&[u8]> for U256 { + type Error = LengthError; + + #[inline(always)] + fn try_from(bytes: &[u8]) -> Result { + match bytes.len() { + 0..=31 => { + let mut buf = [0u8; 32]; + buf[..bytes.len()].copy_from_slice(bytes); + Ok(buf.into()) + } + 32 => Ok(Self(unsafe { + _mm256_loadu_si256(bytes.as_ptr() as *const __m256i) + })), + _ => Err(LengthError), + } + } +} + +/// 1024-bit SIMD buffer decomposed into 8 x [`U128`] values. +#[derive(Clone, Debug, Default)] +#[repr(align(128))] +pub struct U128x8([U128; 8]); + +impl TryFrom<&[u8]> for U128x8 { + type Error = LengthError; + + #[inline(always)] + fn try_from(bytes: &[u8]) -> Result { + let mut chunks = bytes.chunks_exact(16); + let mut result: [U128; 8] = unsafe { mem::zeroed() }; + let mut i = 0; + + for chunk in &mut chunks { + if i == 8 { + return Err(LengthError); + } + + result[i] = unsafe { _mm_loadu_si128(chunk.as_ptr() as *const __m128i) }.into(); + i += 1; + } + + let rem = chunks.remainder(); + + if !rem.is_empty() { + if i < 8 { + result[i] = U128::try_from(rem)?; + i += 1; + } else { + return Err(LengthError); + } + } + + while i < 8 { + result[i] = U128::zero(); + i += 1; + } + + Ok(Self(result)) + } +} + +impl From<[U128; 8]> for U128x8 { + #[inline(always)] + fn from(registers: [U128; 8]) -> U128x8 { + U128x8(registers) + } +} + +impl BitXorAssign for U128x8 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + // TODO(tarcieri): runtime AVX2 feature detection + unsafe { self.bitxor_assign_avx2(&rhs) } + } +} + +impl Index for U128x8 { + type Output = U128; + + #[inline(always)] + fn index(&self, idx: usize) -> &U128 { + self.0.index(idx) + } +} + +impl IndexMut for U128x8 { + #[inline(always)] + fn index_mut(&mut self, idx: usize) -> &mut Self::Output { + self.0.index_mut(idx) + } +} + +impl SimdBuffer for U128x8 { + type Register = U128; + + #[inline(always)] + fn as_slice(&self) -> &[U128] { + &self.0 + } + + #[inline(always)] + fn as_mut_slice(&mut self) -> &mut [U128] { + &mut self.0 + } +} + +impl U128x8 { + /// Perform a [`BitXorAssign`] operation using AVX2 intrinsics. + #[target_feature(enable = "avx")] + #[target_feature(enable = "avx2")] + #[target_feature(enable = "sse2")] + pub unsafe fn bitxor_assign_avx2(&mut self, rhs: &U128x8) { + for (lhs, rhs) in self.0.chunks_exact_mut(2).zip(rhs.0.chunks_exact(2)) { + let out = _mm256_xor_si256( + _mm256_setr_m128i(lhs[0].0, lhs[1].0), + _mm256_setr_m128i(rhs[0].0, rhs[1].0), + ); + + _mm256_storeu2_m128i(&mut lhs[0].0, &mut lhs[1].0, out); + } + } +} diff --git a/simd-buffers/tests/lib.rs b/simd-buffers/tests/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/simd-buffers/tests/lib.rs @@ -0,0 +1 @@ +