diff --git a/.cargo/config.toml b/.cargo/config.toml index abcd047..e9a242c 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,5 @@ [target.wasm32-wasip2] -runner = "wasmtime -Shttp" +# wasmtime is given: +# * AWS auth environment variables, for running the wstd-aws integration tests. +# * . directory is available at . +runner = "wasmtime run -Shttp --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN --dir .::." diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a468cb9..726a7cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,10 @@ on: env: RUSTFLAGS: -Dwarnings +# required for AWS oidc +permissions: + id-token: write + jobs: build_and_test: name: Build and test @@ -26,24 +30,35 @@ jobs: - name: Install wasmtime uses: bytecodealliance/actions/wasmtime/setup@v1 - - name: check - uses: actions-rs/cargo@v1 + # pchickey made a role `wstd-aws-ci-role` and bucket `wstd-example-bucket` + # on his personal aws account 313377415443. The role only provides + # ListBucket and GetObject for the example bucket, which is enough to pass + # the single integration test. The role is configured to trust GitHub + # actions for the bytecodealliance/wstd repo. This action will set the + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN + # environment variables. + - name: get aws credentials + id: creds + uses: aws-actions/configure-aws-credentials@v5.1.0 + continue-on-error: true with: - command: check - args: --workspace --all --bins --examples + aws-region: us-west-2 + role-to-assume: arn:aws:iam::313377415443:role/wstd-aws-ci-role + role-session-name: github-ci + + - name: check + run: cargo check --workspace --all --bins --examples - name: wstd tests - uses: actions-rs/cargo@v1 - with: - command: test - args: -p wstd --target wasm32-wasip2 -- --nocapture + run: cargo test -p wstd -p wstd-axum -p wstd-aws --target wasm32-wasip2 -- --nocapture - - name: example tests - uses: actions-rs/cargo@v1 - with: - command: test - args: -p test-programs -- --nocapture + - name: test-programs tests + run: cargo test -p test-programs -- --nocapture + if: steps.creds.outcome == 'success' + - name: test-programs tests (no aws) + run: cargo test -p test-programs --features no-aws -- --nocapture + if: steps.creds.outcome != 'success' check_fmt_and_docs: name: Checking fmt and docs diff --git a/Cargo.toml b/Cargo.toml index 9417066..ae12bc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true [workspace] -members = [ +members = ["aws", "axum", "axum/macro", "macro", @@ -70,6 +70,11 @@ authors = [ [workspace.dependencies] anyhow = "1" async-task = "4.7" +aws-config = { version = "1.8.8", default-features = false } +aws-sdk-s3 = { version = "1.108.0", default-features = false } +aws-smithy-async = { version = "1.2.6", default-features = false } +aws-smithy-types = { version = "1.3.3", default-features = false } +aws-smithy-runtime-api = { version = "1.9.1", default-features = false } axum = { version = "0.8.6", default-features = false } bytes = "1.10.1" cargo_metadata = "0.22" @@ -88,6 +93,7 @@ quote = "1.0" serde= "1" serde_json = "1" serde_qs = "0.15" +sync_wrapper = "1" slab = "0.4.9" syn = "2.0" test-log = { version = "0.2", features = ["trace"] } diff --git a/aws/.gitignore b/aws/.gitignore new file mode 100644 index 0000000..7d219f9 --- /dev/null +++ b/aws/.gitignore @@ -0,0 +1 @@ +.environment diff --git a/aws/Cargo.toml b/aws/Cargo.toml new file mode 100644 index 0000000..b4038e4 --- /dev/null +++ b/aws/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "wstd-aws" +description = "" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true +authors.workspace = true + +[dependencies] +anyhow.workspace = true +aws-smithy-async = { workspace = true } +aws-smithy-types = { workspace = true, features = ["http-body-1-x"] } +aws-smithy-runtime-api = { workspace = true, features = ["client", "http-1x"] } +http-body-util.workspace = true +sync_wrapper = { workspace = true, features = ["futures"] } +wstd.workspace = true + +[dev-dependencies] +aws-config.workspace = true +aws-sdk-s3.workspace = true +clap.workspace = true diff --git a/aws/README.md b/aws/README.md new file mode 100644 index 0000000..d404bcd --- /dev/null +++ b/aws/README.md @@ -0,0 +1,75 @@ + +# wstd-aws: wstd support for the AWS Rust SDK + +This crate provides support for using the AWS Rust SDK for the `wasm32-wasip2` +target using the [`wstd`] crate. + +In many wasi settings, its necessary or desirable to use the wasi-http +interface to make http requests. Wasi-http interfaces provide an http +implementation, including the sockets layer and TLS, outside of the user's +component. `wstd` provides user-friendly async Rust interfaces to all of the +standardized wasi interfaces, including wasi-http. + +The AWS Rust SDK, by default, depends on `tokio`, `hyper`, and either `rustls` +or `s2n_tls`, and makes http requests over sockets (which can be provided as +wasi-sockets). Those dependencies may not work correctly under `wasm32-wasip2`, +and if they do, they will not use the wasi-http interfaces. To avoid using +http over sockets, make sure to set the `default-features = false` setting +when depending on any `aws-*` crates in your project. + +To configure `wstd`'s wasi-http client for the AWS Rust SDK, provide +`wstd_aws::sleep_impl()` and `wstd_aws::http_client()` to your +[`aws_config::ConfigLoader`]: + +``` + let config = aws_config::defaults(BehaviorVersion::latest()) + .sleep_impl(wstd_aws::sleep_impl()) + .http_client(wstd_aws::http_client()) + ...; +``` + +[`wstd`]: https://docs.rs/wstd/latest/wstd +[`aws_config::ConfigLoader`]: https://docs.rs/aws-config/1.8.8/aws_config/struct.ConfigLoader.html + +## Example + +An example s3 client is provided as a wasi cli command. It accepts command +line arguments with the subcommand `list` to list a bucket's contents, and +`get ` to get an object from a bucket and write it to the filesystem. + +This example *must be compiled in release mode* - in debug mode, the aws +sdk's generated code will overflow the maximum permitted wasm locals in +a single function. + +Compile it with: + +```sh +cargo build -p wstd-aws --target wasm32-wasip2 --release --examples +``` + +When running this example, you will need AWS credentials provided in environment +variables. + +Run it with: +```sh +wasmtime run -Shttp \ + --env AWS_ACCESS_KEY_ID \ + --env AWS_SECRET_ACCESS_KEY \ + --env AWS_SESSION_TOKEN \ + --dir .::. \ + target/wasm32-wasip2/release/examples/s3.wasm +``` + +or alternatively run it with: +```sh +cargo run --target wasm32-wasip2 -p wstd-aws --example s3 +``` + +which uses the wasmtime cli, as above, via configiration found in this +workspace's `.cargo/config`. + +By default, this script accesses the `wstd-example-bucket` in `us-west-2`. +To change the bucket or region, use the `--bucket` and `--region` cli +flags before the subcommand. + + diff --git a/aws/examples/s3.rs b/aws/examples/s3.rs new file mode 100644 index 0000000..3fcf4ae --- /dev/null +++ b/aws/examples/s3.rs @@ -0,0 +1,149 @@ +//! Example s3 client running on `wstd` via `wstd_aws` +//! +//! This example is a wasi cli command. It accepts command line arguments +//! with the subcommand `list` to list a bucket's contents, and `get ` +//! to get an object from a bucket and write it to the filesystem. +//! +//! This example *must be compiled in release mode* - in debug mode, the aws +//! sdk's generated code will overflow the maximum permitted wasm locals in +//! a single function. +//! +//! Compile it with: +//! +//! ```sh +//! cargo build -p wstd-aws --target wasm32-wasip2 --release --examples +//! ``` +//! +//! When running this example, you will need AWS credentials provided in environment +//! variables. +//! +//! Run it with: +//! ```sh +//! wasmtime run -Shttp \ +//! --env AWS_ACCESS_KEY_ID \ +//! --env AWS_SECRET_ACCESS_KEY \ +//! --env AWS_SESSION_TOKEN \ +//! --dir .::. \ +//! target/wasm22-wasip2/release/examples/s3.wasm +//! ``` +//! +//! or alternatively run it with: +//! ```sh +//! cargo run --target wasm32-wasip2 -p wstd-aws --example s3 +//! ``` +//! +//! which uses the wasmtime cli, as above, via configiration found in this +//! workspace's `.cargo/config`. +//! +//! By default, this script accesses the `wstd-example-bucket` in `us-west-2`. +//! To change the bucket or region, use the `--bucket` and `--region` cli +//! flags before the subcommand. + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +use aws_config::{BehaviorVersion, Region}; +use aws_sdk_s3::Client; + +#[derive(Debug, Parser)] +#[command(version, about, long_about = None)] +struct Opts { + /// The AWS Region. Defaults to us-west-2 if not provided. + #[arg(short, long)] + region: Option, + /// The name of the bucket. Defaults to wstd-example-bucket if not + /// provided. + #[arg(short, long)] + bucket: Option, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand, Debug)] +enum Command { + List, + Get { + key: String, + #[arg(short, long)] + out: Option, + }, +} + +#[wstd::main] +async fn main() -> Result<()> { + let opts = Opts::parse(); + let region = opts + .region + .clone() + .unwrap_or_else(|| "us-west-2".to_owned()); + let bucket = opts + .bucket + .clone() + .unwrap_or_else(|| "wstd-example-bucket".to_owned()); + + let config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new(region)) + .sleep_impl(wstd_aws::sleep_impl()) + .http_client(wstd_aws::http_client()) + .load() + .await; + + let client = Client::new(&config); + + match opts.command.as_ref().unwrap_or(&Command::List) { + Command::List => { + let output = list(&bucket, &client).await?; + print!("{}", output); + } + Command::Get { key, out } => { + let contents = get(&bucket, &client, key).await?; + let output: &str = if let Some(out) = out { + out.as_str() + } else { + key.as_str() + }; + std::fs::write(output, contents)?; + } + } + Ok(()) +} + +async fn list(bucket: &str, client: &Client) -> Result { + let mut listing = client + .list_objects_v2() + .bucket(bucket.to_owned()) + .into_paginator() + .send(); + + let mut output = String::new(); + output += "key\tetag\tlast_modified\tstorage_class\n"; + while let Some(res) = listing.next().await { + let object = res?; + for item in object.contents() { + output += &format!( + "{}\t{}\t{}\t{}\n", + item.key().unwrap_or_default(), + item.e_tag().unwrap_or_default(), + item.last_modified() + .map(|lm| format!("{lm}")) + .unwrap_or_default(), + item.storage_class() + .map(|sc| format!("{sc}")) + .unwrap_or_default(), + ); + } + } + Ok(output) +} + +async fn get(bucket: &str, client: &Client, key: &str) -> Result> { + let object = client + .get_object() + .bucket(bucket.to_owned()) + .key(key) + .send() + .await?; + let data = object.body.collect().await?; + Ok(data.to_vec()) +} diff --git a/aws/src/lib.rs b/aws/src/lib.rs new file mode 100644 index 0000000..0a03ac0 --- /dev/null +++ b/aws/src/lib.rs @@ -0,0 +1,101 @@ +use anyhow::anyhow; +use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::retries::ErrorKind; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_runtime_api::http::Response; +use aws_smithy_types::body::SdkBody; +use http_body_util::{BodyStream, StreamBody}; +use std::time::Duration; +use sync_wrapper::SyncStream; +use wstd::http::{Body as WstdBody, BodyExt, Client}; + +pub fn sleep_impl() -> impl AsyncSleep + 'static { + WstdSleep +} + +#[derive(Debug)] +struct WstdSleep; +impl AsyncSleep for WstdSleep { + fn sleep(&self, duration: Duration) -> Sleep { + Sleep::new(async move { + wstd::task::sleep(wstd::time::Duration::from(duration)).await; + }) + } +} + +pub fn http_client() -> impl HttpClient + 'static { + WstdHttpClient +} + +#[derive(Debug)] +struct WstdHttpClient; + +impl HttpClient for WstdHttpClient { + fn http_connector( + &self, + settings: &HttpConnectorSettings, + // afaict, none of these components are relevant to this + // implementation. + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + let mut client = Client::new(); + if let Some(timeout) = settings.connect_timeout() { + client.set_connect_timeout(timeout); + } + if let Some(timeout) = settings.read_timeout() { + client.set_first_byte_timeout(timeout); + } + SharedHttpConnector::new(WstdHttpConnector(client)) + } +} + +#[derive(Debug)] +struct WstdHttpConnector(Client); + +impl HttpConnector for WstdHttpConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + let client = self.0.clone(); + HttpConnectorFuture::new(async move { + let request = request + .try_into_http1x() + // This can only fail if the Extensions fail to convert + .map_err(|e| ConnectorError::other(Box::new(e), None))?; + // smithy's SdkBody Error is a non-'static boxed dyn stderror. + // Anyhow can't represent that, so convert it to the debug impl. + let request = + request.map(|body| WstdBody::from_http_body(body.map_err(|e| anyhow!("{e:?}")))); + // Any error given by send is considered a "ClientError" kind + // which should prevent smithy from retrying like it would for a + // throttling error + let response = client + .send(request) + .await + .map_err(|e| ConnectorError::other(e.into(), Some(ErrorKind::ClientError)))?; + + Response::try_from(response.map(|wstd_body| { + // You'd think that an SdkBody would just be an impl Body with + // the usual error type dance. + let nonsync_body = wstd_body + .into_boxed_body() + .map_err(|e| e.into_boxed_dyn_error()); + // But we have to do this weird dance: because Axum insists + // bodies are not Sync, wstd settled on non-Sync bodies. + // Smithy insists on Sync bodies. The SyncStream type exists + // to assert, because all Stream operations are on &mut self, + // all Streams are Sync. So, turn the Body into a Stream, make + // it sync, then back to a Body. + let nonsync_stream = BodyStream::new(nonsync_body); + let sync_stream = SyncStream::new(nonsync_stream); + let sync_body = StreamBody::new(sync_stream); + SdkBody::from_body_1_x(sync_body) + })) + // This can only fail if the Extensions fail to convert + .map_err(|e| ConnectorError::other(Box::new(e), None)) + }) + } +} diff --git a/ci/publish.rs b/ci/publish.rs index bd20833..cff7b58 100644 --- a/ci/publish.rs +++ b/ci/publish.rs @@ -16,7 +16,13 @@ use std::thread; use std::time::Duration; // note that this list must be topologically sorted by dependencies -const CRATES_TO_PUBLISH: &[&str] = &["wstd-macro", "wstd", "wstd-axum-macro", "wstd-axum"]; +const CRATES_TO_PUBLISH: &[&str] = &[ + "wstd-macro", + "wstd", + "wstd-axum-macro", + "wstd-axum", + "wstd-aws", +]; #[derive(Debug)] struct Workspace { @@ -53,11 +59,13 @@ fn main() { bump_version(&krate, &crates, name == "bump-patch"); } // update the lock file - assert!(Command::new("cargo") - .arg("fetch") - .status() - .unwrap() - .success()); + assert!( + Command::new("cargo") + .arg("fetch") + .status() + .unwrap() + .success() + ); } "publish" => { diff --git a/src/http/client.rs b/src/http/client.rs index a3f9718..de7ff11 100644 --- a/src/http/client.rs +++ b/src/http/client.rs @@ -6,7 +6,7 @@ use crate::time::Duration; use wasip2::http::types::RequestOptions as WasiRequestOptions; /// An HTTP client. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Client { options: Option, } @@ -85,7 +85,7 @@ impl Client { } } -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] struct RequestOptions { connect_timeout: Option, first_byte_timeout: Option, diff --git a/test-programs/Cargo.toml b/test-programs/Cargo.toml index 7f6191e..ba431b4 100644 --- a/test-programs/Cargo.toml +++ b/test-programs/Cargo.toml @@ -15,3 +15,7 @@ ureq.workspace = true [build-dependencies] cargo_metadata.workspace = true heck.workspace = true + +[features] +default = [] +no-aws = [] diff --git a/test-programs/build.rs b/test-programs/build.rs index 12574e6..36a4484 100644 --- a/test-programs/build.rs +++ b/test-programs/build.rs @@ -1,36 +1,25 @@ -use cargo_metadata::TargetKind; +use cargo_metadata::{MetadataCommand, Package, TargetKind}; use heck::ToShoutySnakeCase; use std::env::var_os; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { let out_dir = PathBuf::from(var_os("OUT_DIR").expect("OUT_DIR env var exists")); - let meta = cargo_metadata::MetadataCommand::new() - .exec() - .expect("cargo metadata"); - let wstd_meta = meta - .packages - .iter() - .find(|p| *p.name == "wstd") - .expect("wstd is in cargo metadata"); - let wstd_axum_meta = meta - .packages - .iter() - .find(|p| *p.name == "wstd-axum") - .expect("wstd is in cargo metadata"); + let meta = MetadataCommand::new().exec().expect("cargo metadata"); - let wstd_root = wstd_meta.manifest_path.parent().unwrap(); println!( "cargo:rerun-if-changed={}", - wstd_root.as_os_str().to_str().unwrap() + meta.workspace_root.as_os_str().to_str().unwrap() ); fn build_examples(pkg: &str, out_dir: &PathBuf) { + // release build is required for aws sdk to not overflow wasm locals let status = Command::new("cargo") .arg("build") .arg("--examples") + .arg("--release") .arg("--target=wasm32-wasip2") .arg(format!("--package={pkg}")) .env("CARGO_TARGET_DIR", out_dir) @@ -43,46 +32,59 @@ fn main() { } build_examples("wstd", &out_dir); build_examples("wstd-axum", &out_dir); + build_examples("wstd-aws", &out_dir); let mut generated_code = "// THIS FILE IS GENERATED CODE\n".to_string(); - for binary in wstd_meta - .targets - .iter() - .filter(|t| t.kind == [TargetKind::Example]) - { - let component_path = out_dir - .join("wasm32-wasip2") - .join("debug") - .join("examples") - .join(format!("{}.wasm", binary.name)); + fn module_for(name: &str, out_dir: &Path, meta: &Package) -> String { + let mut generated_code = String::new(); + generated_code += &format!("pub mod {name} {{"); + for binary in meta + .targets + .iter() + .filter(|t| t.kind == [TargetKind::Example]) + { + let component_path = out_dir + .join("wasm32-wasip2") + .join("release") + .join("examples") + .join(format!("{}.wasm", binary.name)); - let const_name = binary.name.to_shouty_snake_case(); - generated_code += &format!( - "pub const {const_name}: &str = {:?};\n", - component_path.as_os_str().to_str().expect("path is str") - ); + let const_name = binary.name.to_shouty_snake_case(); + generated_code += &format!( + "pub const {const_name}: &str = {:?};\n", + component_path.as_os_str().to_str().expect("path is str") + ); + } + generated_code += "}\n\n"; // end `pub mod {name}` + generated_code } - generated_code += "pub mod axum {"; - for binary in wstd_axum_meta - .targets - .iter() - .filter(|t| t.kind == [TargetKind::Example]) - { - let component_path = out_dir - .join("wasm32-wasip2") - .join("debug") - .join("examples") - .join(format!("{}.wasm", binary.name)); - - let const_name = binary.name.to_shouty_snake_case(); - generated_code += &format!( - "pub const {const_name}: &str = {:?};\n", - component_path.as_os_str().to_str().expect("path is str") - ); - } - generated_code += "}"; // end `pub mod axum` + generated_code += &module_for( + "_wstd", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd") + .expect("wstd is in cargo metadata"), + ); + generated_code += "pub use _wstd::*;\n\n"; + generated_code += &module_for( + "axum", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd-axum") + .expect("wstd-axum is in cargo metadata"), + ); + generated_code += &module_for( + "aws", + &out_dir, + meta.packages + .iter() + .find(|p| *p.name == "wstd-aws") + .expect("wstd-aws is in cargo metadata"), + ); std::fs::write(out_dir.join("gen.rs"), generated_code).unwrap(); } diff --git a/test-programs/tests/aws_s3.rs b/test-programs/tests/aws_s3.rs new file mode 100644 index 0000000..c411d27 --- /dev/null +++ b/test-programs/tests/aws_s3.rs @@ -0,0 +1,59 @@ +use anyhow::Result; +use std::path::Path; +use std::process::Command; + +fn run_s3_example() -> Command { + let mut command = Command::new("wasmtime"); + command.arg("run"); + command.arg("-Shttp"); + command.args(["--env", "AWS_ACCESS_KEY_ID"]); + command.args(["--env", "AWS_SECRET_ACCESS_KEY"]); + command.args(["--env", "AWS_SESSION_TOKEN"]); + command.args(["--dir", ".::."]); + command.arg(test_programs::aws::S3); + command +} + +#[test_log::test] +#[cfg_attr(feature = "no-aws", ignore)] +fn aws_s3() -> Result<()> { + // bucket list command + let output = run_s3_example() + .arg(format!( + "--region={}", + std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) + )) + .arg(format!( + "--bucket={}", + std::env::var("WSTD_EXAMPLE_BUCKET") + .unwrap_or_else(|_| "wstd-example-bucket".to_owned()) + )) + .arg("list") + .output()?; + println!("{:?}", output); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("fluff.jpg")); + assert!(stdout.contains("shoug.jpg")); + + // bucket get command + let output = run_s3_example() + .arg(format!( + "--region={}", + std::env::var("AWS_REGION").unwrap_or_else(|_| "us-west-2".to_owned()) + )) + .arg(format!( + "--bucket={}", + std::env::var("WSTD_EXAMPLE_BUCKET") + .unwrap_or_else(|_| "wstd-example-bucket".to_owned()) + )) + .arg("get") + .arg("shoug.jpg") + .output()?; + println!("{:?}", output); + assert!(output.status.success()); + + assert!(Path::new("shoug.jpg").exists()); + + Ok(()) +}