From 0caee95c3d77c27f6e7ad8109d7087f22c955fe5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 5 Oct 2022 08:48:33 -0700 Subject: [PATCH] Compile test cases with `wasm32-wasi` again This commit is the next step in integrating `wasm32-wasi`, `wit-bindgen`, tests, and components all together. Tests are now again compiled with `wasm32-wasi` and use a repo-specific adapter module (with support from #338) to support transforming the final module into an actual component. In supporting this feature the support from #331 is refactored into a new `extract` Rust module so the functionality can be shared between the ingestion of the main module as well as ingestion of adapter modules. Adapter modules now are also supported on the CLI as a standalone file without having to specify other options. Note that the actual `wasi_snapshot_preview1.wasm` adapter is non-functional in this commit and doesn't do anything fancy. The tests in this repository don't really need all that much and I suspect all we'll really need to implement is `fd_write` for fd 1 (as that's stdout). --- Cargo.lock | 8 ++ Cargo.toml | 1 + crates/test-helpers/build.rs | 27 ++++-- crates/wasi_snapshot_preview1/Cargo.toml | 12 +++ crates/wasi_snapshot_preview1/src/lib.rs | 44 +++++++++ crates/wit-component/src/cli.rs | 20 ++++- crates/wit-component/src/encoding.rs | 108 +++++++++++++---------- crates/wit-component/src/extract.rs | 72 +++++++++++++++ crates/wit-component/src/lib.rs | 2 + 9 files changed, 235 insertions(+), 59 deletions(-) create mode 100644 crates/wasi_snapshot_preview1/Cargo.toml create mode 100644 crates/wasi_snapshot_preview1/src/lib.rs create mode 100644 crates/wit-component/src/extract.rs diff --git a/Cargo.lock b/Cargo.lock index 428d6b7bf..2ff529ac9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1582,6 +1582,14 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "wasi_snapshot_preview1" +version = "0.0.0" +dependencies = [ + "wasi", + "wit-bindgen-guest-rust", +] + [[package]] name = "wasm-encoder" version = "0.17.0" diff --git a/Cargo.toml b/Cargo.toml index cd27e5212..a05172214 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/test-rust-wasm", "crates/wit-bindgen-demo", "crates/wit-component", + "crates/wasi_snapshot_preview1", ] resolver = "2" diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index 2b0c880fd..da10ae57f 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -10,24 +10,33 @@ fn main() { let mut wasms = Vec::new(); + // Build the `wasi_snapshot_preview1.wasm` adapter which is used to convert + // core wasm modules below into components via `wit-component`. + let mut cmd = Command::new("cargo"); + cmd.arg("build") + .arg("--release") + .current_dir("../wasi_snapshot_preview1") + .arg("--target=wasm32-unknown-unknown") + .env("CARGO_TARGET_DIR", &out_dir) + .env("RUSTFLAGS", "-Clink-args=--import-memory") + .env_remove("CARGO_ENCODED_RUSTFLAGS"); + let status = cmd.status().unwrap(); + assert!(status.success()); + println!("cargo:rerun-if-changed=../wasi_snapshot_preview1"); + let wasi_adapter = out_dir.join("wasm32-unknown-unknown/release/wasi_snapshot_preview1.wasm"); + if cfg!(feature = "guest-rust") { let mut cmd = Command::new("cargo"); cmd.arg("build") .current_dir("../test-rust-wasm") - // TODO: this should go back to wasm32-wasi once we have an adapter - // for snapshot 1 to a component - .arg("--target=wasm32-unknown-unknown") + .arg("--target=wasm32-wasi") .env("CARGO_TARGET_DIR", &out_dir) .env("CARGO_PROFILE_DEV_DEBUG", "1") .env("RUSTFLAGS", "-Clink-args=--export-table") .env_remove("CARGO_ENCODED_RUSTFLAGS"); let status = cmd.status().unwrap(); assert!(status.success()); - for file in out_dir - .join("wasm32-unknown-unknown/debug") - .read_dir() - .unwrap() - { + for file in out_dir.join("wasm32-wasi/debug").read_dir().unwrap() { let file = file.unwrap().path(); if file.extension().and_then(|s| s.to_str()) != Some("wasm") { continue; @@ -46,6 +55,8 @@ fn main() { .module(module.as_slice()) .expect("pull custom sections from module") .validate(true) + .adapter_file(&wasi_adapter) + .expect("adapter failed to get loaded") .encode() .expect("module can be translated to a component"); diff --git a/crates/wasi_snapshot_preview1/Cargo.toml b/crates/wasi_snapshot_preview1/Cargo.toml new file mode 100644 index 000000000..5ad16c3ff --- /dev/null +++ b/crates/wasi_snapshot_preview1/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "wasi_snapshot_preview1" +version = "0.0.0" +edition.workspace = true + +[dependencies] +wasi = "0.11.0" +wit-bindgen-guest-rust = { workspace = true } + +[lib] +crate-type = ["cdylib"] +test = false diff --git a/crates/wasi_snapshot_preview1/src/lib.rs b/crates/wasi_snapshot_preview1/src/lib.rs new file mode 100644 index 000000000..15b77c0dc --- /dev/null +++ b/crates/wasi_snapshot_preview1/src/lib.rs @@ -0,0 +1,44 @@ +//! This module is intended to be an "adapter module" fed into `wit-component` +//! to translate the `wasi_snapshot_preview1` ABI into an ABI that uses the +//! component model. This library is compiled as a standalone wasm file and is +//! used to implement `wasi_snapshot_preview1` interfaces required by the tests +//! throughout the `wit-bindgen` repository. +//! +//! This is not intended to be a comprehensive polyfill. Instead this is just +//! the bare bones necessary to get `wit-bindgen` itself and its tests working. +//! +//! Currently all functions are trapping stubs since nothing actually runs the +//! output component just yet. These stubs should get filled in as necessary +//! once hosts start running components. The current assumption is that the +//! imports will be adapted to a custom `wit-bindgen`-specific host `*.wit` file +//! which is only suitable for `wit-bindgen` tests. + +#![allow(unused_variables)] + +use std::arch::wasm32::unreachable; +use wasi::*; + +#[no_mangle] +pub extern "C" fn environ_get(environ: *mut *mut u8, environ_buf: *mut u8) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn environ_sizes_get(environc: *mut Size, environ_buf_size: *mut Size) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn fd_write( + fd: Fd, + iovs_ptr: *const Ciovec, + iovs_len: usize, + nwritten: *mut Size, +) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn proc_exit(rval: Exitcode) -> ! { + unreachable() +} diff --git a/crates/wit-component/src/cli.rs b/crates/wit-component/src/cli.rs index e0d375b1e..6bac00dce 100644 --- a/crates/wit-component/src/cli.rs +++ b/crates/wit-component/src/cli.rs @@ -2,6 +2,7 @@ #![deny(missing_docs)] +use crate::extract::{extract_module_interfaces, ModuleInterfaces}; use crate::{ decode_interface_component, ComponentEncoder, InterfaceEncoder, InterfacePrinter, StringEncoding, @@ -61,10 +62,21 @@ fn parse_adapter(s: &str) -> Result<(String, Vec, Interface)> { Ok((name.to_string(), wasm, interface)) } None => { - // TODO: implement inferring the `interface` from the `wasm` - // specified - drop((name, wasm)); - bail!("inferring from the core wasm module is not supported at this time") + let ModuleInterfaces { + mut imports, + exports, + wasm, + interface, + } = extract_module_interfaces(&wasm)?; + if exports.len() > 0 || interface.is_some() { + bail!("adapter modules cannot have an exported interface"); + } + let import = match imports.len() { + 0 => Interface::default(), + 1 => imports.remove(0), + _ => bail!("adapter modules can only import one interface at this time"), + }; + Ok((name.to_string(), wasm, import)) } } } diff --git a/crates/wit-component/src/encoding.rs b/crates/wit-component/src/encoding.rs index afa080c0c..86e37edce 100644 --- a/crates/wit-component/src/encoding.rs +++ b/crates/wit-component/src/encoding.rs @@ -52,8 +52,8 @@ //! otherwise there's no way to run a `wasi_snapshot_preview1` module within the //! component model. +use crate::extract::{extract_module_interfaces, ModuleInterfaces}; use crate::{ - decode_interface_component, validation::{ expected_export_name, validate_adapter_module, validate_module, ValidatedAdapter, ValidatedModule, @@ -64,6 +64,7 @@ use anyhow::{anyhow, bail, Context, Result}; use indexmap::{map::Entry, IndexMap, IndexSet}; use std::hash::{Hash, Hasher}; use std::mem; +use std::path::Path; use wasm_encoder::*; use wasmparser::{Validator, WasmFeatures}; use wit_parser::{ @@ -2095,55 +2096,32 @@ impl<'a> ImportEncoder<'a> { /// An encoder of components based on `wit` interface definitions. #[derive(Default)] -pub struct ComponentEncoder<'a> { - module: &'a [u8], +pub struct ComponentEncoder { + module: Vec, encoding: StringEncoding, interface: Option, imports: Vec, exports: Vec, validate: bool, types_only: bool, - adapters: IndexMap<&'a str, (&'a [u8], &'a Interface)>, + adapters: IndexMap, Interface)>, } -impl<'a> ComponentEncoder<'a> { +impl ComponentEncoder { /// Set the core module to encode as a component. /// This method will also parse any component type information stored in custom sections /// inside the module, and add them as the interface, imports, and exports. - pub fn module(mut self, module: &'a [u8]) -> Result { - for payload in wasmparser::Parser::new(0).parse_all(&module) { - match payload.context("decoding item in module")? { - wasmparser::Payload::CustomSection(cs) => { - if let Some(export) = cs.name().strip_prefix("component-type:export:") { - let mut i = decode_interface_component(cs.data()).with_context(|| { - format!("decoding component-type in export section {}", export) - })?; - i.name = export.to_owned(); - self.interface = Some(i); - } else if let Some(import) = cs.name().strip_prefix("component-type:import:") { - let mut i = decode_interface_component(cs.data()).with_context(|| { - format!("decoding component-type in import section {}", import) - })?; - i.name = import.to_owned(); - self.imports.push(i); - } else if let Some(export_instance) = - cs.name().strip_prefix("component-type:export-instance:") - { - let mut i = decode_interface_component(cs.data()).with_context(|| { - format!( - "decoding component-type in export-instance section {}", - export_instance - ) - })?; - i.name = export_instance.to_owned(); - self.exports.push(i); - } - } - _ => {} - } - } - - self.module = module; + pub fn module(mut self, module: &[u8]) -> Result { + let ModuleInterfaces { + imports, + exports, + wasm, + interface, + } = extract_module_interfaces(module)?; + self.interface = interface; + self.imports.extend(imports); + self.exports.extend(exports); + self.module = wasm; Ok(self) } @@ -2160,13 +2138,13 @@ impl<'a> ComponentEncoder<'a> { } /// Set the default interface exported by the component. - pub fn interface(mut self, interface: &'a Interface) -> Self { + pub fn interface(mut self, interface: &Interface) -> Self { self.interface = Some(interface.clone()); self } /// Set the interfaces the component imports. - pub fn imports(mut self, imports: &'a [Interface]) -> Self { + pub fn imports(mut self, imports: &[Interface]) -> Self { for i in imports { self.imports.push(i.clone()) } @@ -2174,7 +2152,7 @@ impl<'a> ComponentEncoder<'a> { } /// Set the interfaces the component exports. - pub fn exports(mut self, exports: &'a [Interface]) -> Self { + pub fn exports(mut self, exports: &[Interface]) -> Self { for e in exports { self.exports.push(e.clone()) } @@ -2198,17 +2176,53 @@ impl<'a> ComponentEncoder<'a> { /// wasm module specified by `bytes` imports. The `bytes` will then import /// `interface` and export functions to get imported from the module `name` /// in the core wasm that's being wrapped. - pub fn adapter(mut self, name: &'a str, bytes: &'a [u8], interface: &'a Interface) -> Self { - self.adapters.insert(name, (bytes, interface)); + pub fn adapter(mut self, name: &str, bytes: &[u8], interface: &Interface) -> Self { + self.adapters + .insert(name.to_string(), (bytes.to_vec(), interface.clone())); self } + /// This is a convenience method for [`ComponentEncoder::adapter`] for + /// inferring everything from just one `path` specified. + /// + /// The wasm binary at `path` is read and parsed and is assumed to have + /// embedded information about its imported interfaces. Additionally the + /// name of the adapter is inferred from the file name itself as the file + /// stem. + pub fn adapter_file(mut self, path: &Path) -> Result { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| anyhow!("input filename was not valid utf-8"))?; + let wasm = wat::parse_file(path)?; + let ModuleInterfaces { + mut imports, + exports, + wasm, + interface, + } = extract_module_interfaces(&wasm)?; + if exports.len() > 0 || interface.is_some() { + bail!("adapter modules cannot have an exported interface"); + } + let import = match imports.len() { + 0 => Interface::default(), + 1 => imports.remove(0), + _ => bail!("adapter modules can only import one interface at this time"), + }; + self.adapters.insert(name.to_string(), (wasm, import)); + Ok(self) + } + /// Encode the component and return the bytes. pub fn encode(&self) -> Result> { let info = if !self.module.is_empty() { - let adapters = self.adapters.keys().copied().collect::>(); + let adapters = self + .adapters + .keys() + .map(|s| s.as_str()) + .collect::>(); validate_module( - self.module, + &self.module, &self.interface, &self.imports, &self.exports, @@ -2257,7 +2271,7 @@ impl<'a> ComponentEncoder<'a> { types.finish(&mut state.component); state.encode_imports(&imports); - state.encode_core_module(self.module); + state.encode_core_module(&self.module); state.encode_core_instantiation(self.encoding, &imports, &info)?; state.encode_exports(self.encoding, exports, &types.func_type_map)?; } diff --git a/crates/wit-component/src/extract.rs b/crates/wit-component/src/extract.rs new file mode 100644 index 000000000..00bf036aa --- /dev/null +++ b/crates/wit-component/src/extract.rs @@ -0,0 +1,72 @@ +use crate::decode_interface_component; +use anyhow::{Context, Result}; +use wit_parser::Interface; + +/// Result of extracting interfaces embedded within a core wasm file. +/// +/// This structure is reated by the [`extract_module_interfaces`] function. +#[derive(Default)] +pub struct ModuleInterfaces { + /// The core wasm binary with custom sections removed. + pub wasm: Vec, + + /// Imported interfaces found in custom sections. + pub imports: Vec, + + /// Exported interfaces found in custom sections. + pub exports: Vec, + + /// The default exported interface found in a custom section. + pub interface: Option, +} + +/// This function will parse the `wasm` binary given as input and return a +/// [`ModuleInterfaces`] which extracts the custom sections describing +/// component-level types from within the binary itself. +/// +/// This is used to parse the output of `wit-bindgen`-generated modules and is +/// one of the earliest phases in transitioning such a module to a component. +/// The extraction here provides the metadata necessary to continue the process +/// later on. +pub fn extract_module_interfaces(wasm: &[u8]) -> Result { + let mut ret = ModuleInterfaces::default(); + + for payload in wasmparser::Parser::new(0).parse_all(wasm) { + match payload.context("decoding item in module")? { + wasmparser::Payload::CustomSection(cs) => { + if let Some(export) = cs.name().strip_prefix("component-type:export:") { + let mut i = decode_interface_component(cs.data()).with_context(|| { + format!("decoding component-type in export section {}", export) + })?; + i.name = export.to_owned(); + ret.interface = Some(i); + } else if let Some(import) = cs.name().strip_prefix("component-type:import:") { + let mut i = decode_interface_component(cs.data()).with_context(|| { + format!("decoding component-type in import section {}", import) + })?; + i.name = import.to_owned(); + ret.imports.push(i); + } else if let Some(export_instance) = + cs.name().strip_prefix("component-type:export-instance:") + { + let mut i = decode_interface_component(cs.data()).with_context(|| { + format!( + "decoding component-type in export-instance section {}", + export_instance + ) + })?; + i.name = export_instance.to_owned(); + ret.exports.push(i); + } + } + _ => {} + } + } + + // TODO: should remove the custom setions decoded above from the wasm binary + // created here, and bytecodealliance/wasmparser#792 should help with that + // to make the loop above pretty small. + ret.wasm = wasm.to_vec(); + + Ok(ret) +} diff --git a/crates/wit-component/src/lib.rs b/crates/wit-component/src/lib.rs index a1f51744e..6447470da 100644 --- a/crates/wit-component/src/lib.rs +++ b/crates/wit-component/src/lib.rs @@ -11,11 +11,13 @@ use wit_parser::Interface; pub mod cli; mod decoding; mod encoding; +mod extract; mod gc; mod printing; mod validation; pub use encoding::*; +pub use extract::*; pub use printing::*; /// Supported string encoding formats.