From 2fb775be4e2801a6a65f976479484e138afb458e Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 4 Oct 2022 18:37:05 -0700 Subject: [PATCH 01/11] bindgen-core: Direction gets an arbitrary Default so that I can use it in all these structs that derive Default which peter likes to use as a constructor. Its a nice pattern tbh --- crates/bindgen-core/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/bindgen-core/src/lib.rs b/crates/bindgen-core/src/lib.rs index a6869d45c..4ee2a00c2 100644 --- a/crates/bindgen-core/src/lib.rs +++ b/crates/bindgen-core/src/lib.rs @@ -33,8 +33,9 @@ pub use ns::Ns; /// `export` means I'm exporting functions to be called, and `import` means I'm /// importing functions that I'm going to call, in both wasm modules and host /// code. The enum here represents this user perspective. -#[derive(Copy, Clone, Eq, PartialEq)] +#[derive(Copy, Clone, Eq, PartialEq, Default)] pub enum Direction { + #[default] Import, Export, } From 977d718ac7221153f18dfb7723f81bc47bfdd88a Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 4 Oct 2022 18:38:03 -0700 Subject: [PATCH 02/11] gen-guest-c: spit out an object file containing the component type and use some symbol trickery to try to get it linked into the final binary --- Cargo.lock | 3 + crates/gen-guest-c/Cargo.toml | 3 + .../gen-guest-c/src/component_type_object.rs | 70 +++++++++++++++++++ crates/gen-guest-c/src/lib.rs | 23 ++++++ 4 files changed, 99 insertions(+) create mode 100644 crates/gen-guest-c/src/component_type_object.rs diff --git a/Cargo.lock b/Cargo.lock index 2ff529ac9..d99efec09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2038,10 +2038,13 @@ dependencies = [ name = "wit-bindgen-gen-guest-c" version = "0.2.0" dependencies = [ + "anyhow", "clap", "heck 0.3.3", "test-helpers", + "wasm-encoder 0.18.0", "wit-bindgen-core", + "wit-component", ] [[package]] diff --git a/crates/gen-guest-c/Cargo.toml b/crates/gen-guest-c/Cargo.toml index 53da2923d..33de6c26f 100644 --- a/crates/gen-guest-c/Cargo.toml +++ b/crates/gen-guest-c/Cargo.toml @@ -10,6 +10,9 @@ test = false [dependencies] wit-bindgen-core = { workspace = true } +wit-component = { workspace = true } +wasm-encoder = { workspace = true } +anyhow = { workspace = true } heck = { workspace = true } clap = { workspace = true, optional = true } diff --git a/crates/gen-guest-c/src/component_type_object.rs b/crates/gen-guest-c/src/component_type_object.rs new file mode 100644 index 000000000..0612ecdbc --- /dev/null +++ b/crates/gen-guest-c/src/component_type_object.rs @@ -0,0 +1,70 @@ +use anyhow::{Context, Result}; +use wasm_encoder::{ + CodeSection, CustomSection, Encode, Function, FunctionSection, Module, TypeSection, +}; +use wit_bindgen_core::{wit_parser::Interface, Direction}; +use wit_component::InterfaceEncoder; + +pub fn linking_symbol(iface: &Interface, direction: Direction) -> String { + format!( + "__component_type_object_force_link_{}_{}", + iface.name, + match direction { + Direction::Import => "import", + Direction::Export => "export", + } + ) +} + +pub fn object(iface: &Interface, direction: Direction) -> Result> { + let mut module = Module::new(); + + // Build a module with one function that's a "dummy function" + let mut types = TypeSection::new(); + types.function([], []); + module.section(&types); + let mut funcs = FunctionSection::new(); + funcs.function(0); + module.section(&funcs); + let mut code = CodeSection::new(); + code.function(&Function::new([])); + module.section(&code); + + let name = format!( + "component-type:{}:{}", + match direction { + Direction::Import => "import", + Direction::Export => "export", + }, + iface.name + ); + let data = InterfaceEncoder::new(iface) + .encode() + .with_context(|| format!("translating interface {} to component type", iface.name))?; + // Add our custom section + module.section(&CustomSection { + name: &name, + data: data.as_slice(), + }); + + // Append the `.linking` section + let mut data = Vec::new(); + data.push(0x02); // version 2 + { + let mut subsection = Vec::::new(); + subsection.push(0x01); // syminfo count + subsection.push(0x00); // SYMTAB_FUNCTION + 0u32.encode(&mut subsection); // flags + 0u32.encode(&mut subsection); // index + linking_symbol(iface, direction).encode(&mut subsection); // name + + data.push(0x08); // `WASM_SYMBOL_TABLE` + subsection.encode(&mut data); + } + module.section(&CustomSection { + name: "linking", + data: &data, + }); + + Ok(module.finish()) +} diff --git a/crates/gen-guest-c/src/lib.rs b/crates/gen-guest-c/src/lib.rs index f0be76805..d621c3d5d 100644 --- a/crates/gen-guest-c/src/lib.rs +++ b/crates/gen-guest-c/src/lib.rs @@ -1,3 +1,5 @@ +mod component_type_object; + use heck::*; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Write; @@ -34,6 +36,8 @@ pub struct C { types: HashMap, needs_string: bool, + + direction: Direction, } struct Func { @@ -1064,6 +1068,9 @@ impl Generator for C { func.name.to_snake_case() )); + // need to copy this before mutable borrow of self + let direction = self.direction; + let mut f = FunctionBindgen::new(self, c_sig, &import_name); match sig.results.len() { 0 => f.gen.src.c("void"), @@ -1086,6 +1093,13 @@ impl Generator for C { } f.gen.src.c(") {\n"); + // Force linking to the component type object if this function is live + uwrite!( + f.gen.src.c, + "(void) {};", + component_type_object::linking_symbol(iface, direction) + ); + // Perform all lifting/lowering and append it to our src. iface.call( AbiVariant::GuestExport, @@ -1161,8 +1175,11 @@ impl Generator for C { "\ #include #include <{}.h> + + extern void {}(void); ", iface.name.to_kebab_case(), + component_type_object::linking_symbol(iface, self.direction), ); self.print_intrinsics(); @@ -1289,6 +1306,12 @@ impl Generator for C { &format!("{}.h", iface.name.to_kebab_case()), self.src.h.as_bytes(), ); + files.push( + &format!("{}_component_type.o", iface.name.to_kebab_case()), + component_type_object::object(iface, self.direction) + .unwrap() + .as_slice(), + ); } } From 586ed3954ee65cab76f0a58d7366621a359b37c9 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Tue, 4 Oct 2022 18:50:03 -0700 Subject: [PATCH 03/11] link component types into guest c wasm. Can't transcode to component yet. --- crates/gen-guest-c/src/lib.rs | 1 + crates/test-helpers/build.rs | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/crates/gen-guest-c/src/lib.rs b/crates/gen-guest-c/src/lib.rs index d621c3d5d..a4af89a44 100644 --- a/crates/gen-guest-c/src/lib.rs +++ b/crates/gen-guest-c/src/lib.rs @@ -704,6 +704,7 @@ impl Return { impl Generator for C { fn preprocess_one(&mut self, iface: &Interface, dir: Direction) { + self.direction = dir; let variant = Self::abi_variant(dir); self.sizes.fill(iface); self.in_import = variant == AbiVariant::GuestImport; diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index da10ae57f..f31de70cf 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -118,7 +118,9 @@ fn main() { cmd.arg("--sysroot").arg(path.join("share/wasi-sysroot")); cmd.arg(c_impl) .arg(out_dir.join("imports.c")) + .arg(out_dir.join("imports_component_type.o")) .arg(out_dir.join("exports.c")) + .arg(out_dir.join("exports_component_type.o")) .arg("-I") .arg(&out_dir) .arg("-Wall") @@ -149,6 +151,26 @@ fn main() { test_dir.file_stem().unwrap().to_str().unwrap().to_string(), out_wasm.to_str().unwrap().to_string(), )); + + // The "invalid" test doesn't actually use the rust-guest macro + // and doesn't put the custom sections in, so component translation + // will fail. + if test_dir.file_stem().unwrap().to_str().unwrap() != "invalid" { + // Validate that the module can be translated to a component, using + // the component-type custom sections. We don't yet consume this component + // anywhere. + // FIXME need a wasi_snapshot_preview1 shim in order to be able to encode as a + // component - then we can uncomment below + /* + let module = fs::read(&out_wasm).expect("failed to read wasm file"); + ComponentEncoder::default() + .module(module.as_slice()) + .expect("pull custom sections from module") + .validate(true) + .encode() + .expect("module can be translated to a component"); + */ + } } } From 936ea7e90c3cf87585136f7d4949aafc45adaeb6 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 10:34:32 -0700 Subject: [PATCH 04/11] snake case iface name in c identifier --- crates/gen-guest-c/src/component_type_object.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/gen-guest-c/src/component_type_object.rs b/crates/gen-guest-c/src/component_type_object.rs index 0612ecdbc..d2ea7d9e6 100644 --- a/crates/gen-guest-c/src/component_type_object.rs +++ b/crates/gen-guest-c/src/component_type_object.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use heck::SnakeCase; use wasm_encoder::{ CodeSection, CustomSection, Encode, Function, FunctionSection, Module, TypeSection, }; @@ -8,7 +9,7 @@ use wit_component::InterfaceEncoder; pub fn linking_symbol(iface: &Interface, direction: Direction) -> String { format!( "__component_type_object_force_link_{}_{}", - iface.name, + iface.name.to_snake_case(), match direction { Direction::Import => "import", Direction::Export => "export", From b195aee0588024c0d2274cf786044bcb07868042 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 10:43:58 -0700 Subject: [PATCH 05/11] wasi_snapshot_preview1: add stubs for fd_seek and fd_close wasi-libc ends up always importing these for stdio vtables --- crates/wasi_snapshot_preview1/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/wasi_snapshot_preview1/src/lib.rs b/crates/wasi_snapshot_preview1/src/lib.rs index 15b77c0dc..d0b8c4841 100644 --- a/crates/wasi_snapshot_preview1/src/lib.rs +++ b/crates/wasi_snapshot_preview1/src/lib.rs @@ -38,6 +38,16 @@ pub extern "C" fn fd_write( unreachable() } +#[no_mangle] +pub extern "C" fn fd_seek(fd: Fd, offset: Filedelta, whence: Whence, filesize: *mut Size) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn fd_close(fd: Fd) -> Errno { + unreachable() +} + #[no_mangle] pub extern "C" fn proc_exit(rval: Exitcode) -> ! { unreachable() From 3a92b7455208b45fc5677b52d5a2986fb7243687 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 10:44:38 -0700 Subject: [PATCH 06/11] test-helpers: validate that all C guests are encodable as components --- crates/test-helpers/build.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index f31de70cf..81d00dfef 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -159,17 +159,15 @@ fn main() { // Validate that the module can be translated to a component, using // the component-type custom sections. We don't yet consume this component // anywhere. - // FIXME need a wasi_snapshot_preview1 shim in order to be able to encode as a - // component - then we can uncomment below - /* let module = fs::read(&out_wasm).expect("failed to read wasm file"); ComponentEncoder::default() .module(module.as_slice()) .expect("pull custom sections from module") .validate(true) + .adapter_file(&wasi_adapter) + .expect("adapter failed to get loaded") .encode() .expect("module can be translated to a component"); - */ } } } From f928c5e551eeea50a750e064947502f193262917 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 12:00:44 -0700 Subject: [PATCH 07/11] no need to special-case invalid test anymore --- crates/test-helpers/build.rs | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index 81d00dfef..cf438e34f 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -152,23 +152,18 @@ fn main() { out_wasm.to_str().unwrap().to_string(), )); - // The "invalid" test doesn't actually use the rust-guest macro - // and doesn't put the custom sections in, so component translation - // will fail. - if test_dir.file_stem().unwrap().to_str().unwrap() != "invalid" { - // Validate that the module can be translated to a component, using - // the component-type custom sections. We don't yet consume this component - // anywhere. - let module = fs::read(&out_wasm).expect("failed to read wasm file"); - ComponentEncoder::default() - .module(module.as_slice()) - .expect("pull custom sections from module") - .validate(true) - .adapter_file(&wasi_adapter) - .expect("adapter failed to get loaded") - .encode() - .expect("module can be translated to a component"); - } + // Validate that the module can be translated to a component, using + // the component-type custom sections. We don't yet consume this component + // anywhere. + let module = fs::read(&out_wasm).expect("failed to read wasm file"); + ComponentEncoder::default() + .module(module.as_slice()) + .expect("pull custom sections from module") + .validate(true) + .adapter_file(&wasi_adapter) + .expect("adapter failed to get loaded") + .encode() + .expect("module can be translated to a component"); } } From 9a84bfa69869964dbc8a1b91bbf9efad34d2544d Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 12:01:00 -0700 Subject: [PATCH 08/11] gen-guest-c: refer to the linking symbol from a public func which is never used --- crates/gen-guest-c/src/lib.rs | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/gen-guest-c/src/lib.rs b/crates/gen-guest-c/src/lib.rs index a4af89a44..a810b3091 100644 --- a/crates/gen-guest-c/src/lib.rs +++ b/crates/gen-guest-c/src/lib.rs @@ -1069,9 +1069,6 @@ impl Generator for C { func.name.to_snake_case() )); - // need to copy this before mutable borrow of self - let direction = self.direction; - let mut f = FunctionBindgen::new(self, c_sig, &import_name); match sig.results.len() { 0 => f.gen.src.c("void"), @@ -1094,13 +1091,6 @@ impl Generator for C { } f.gen.src.c(") {\n"); - // Force linking to the component type object if this function is live - uwrite!( - f.gen.src.c, - "(void) {};", - component_type_object::linking_symbol(iface, direction) - ); - // Perform all lifting/lowering and append it to our src. iface.call( AbiVariant::GuestExport, @@ -1171,16 +1161,22 @@ impl Generator for C { ", iface.name.to_shouty_snake_case(), ); + let linking_symbol = component_type_object::linking_symbol(iface, self.direction); uwrite!( self.src.c, "\ #include #include <{}.h> - extern void {}(void); + // The following symbols are never called, but they are sufficient + // to get the custom sections in the component type object linked + // into the wasm when this compilation unit is linked. + extern void {linking_symbol}(void); + void {linking_symbol}_public_use_in_this_compilation_unit(void) {{ + {linking_symbol}(); + }} ", iface.name.to_kebab_case(), - component_type_object::linking_symbol(iface, self.direction), ); self.print_intrinsics(); From b2552ea8f214fc4c29c07b14c4c5c369f7cdb144 Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 12:27:35 -0700 Subject: [PATCH 09/11] help me debug failures better --- crates/test-helpers/build.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index cf438e34f..930c87410 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -58,7 +58,10 @@ fn main() { .adapter_file(&wasi_adapter) .expect("adapter failed to get loaded") .encode() - .expect("module can be translated to a component"); + .expect(&format!( + "module {:?} can be translated to a component", + file + )); let dep_file = file.with_extension("d"); let deps = fs::read_to_string(&dep_file).expect("failed to read dep file"); @@ -163,7 +166,10 @@ fn main() { .adapter_file(&wasi_adapter) .expect("adapter failed to get loaded") .encode() - .expect("module can be translated to a component"); + .expect(&format!( + "module {:?} can be translated to a component", + out_wasm + )); } } From b8c10689f04dc1811293d2ad78b7bcc16b40f63b Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 14:55:28 -0700 Subject: [PATCH 10/11] wasi stubs: add args_get, args_sizes_get, and clock_time_get for teavm --- crates/wasi_snapshot_preview1/src/lib.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/wasi_snapshot_preview1/src/lib.rs b/crates/wasi_snapshot_preview1/src/lib.rs index d0b8c4841..738c4a14a 100644 --- a/crates/wasi_snapshot_preview1/src/lib.rs +++ b/crates/wasi_snapshot_preview1/src/lib.rs @@ -28,6 +28,25 @@ pub extern "C" fn environ_sizes_get(environc: *mut Size, environ_buf_size: *mut unreachable() } +#[no_mangle] +pub extern "C" fn args_get(args: *mut *mut u8, args_buf: *mut u8) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn args_sizes_get(argc: *mut Size, arg_buf_size: *mut Size) -> Errno { + unreachable() +} + +#[no_mangle] +pub extern "C" fn clock_time_get( + clockid: Clockid, + precision: Timestamp, + out: *mut Timestamp, +) -> Errno { + unreachable() +} + #[no_mangle] pub extern "C" fn fd_write( fd: Fd, From 31b737c7cce9ae31248666f9a6afd81999ccbc1e Mon Sep 17 00:00:00 2001 From: Pat Hickey Date: Wed, 5 Oct 2022 14:55:49 -0700 Subject: [PATCH 11/11] test-helpers/build.rs: test encoding teavm guests as a component as well --- crates/test-helpers/build.rs | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/crates/test-helpers/build.rs b/crates/test-helpers/build.rs index 930c87410..1e6149296 100644 --- a/crates/test-helpers/build.rs +++ b/crates/test-helpers/build.rs @@ -251,15 +251,35 @@ fn main() { panic!("failed to build"); } + let out_wasm = out_dir.join("target/generated/wasm/teavm-wasm/classes.wasm"); + wasms.push(( "java", test_dir.file_stem().unwrap().to_str().unwrap().to_string(), - out_dir - .join("target/generated/wasm/teavm-wasm/classes.wasm") - .to_str() - .unwrap() - .to_string(), + out_wasm.to_str().unwrap().to_string(), )); + + let imports = [Interface::parse_file(test_dir.join("imports.wit")).unwrap()]; + let interface = Interface::parse_file(test_dir.join("exports.wit")).unwrap(); + + // Validate that the module can be translated to a component, using + // wit interfaces explicitly passed to ComponentEncoder, because the + // TeaVM guest doesnt yet support putting component types into custom + // sections. + let module = fs::read(&out_wasm).expect("failed to read wasm file"); + ComponentEncoder::default() + .imports(&imports) + .interface(&interface) + .module(module.as_slice()) + .expect("pull custom sections from module") + .validate(true) + .adapter_file(&wasi_adapter) + .expect("adapter failed to get loaded") + .encode() + .expect(&format!( + "module {:?} can be translated to a component", + out_wasm + )); } }