From f0af35a5c246c6ec01e8d9dcf72b0c24df48cf98 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 27 Mar 2024 17:58:48 -0700 Subject: [PATCH 1/5] Add a `compile` feature to `wasmtime-environ` This commit adds a compile-time feature to remove some dependencies of the `wasmtime-environ` crate. This compiles out support for compiling modules/components and makes the crate slimmer in terms of amount of code compiled along with its dependencies. Much of this should already have been statically removed by native linkers so this likely won't have any compile-size impact, but it's a nice-to-have in terms of organization. This has a fair bit of shuffling around of code, but apart from renamings and movement there are no major changes here. --- crates/cranelift/Cargo.toml | 2 +- crates/environ/Cargo.toml | 5 +- crates/environ/src/address_map.rs | 68 +--- crates/environ/src/compile/address_map.rs | 72 ++++ .../src/{compilation.rs => compile/mod.rs} | 71 +--- .../environ/src/compile/module_artifacts.rs | 300 +++++++++++++++ crates/environ/src/compile/trap_encoding.rs | 69 ++++ crates/environ/src/component.rs | 16 +- crates/environ/src/component/artifacts.rs | 29 +- crates/environ/src/component/compiler.rs | 30 +- crates/environ/src/component/info.rs | 77 +++- crates/environ/src/fact.rs | 4 +- crates/environ/src/fact/trampoline.rs | 10 +- crates/environ/src/fact/transcode.rs | 81 +---- crates/environ/src/lib.rs | 12 +- crates/environ/src/module_artifacts.rs | 342 +++--------------- crates/environ/src/trap_encoding.rs | 69 +--- crates/wasmtime/src/compile.rs | 5 + crates/wasmtime/src/compile/code_builder.rs | 102 +----- crates/wasmtime/src/compile/runtime.rs | 175 +++++++++ crates/wasmtime/src/config.rs | 5 +- crates/wasmtime/src/engine.rs | 1 + crates/wasmtime/src/engine/serialization.rs | 2 + crates/wasmtime/src/runtime/instantiate.rs | 86 +---- .../wasmtime/src/runtime/trampoline/func.rs | 2 +- 25 files changed, 834 insertions(+), 801 deletions(-) create mode 100644 crates/environ/src/compile/address_map.rs rename crates/environ/src/{compilation.rs => compile/mod.rs} (88%) create mode 100644 crates/environ/src/compile/module_artifacts.rs create mode 100644 crates/environ/src/compile/trap_encoding.rs create mode 100644 crates/wasmtime/src/compile/runtime.rs diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index ad2aa057a195..47a1a86360cd 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -16,7 +16,7 @@ workspace = true [dependencies] anyhow = { workspace = true } log = { workspace = true } -wasmtime-environ = { workspace = true } +wasmtime-environ = { workspace = true, features = ['compile'] } cranelift-wasm = { workspace = true } cranelift-codegen = { workspace = true, features = ["host-arch"] } cranelift-frontend = { workspace = true } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 50093112220b..39e536fc0c07 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -25,8 +25,8 @@ thiserror = { workspace = true } serde = "1.0.188" serde_derive = "1.0.188" log = { workspace = true } -gimli = { workspace = true, features = ["write"] } -object = { workspace = true, features = ['write_core'] } +gimli = { workspace = true } +object = { workspace = true } rustc-demangle = { version = "0.1.16", optional = true } target-lexicon = { workspace = true } wasm-encoder = { workspace = true, optional = true } @@ -50,3 +50,4 @@ component-model = [ ] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] gc = [] +compile = ['gimli/write', 'object/write_core'] diff --git a/crates/environ/src/address_map.rs b/crates/environ/src/address_map.rs index d8faba78a973..a707d69b4708 100644 --- a/crates/environ/src/address_map.rs +++ b/crates/environ/src/address_map.rs @@ -1,10 +1,7 @@ //! Data structures to provide transformation of the source -use crate::obj::ELF_WASMTIME_ADDRMAP; -use object::write::{Object, StandardSegment}; -use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; +use object::{Bytes, LittleEndian, U32Bytes}; use serde_derive::{Deserialize, Serialize}; -use std::ops::Range; /// Single source location to generated address mapping. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -52,69 +49,6 @@ impl Default for FilePos { } } -/// Builder for the address map section of a wasmtime compilation image. -/// -/// This builder is used to conveniently built the `ELF_WASMTIME_ADDRMAP` -/// section by compilers, and provides utilities to directly insert the results -/// into an `Object`. -#[derive(Default)] -pub struct AddressMapSection { - offsets: Vec>, - positions: Vec>, - last_offset: u32, -} - -impl AddressMapSection { - /// Pushes a new set of instruction mapping information for a function added - /// in the exectuable. - /// - /// The `func` argument here is the range of the function, relative to the - /// start of the text section in the executable. The `instrs` provided are - /// the descriptors for instructions in the function and their various - /// mappings back to original source positions. - /// - /// This is required to be called for `func` values that are strictly - /// increasing in addresses (e.g. as the object is built). Additionally the - /// `instrs` map must be sorted based on code offset in the native text - /// section. - pub fn push(&mut self, func: Range, instrs: &[InstructionAddressMap]) { - // NB: for now this only supports <=4GB text sections in object files. - // Alternative schemes will need to be created for >32-bit offsets to - // avoid making this section overly large. - let func_start = u32::try_from(func.start).unwrap(); - let func_end = u32::try_from(func.end).unwrap(); - - self.offsets.reserve(instrs.len()); - self.positions.reserve(instrs.len()); - for map in instrs { - // Sanity-check to ensure that functions are pushed in-order, otherwise - // the `offsets` array won't be sorted which is our goal. - let pos = func_start + map.code_offset; - assert!(pos >= self.last_offset); - self.offsets.push(U32Bytes::new(LittleEndian, pos)); - self.positions - .push(U32Bytes::new(LittleEndian, map.srcloc.0)); - self.last_offset = pos; - } - self.last_offset = func_end; - } - - /// Finishes encoding this section into the `Object` provided. - pub fn append_to(self, obj: &mut Object) { - let section = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASMTIME_ADDRMAP.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - - // NB: this matches the encoding expected by `lookup` below. - let amt = u32::try_from(self.offsets.len()).unwrap(); - obj.append_section_data(section, &amt.to_le_bytes(), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.positions), 1); - } -} - /// Parse an `ELF_WASMTIME_ADDRMAP` section, returning the slice of code offsets /// and the slice of associated file positions for each offset. fn parse_address_map( diff --git a/crates/environ/src/compile/address_map.rs b/crates/environ/src/compile/address_map.rs new file mode 100644 index 000000000000..614ccc3cbc94 --- /dev/null +++ b/crates/environ/src/compile/address_map.rs @@ -0,0 +1,72 @@ +//! Data structures to provide transformation of the source + +use crate::obj::ELF_WASMTIME_ADDRMAP; +use crate::InstructionAddressMap; +use object::write::{Object, StandardSegment}; +use object::{LittleEndian, SectionKind, U32Bytes}; +use std::ops::Range; + +/// Builder for the address map section of a wasmtime compilation image. +/// +/// This builder is used to conveniently built the `ELF_WASMTIME_ADDRMAP` +/// section by compilers, and provides utilities to directly insert the results +/// into an `Object`. +#[derive(Default)] +pub struct AddressMapSection { + offsets: Vec>, + positions: Vec>, + last_offset: u32, +} + +impl AddressMapSection { + /// Pushes a new set of instruction mapping information for a function added + /// in the exectuable. + /// + /// The `func` argument here is the range of the function, relative to the + /// start of the text section in the executable. The `instrs` provided are + /// the descriptors for instructions in the function and their various + /// mappings back to original source positions. + /// + /// This is required to be called for `func` values that are strictly + /// increasing in addresses (e.g. as the object is built). Additionally the + /// `instrs` map must be sorted based on code offset in the native text + /// section. + pub fn push(&mut self, func: Range, instrs: &[InstructionAddressMap]) { + // NB: for now this only supports <=4GB text sections in object files. + // Alternative schemes will need to be created for >32-bit offsets to + // avoid making this section overly large. + let func_start = u32::try_from(func.start).unwrap(); + let func_end = u32::try_from(func.end).unwrap(); + + self.offsets.reserve(instrs.len()); + self.positions.reserve(instrs.len()); + for map in instrs { + // Sanity-check to ensure that functions are pushed in-order, otherwise + // the `offsets` array won't be sorted which is our goal. + let pos = func_start + map.code_offset; + assert!(pos >= self.last_offset); + self.offsets.push(U32Bytes::new(LittleEndian, pos)); + self.positions.push(U32Bytes::new( + LittleEndian, + map.srcloc.file_offset().unwrap_or(u32::MAX), + )); + self.last_offset = pos; + } + self.last_offset = func_end; + } + + /// Finishes encoding this section into the `Object` provided. + pub fn append_to(self, obj: &mut Object) { + let section = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + ELF_WASMTIME_ADDRMAP.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + + // NB: this matches the encoding expected by `lookup` below. + let amt = u32::try_from(self.offsets.len()).unwrap(); + obj.append_section_data(section, &amt.to_le_bytes(), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.positions), 1); + } +} diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compile/mod.rs similarity index 88% rename from crates/environ/src/compilation.rs rename to crates/environ/src/compile/mod.rs index b3c5863fa68c..993a6c6cddd1 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compile/mod.rs @@ -3,13 +3,13 @@ use crate::{obj, Tunables}; use crate::{ - BuiltinFunctionIndex, DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData, - ModuleTranslation, ModuleTypesBuilder, PrimaryMap, StackMap, WasmError, WasmFuncType, + BuiltinFunctionIndex, DefinedFuncIndex, FlagValue, FuncIndex, FunctionBodyData, FunctionLoc, + ModuleTranslation, ModuleTypesBuilder, ObjectKind, PrimaryMap, WasmError, WasmFuncType, + WasmFunctionInfo, }; use anyhow::Result; use object::write::{Object, SymbolId}; use object::{Architecture, BinaryFormat, FileFlags}; -use serde_derive::{Deserialize, Serialize}; use std::any::Any; use std::borrow::Cow; use std::fmt; @@ -17,37 +17,13 @@ use std::path; use std::sync::Arc; use thiserror::Error; -/// Information about a function, such as trap information, address map, -/// and stack maps. -#[derive(Serialize, Deserialize, Default)] -#[allow(missing_docs)] -pub struct WasmFunctionInfo { - pub start_srcloc: FilePos, - pub stack_maps: Box<[StackMapInformation]>, -} - -/// Description of where a function is located in the text section of a -/// compiled image. -#[derive(Copy, Clone, Serialize, Deserialize)] -pub struct FunctionLoc { - /// The byte offset from the start of the text section where this - /// function starts. - pub start: u32, - /// The byte length of this function's function body. - pub length: u32, -} +mod address_map; +mod module_artifacts; +mod trap_encoding; -/// The offset within a function of a GC safepoint, and its associated stack -/// map. -#[derive(Serialize, Deserialize, Debug)] -pub struct StackMapInformation { - /// The offset of the GC safepoint within the function's native code. It is - /// relative to the beginning of the function. - pub code_offset: u32, - - /// The stack map for identifying live GC refs at the GC safepoint. - pub stack_map: StackMap, -} +pub use self::address_map::*; +pub use self::module_artifacts::*; +pub use self::trap_encoding::*; /// An error while compiling WebAssembly to machine code. #[derive(Error, Debug)] @@ -171,14 +147,6 @@ pub enum SettingKind { Preset, } -/// Types of objects that can be created by `Compiler::object` -pub enum ObjectKind { - /// A core wasm compilation artifact - Module, - /// A component compilation artifact - Component, -} - /// An implementation of a compiler which can compile WebAssembly functions to /// machine code and perform other miscellaneous tasks needed by the JIT runtime. pub trait Compiler: Send + Sync { @@ -419,24 +387,3 @@ pub trait Compiler: Send + Sync { None } } - -/// Value of a configured setting for a [`Compiler`] -#[derive(Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] -pub enum FlagValue<'a> { - /// Name of the value that has been configured for this setting. - Enum(&'a str), - /// The numerical value of the configured settings. - Num(u8), - /// Whether the setting is on or off. - Bool(bool), -} - -impl fmt::Display for FlagValue<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::Enum(v) => v.fmt(f), - Self::Num(v) => v.fmt(f), - Self::Bool(v) => v.fmt(f), - } - } -} diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs new file mode 100644 index 000000000000..7210f9139c62 --- /dev/null +++ b/crates/environ/src/compile/module_artifacts.rs @@ -0,0 +1,300 @@ +//! Definitions of runtime structures and metadata which are serialized into ELF +//! with `bincode` as part of a module's compilation process. + +use crate::{ + obj, CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FunctionLoc, FunctionName, + MemoryInitialization, Metadata, ModuleTranslation, PrimaryMap, Tunables, +}; +use anyhow::{bail, Result}; +use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; +use object::SectionKind; +use std::ops::Range; +use wasmtime_types::ModuleInternedTypeIndex; + +/// Helper structure to create an ELF file as a compilation artifact. +/// +/// This structure exposes the process which Wasmtime will encode a core wasm +/// module into an ELF file, notably managing data sections and all that good +/// business going into the final file. +pub struct ObjectBuilder<'a> { + /// The `object`-crate-defined ELF file write we're using. + obj: Object<'a>, + + /// General compilation configuration. + tunables: &'a Tunables, + + /// The section identifier for "rodata" which is where wasm data segments + /// will go. + data: SectionId, + + /// The section identifier for function name information, or otherwise where + /// the `name` custom section of wasm is copied into. + /// + /// This is optional and lazily created on demand. + names: Option, + + /// The section identifier for dwarf information copied from the original + /// wasm files. + /// + /// This is optional and lazily created on demand. + dwarf: Option, +} + +impl<'a> ObjectBuilder<'a> { + /// Creates a new builder for the `obj` specified. + pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { + let data = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_WASM_DATA.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + ObjectBuilder { + obj, + tunables, + data, + names: None, + dwarf: None, + } + } + + /// Completes compilation of the `translation` specified, inserting + /// everything necessary into the `Object` being built. + /// + /// This function will consume the final results of compiling a wasm module + /// and finish the ELF image in-progress as part of `self.obj` by appending + /// any compiler-agnostic sections. + /// + /// The auxiliary `CompiledModuleInfo` structure returned here has also been + /// serialized into the object returned, but if the caller will quickly + /// turn-around and invoke `CompiledModule::from_artifacts` after this then + /// the information can be passed to that method to avoid extra + /// deserialization. This is done to avoid a serialize-then-deserialize for + /// API calls like `Module::new` where the compiled module is immediately + /// going to be used. + /// + /// The various arguments here are: + /// + /// * `translation` - the core wasm translation that's being completed. + /// + /// * `funcs` - compilation metadata about functions within the translation + /// as well as where the functions are located in the text section and any + /// associated trampolines. + /// + /// * `wasm_to_native_trampolines` - list of all trampolines necessary for + /// Wasm callers calling native callees (e.g. `Func::wrap`). One for each + /// function signature in the module. Must be sorted by `SignatureIndex`. + /// + /// Returns the `CompiledModuleInfo` corresponding to this core Wasm module + /// as a result of this append operation. This is then serialized into the + /// final artifact by the caller. + pub fn append( + &mut self, + translation: ModuleTranslation<'_>, + funcs: PrimaryMap, + wasm_to_native_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>, + ) -> Result { + let ModuleTranslation { + mut module, + debuginfo, + has_unparsed_debuginfo, + data, + data_align, + passive_data, + .. + } = translation; + + // Place all data from the wasm module into a section which will the + // source of the data later at runtime. This additionally keeps track of + // the offset of + let mut total_data_len = 0; + let data_offset = self + .obj + .append_section_data(self.data, &[], data_align.unwrap_or(1)); + for (i, data) in data.iter().enumerate() { + // The first data segment has its alignment specified as the alignment + // for the entire section, but everything afterwards is adjacent so it + // has alignment of 1. + let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; + self.obj.append_section_data(self.data, data, align); + total_data_len += data.len(); + } + for data in passive_data.iter() { + self.obj.append_section_data(self.data, data, 1); + } + + // If any names are present in the module then the `ELF_NAME_DATA` section + // is create and appended. + let mut func_names = Vec::new(); + if debuginfo.name_section.func_names.len() > 0 { + let name_id = *self.names.get_or_insert_with(|| { + self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_NAME_DATA.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ) + }); + let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); + sorted_names.sort_by_key(|(idx, _name)| *idx); + for (idx, name) in sorted_names { + let offset = self.obj.append_section_data(name_id, name.as_bytes(), 1); + let offset = match u32::try_from(offset) { + Ok(offset) => offset, + Err(_) => bail!("name section too large (> 4gb)"), + }; + let len = u32::try_from(name.len()).unwrap(); + func_names.push(FunctionName { + idx: *idx, + offset, + len, + }); + } + } + + // Data offsets in `MemoryInitialization` are offsets within the + // `translation.data` list concatenated which is now present in the data + // segment that's appended to the object. Increase the offsets by + // `self.data_size` to account for any previously added module. + let data_offset = u32::try_from(data_offset).unwrap(); + match &mut module.memory_initialization { + MemoryInitialization::Segmented(list) => { + for segment in list { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + MemoryInitialization::Static { map } => { + for (_, segment) in map { + if let Some(segment) = segment { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + } + } + + // Data offsets for passive data are relative to the start of + // `translation.passive_data` which was appended to the data segment + // of this object, after active data in `translation.data`. Update the + // offsets to account prior modules added in addition to active data. + let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); + for (_, range) in module.passive_data_map.iter_mut() { + range.start = range.start.checked_add(data_offset).unwrap(); + range.end = range.end.checked_add(data_offset).unwrap(); + } + + // Insert the wasm raw wasm-based debuginfo into the output, if + // requested. Note that this is distinct from the native debuginfo + // possibly generated by the native compiler, hence these sections + // getting wasm-specific names. + let mut dwarf = Vec::new(); + if self.tunables.parse_wasm_debuginfo { + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_abbrev); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_addr); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_aranges); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_info); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line_str); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str_offsets); + self.push_debug(&mut dwarf, &debuginfo.debug_ranges); + self.push_debug(&mut dwarf, &debuginfo.debug_rnglists); + } + // Sort this for binary-search-lookup later in `symbolize_context`. + dwarf.sort_by_key(|(id, _)| *id); + + Ok(CompiledModuleInfo { + module, + funcs, + wasm_to_native_trampolines, + func_names, + meta: Metadata { + native_debug_info_present: self.tunables.generate_native_debuginfo, + has_unparsed_debuginfo, + code_section_offset: debuginfo.wasm_file.code_section_offset, + has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, + dwarf, + }, + }) + } + + fn push_debug<'b, T>(&mut self, dwarf: &mut Vec<(u8, Range)>, section: &T) + where + T: gimli::Section>, + { + let data = section.reader().slice(); + if data.is_empty() { + return; + } + let section_id = *self.dwarf.get_or_insert_with(|| { + self.obj.add_section( + self.obj.segment_name(StandardSegment::Debug).to_vec(), + obj::ELF_WASMTIME_DWARF.as_bytes().to_vec(), + SectionKind::Debug, + ) + }); + let offset = self.obj.append_section_data(section_id, data, 1); + dwarf.push((T::id() as u8, offset..offset + data.len() as u64)); + } + + /// Creates the `ELF_WASMTIME_INFO` section from the given serializable data + /// structure. + pub fn serialize_info(&mut self, info: &T) + where + T: serde::Serialize, + { + let section = self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_WASMTIME_INFO.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + let data = bincode::serialize(info).unwrap(); + self.obj.set_section_data(section, data, 1); + } + + /// Serializes `self` into a buffer. This can be used for execution as well + /// as serialization. + pub fn finish(self, t: &mut T) -> Result<()> { + self.obj.emit(t).map_err(|e| e.into()) + } +} + +/// A type which can be the result of serializing an object. +pub trait FinishedObject: Sized { + /// Emit the object as `Self`. + fn finish_object(obj: ObjectBuilder<'_>) -> Result; +} + +impl FinishedObject for Vec { + fn finish_object(obj: ObjectBuilder<'_>) -> Result { + let mut result = ObjectVec::default(); + obj.finish(&mut result)?; + return Ok(result.0); + + #[derive(Default)] + struct ObjectVec(Vec); + + impl WritableBuffer for ObjectVec { + fn len(&self) -> usize { + self.0.len() + } + + fn reserve(&mut self, additional: usize) -> Result<(), ()> { + assert_eq!(self.0.len(), 0, "cannot reserve twice"); + self.0 = Vec::with_capacity(additional); + Ok(()) + } + + fn resize(&mut self, new_len: usize) { + if new_len <= self.0.len() { + self.0.truncate(new_len) + } else { + self.0.extend(vec![0; new_len - self.0.len()]) + } + } + + fn write_bytes(&mut self, val: &[u8]) { + self.0.extend(val); + } + } + } +} diff --git a/crates/environ/src/compile/trap_encoding.rs b/crates/environ/src/compile/trap_encoding.rs new file mode 100644 index 000000000000..7f38523f1721 --- /dev/null +++ b/crates/environ/src/compile/trap_encoding.rs @@ -0,0 +1,69 @@ +use crate::obj::ELF_WASMTIME_TRAPS; +use crate::TrapInformation; +use object::write::{Object, StandardSegment}; +use object::{LittleEndian, SectionKind, U32Bytes}; +use std::ops::Range; + +/// A helper structure to build the custom-encoded section of a wasmtime +/// compilation image which encodes trap information. +/// +/// This structure is incrementally fed the results of compiling individual +/// functions and handles all the encoding internally, allowing usage of +/// `lookup_trap_code` below with the resulting section. +#[derive(Default)] +pub struct TrapEncodingBuilder { + offsets: Vec>, + traps: Vec, + last_offset: u32, +} + +impl TrapEncodingBuilder { + /// Appends trap information about a function into this section. + /// + /// This function is called to describe traps for the `func` range + /// specified. The `func` offsets are specified relative to the text section + /// itself, and the `traps` offsets are specified relative to the start of + /// `func`. + /// + /// This is required to be called in-order for increasing ranges of `func` + /// to ensure the final array is properly sorted. Additionally `traps` must + /// be sorted. + pub fn push(&mut self, func: Range, traps: &[TrapInformation]) { + // NB: for now this only supports <=4GB text sections in object files. + // Alternative schemes will need to be created for >32-bit offsets to + // avoid making this section overly large. + let func_start = u32::try_from(func.start).unwrap(); + let func_end = u32::try_from(func.end).unwrap(); + + // Sanity-check to ensure that functions are pushed in-order, otherwise + // the `offsets` array won't be sorted which is our goal. + assert!(func_start >= self.last_offset); + + self.offsets.reserve(traps.len()); + self.traps.reserve(traps.len()); + for info in traps { + let pos = func_start + info.code_offset; + assert!(pos >= self.last_offset); + self.offsets.push(U32Bytes::new(LittleEndian, pos)); + self.traps.push(info.trap_code as u8); + self.last_offset = pos; + } + + self.last_offset = func_end; + } + + /// Encodes this section into the object provided. + pub fn append_to(self, obj: &mut Object) { + let section = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + ELF_WASMTIME_TRAPS.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + + // NB: this matches the encoding expected by `lookup` below. + let amt = u32::try_from(self.traps.len()).unwrap(); + obj.append_section_data(section, &amt.to_le_bytes(), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); + obj.append_section_data(section, &self.traps, 1); + } +} diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rs index 4ec045a1e55c..67dce36814c0 100644 --- a/crates/environ/src/component.rs +++ b/crates/environ/src/component.rs @@ -38,19 +38,25 @@ pub const MAX_FLAT_PARAMS: usize = 16; pub const MAX_FLAT_RESULTS: usize = 1; mod artifacts; -mod compiler; -pub mod dfg; mod info; -mod translate; mod types; mod vmcomponent_offsets; pub use self::artifacts::*; -pub use self::compiler::*; pub use self::info::*; -pub use self::translate::*; pub use self::types::*; pub use self::vmcomponent_offsets::*; +#[cfg(feature = "compile")] +mod compiler; +#[cfg(feature = "compile")] +pub mod dfg; +#[cfg(feature = "compile")] +mod translate; +#[cfg(feature = "compile")] +pub use self::compiler::*; +#[cfg(feature = "compile")] +pub use self::translate::*; + /// Helper macro to iterate over the transcoders that the host will provide /// adapter modules through libcalls. #[macro_export] diff --git a/crates/environ/src/component/artifacts.rs b/crates/environ/src/component/artifacts.rs index 3acafbd30a1b..d6cd79b73b4d 100644 --- a/crates/environ/src/component/artifacts.rs +++ b/crates/environ/src/component/artifacts.rs @@ -2,7 +2,7 @@ //! which are serialized with `bincode` into output ELF files. use crate::{ - component::{AllCallFunc, Component, ComponentTypes, TrampolineIndex, TypeComponentIndex}, + component::{Component, ComponentTypes, TrampolineIndex, TypeComponentIndex}, CompiledModuleInfo, FunctionLoc, PrimaryMap, StaticModuleIndex, }; use serde_derive::{Deserialize, Serialize}; @@ -43,3 +43,30 @@ pub struct CompiledComponentInfo { /// intrinsic. pub resource_drop_wasm_to_native_trampoline: Option, } + +/// A triple of related functions/trampolines variants with differing calling +/// conventions: `{wasm,array,native}_call`. +/// +/// Generic so we can use this with either the `Box`s that +/// implementations of the compiler trait return or with `FunctionLoc`s inside +/// an object file, for example. +#[derive(Clone, Copy, Default, Serialize, Deserialize)] +pub struct AllCallFunc { + /// The function exposing the Wasm calling convention. + pub wasm_call: T, + /// The function exposing the array calling convention. + pub array_call: T, + /// The function exposing the native calling convention. + pub native_call: T, +} + +impl AllCallFunc { + /// Map an `AllCallFunc` into an `AllCallFunc`. + pub fn map(self, mut f: impl FnMut(T) -> U) -> AllCallFunc { + AllCallFunc { + wasm_call: f(self.wasm_call), + array_call: f(self.array_call), + native_call: f(self.native_call), + } + } +} diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs index b89f67a77fc3..ee0fe41a6eeb 100644 --- a/crates/environ/src/component/compiler.rs +++ b/crates/environ/src/component/compiler.rs @@ -1,35 +1,7 @@ -use crate::component::{ComponentTranslation, ComponentTypesBuilder, TrampolineIndex}; +use crate::component::{AllCallFunc, ComponentTranslation, ComponentTypesBuilder, TrampolineIndex}; use anyhow::Result; -use serde_derive::{Deserialize, Serialize}; use std::any::Any; -/// A triple of related functions/trampolines variants with differing calling -/// conventions: `{wasm,array,native}_call`. -/// -/// Generic so we can use this with either the `Box`s that -/// implementations of the compiler trait return or with `FunctionLoc`s inside -/// an object file, for example. -#[derive(Clone, Copy, Default, Serialize, Deserialize)] -pub struct AllCallFunc { - /// The function exposing the Wasm calling convention. - pub wasm_call: T, - /// The function exposing the array calling convention. - pub array_call: T, - /// The function exposing the native calling convention. - pub native_call: T, -} - -impl AllCallFunc { - /// Map an `AllCallFunc` into an `AllCallFunc`. - pub fn map(self, mut f: impl FnMut(T) -> U) -> AllCallFunc { - AllCallFunc { - wasm_call: f(self.wasm_call), - array_call: f(self.array_call), - native_call: f(self.native_call), - } - } -} - /// Compilation support necessary for components. pub trait ComponentCompiler: Send + Sync { /// Compiles the pieces necessary to create a `VMFuncRef` for the diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index fc47dbc3082c..1f42330f0175 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -455,7 +455,82 @@ pub enum StringEncoding { CompactUtf16, } -pub use crate::fact::{FixedEncoding, Transcode}; +/// Possible transcoding operations that must be provided by the host. +/// +/// Note that each transcoding operation may have a unique signature depending +/// on the precise operation. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] +pub enum Transcode { + Copy(FixedEncoding), + Latin1ToUtf16, + Latin1ToUtf8, + Utf16ToCompactProbablyUtf16, + Utf16ToCompactUtf16, + Utf16ToLatin1, + Utf16ToUtf8, + Utf8ToCompactUtf16, + Utf8ToLatin1, + Utf8ToUtf16, +} + +impl Transcode { + /// Get this transcoding's symbol fragment. + pub fn symbol_fragment(&self) -> &'static str { + match self { + Transcode::Copy(x) => match x { + FixedEncoding::Utf8 => "copy_utf8", + FixedEncoding::Utf16 => "copy_utf16", + FixedEncoding::Latin1 => "copy_latin1", + }, + Transcode::Latin1ToUtf16 => "latin1_to_utf16", + Transcode::Latin1ToUtf8 => "latin1_to_utf8", + Transcode::Utf16ToCompactProbablyUtf16 => "utf16_to_compact_probably_utf16", + Transcode::Utf16ToCompactUtf16 => "utf16_to_compact_utf16", + Transcode::Utf16ToLatin1 => "utf16_to_latin1", + Transcode::Utf16ToUtf8 => "utf16_to_utf8", + Transcode::Utf8ToCompactUtf16 => "utf8_to_compact_utf16", + Transcode::Utf8ToLatin1 => "utf8_to_latin1", + Transcode::Utf8ToUtf16 => "utf8_to_utf16", + } + } + + /// Returns a human-readable description for this transcoding operation. + pub fn desc(&self) -> &'static str { + match self { + Transcode::Copy(FixedEncoding::Utf8) => "utf8-to-utf8", + Transcode::Copy(FixedEncoding::Utf16) => "utf16-to-utf16", + Transcode::Copy(FixedEncoding::Latin1) => "latin1-to-latin1", + Transcode::Latin1ToUtf16 => "latin1-to-utf16", + Transcode::Latin1ToUtf8 => "latin1-to-utf8", + Transcode::Utf16ToCompactProbablyUtf16 => "utf16-to-compact-probably-utf16", + Transcode::Utf16ToCompactUtf16 => "utf16-to-compact-utf16", + Transcode::Utf16ToLatin1 => "utf16-to-latin1", + Transcode::Utf16ToUtf8 => "utf16-to-utf8", + Transcode::Utf8ToCompactUtf16 => "utf8-to-compact-utf16", + Transcode::Utf8ToLatin1 => "utf8-to-latin1", + Transcode::Utf8ToUtf16 => "utf8-to-utf16", + } + } +} + +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum FixedEncoding { + Utf8, + Utf16, + Latin1, +} + +impl FixedEncoding { + pub(crate) fn width(&self) -> u8 { + match self { + FixedEncoding::Utf8 => 1, + FixedEncoding::Utf16 => 2, + FixedEncoding::Latin1 => 1, + } + } +} /// Description of a new resource declared in a `GlobalInitializer::Resource` /// variant. diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index 70437996c996..0ad0b119d899 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -21,7 +21,7 @@ use crate::component::dfg::CoreDef; use crate::component::{ Adapter, AdapterOptions as AdapterOptionsDfg, ComponentTypesBuilder, FlatType, InterfaceType, - StringEncoding, TypeFuncIndex, + StringEncoding, Transcode, TypeFuncIndex, }; use crate::fact::transcode::Transcoder; use crate::{EntityRef, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap}; @@ -35,8 +35,6 @@ mod trampoline; mod transcode; mod traps; -pub use self::transcode::{FixedEncoding, Transcode}; - /// Representation of an adapter module. pub struct Module<'a> { /// Whether or not debug code is inserted into the adapters themselves. diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index f85a60c6dc45..a71ffacb1e5c 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -16,13 +16,13 @@ //! can be somewhat arbitrary, an intentional decision. use crate::component::{ - CanonicalAbiInfo, ComponentTypesBuilder, FlatType, InterfaceType, StringEncoding, - TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex, - TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, VariantInfo, - FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, + CanonicalAbiInfo, ComponentTypesBuilder, FixedEncoding as FE, FlatType, InterfaceType, + StringEncoding, Transcode, TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, + TypeRecordIndex, TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, + VariantInfo, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, }; use crate::fact::signature::Signature; -use crate::fact::transcode::{FixedEncoding as FE, Transcode, Transcoder}; +use crate::fact::transcode::Transcoder; use crate::fact::traps::Trap; use crate::fact::{ AdapterData, Body, Context, Function, FunctionId, Helper, HelperLocation, HelperType, Module, diff --git a/crates/environ/src/fact/transcode.rs b/crates/environ/src/fact/transcode.rs index ca34a8583156..e41b45c1ec91 100644 --- a/crates/environ/src/fact/transcode.rs +++ b/crates/environ/src/fact/transcode.rs @@ -1,6 +1,6 @@ +use crate::component::Transcode; use crate::fact::core_types::CoreTypes; use crate::MemoryIndex; -use serde_derive::{Deserialize, Serialize}; use wasm_encoder::{EntityType, ValType}; #[derive(Copy, Clone, Hash, Eq, PartialEq)] @@ -12,55 +12,6 @@ pub struct Transcoder { pub op: Transcode, } -/// Possible transcoding operations that must be provided by the host. -/// -/// Note that each transcoding operation may have a unique signature depending -/// on the precise operation. -#[allow(missing_docs)] -#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] -pub enum Transcode { - Copy(FixedEncoding), - Latin1ToUtf16, - Latin1ToUtf8, - Utf16ToCompactProbablyUtf16, - Utf16ToCompactUtf16, - Utf16ToLatin1, - Utf16ToUtf8, - Utf8ToCompactUtf16, - Utf8ToLatin1, - Utf8ToUtf16, -} - -impl Transcode { - /// Get this transcoding's symbol fragment. - pub fn symbol_fragment(&self) -> &'static str { - match self { - Transcode::Copy(x) => match x { - FixedEncoding::Utf8 => "copy_utf8", - FixedEncoding::Utf16 => "copy_utf16", - FixedEncoding::Latin1 => "copy_latin1", - }, - Transcode::Latin1ToUtf16 => "latin1_to_utf16", - Transcode::Latin1ToUtf8 => "latin1_to_utf8", - Transcode::Utf16ToCompactProbablyUtf16 => "utf16_to_compact_probably_utf16", - Transcode::Utf16ToCompactUtf16 => "utf16_to_compact_utf16", - Transcode::Utf16ToLatin1 => "utf16_to_latin1", - Transcode::Utf16ToUtf8 => "utf16_to_utf8", - Transcode::Utf8ToCompactUtf16 => "utf8_to_compact_utf16", - Transcode::Utf8ToLatin1 => "utf8_to_latin1", - Transcode::Utf8ToUtf16 => "utf8_to_utf16", - } - } -} - -#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] -#[allow(missing_docs)] -pub enum FixedEncoding { - Utf8, - Utf16, - Latin1, -} - impl Transcoder { pub fn name(&self) -> String { format!( @@ -136,33 +87,3 @@ impl Transcoder { EntityType::Function(ty) } } - -impl Transcode { - /// Returns a human-readable description for this transcoding operation. - pub fn desc(&self) -> &'static str { - match self { - Transcode::Copy(FixedEncoding::Utf8) => "utf8-to-utf8", - Transcode::Copy(FixedEncoding::Utf16) => "utf16-to-utf16", - Transcode::Copy(FixedEncoding::Latin1) => "latin1-to-latin1", - Transcode::Latin1ToUtf16 => "latin1-to-utf16", - Transcode::Latin1ToUtf8 => "latin1-to-utf8", - Transcode::Utf16ToCompactProbablyUtf16 => "utf16-to-compact-probably-utf16", - Transcode::Utf16ToCompactUtf16 => "utf16-to-compact-utf16", - Transcode::Utf16ToLatin1 => "utf16-to-latin1", - Transcode::Utf16ToUtf8 => "utf16-to-utf8", - Transcode::Utf8ToCompactUtf16 => "utf8-to-compact-utf16", - Transcode::Utf8ToLatin1 => "utf8-to-latin1", - Transcode::Utf8ToUtf16 => "utf8-to-utf16", - } - } -} - -impl FixedEncoding { - pub(crate) fn width(&self) -> u8 { - match self { - FixedEncoding::Utf8 => 1, - FixedEncoding::Utf16 => 2, - FixedEncoding::Latin1 => 1, - } - } -} diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 4df17658a11f..97c748359353 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -8,7 +8,6 @@ mod address_map; mod builtin; -mod compilation; mod demangling; mod module; mod module_artifacts; @@ -24,9 +23,9 @@ mod vmoffsets; pub use crate::address_map::*; pub use crate::builtin::*; -pub use crate::compilation::*; pub use crate::demangling::*; pub use crate::module::*; +pub use crate::module_artifacts::*; pub use crate::module_environ::*; pub use crate::module_types::*; pub use crate::ref_bits::*; @@ -37,13 +36,14 @@ pub use crate::tunables::*; pub use crate::vmoffsets::*; pub use object; -pub use crate::module_artifacts::{ - CompiledFunctionInfo, CompiledModuleInfo, FinishedObject, FunctionName, Metadata, ObjectBuilder, -}; +#[cfg(feature = "compile")] +mod compile; +#[cfg(feature = "compile")] +pub use crate::compile::*; #[cfg(feature = "component-model")] pub mod component; -#[cfg(feature = "component-model")] +#[cfg(all(feature = "component-model", feature = "compile"))] pub mod fact; // Reexport all of these type-level since they're quite commonly used and it's diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 65b38b2de030..aa1e739519b9 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -1,14 +1,9 @@ //! Definitions of runtime structures and metadata which are serialized into ELF //! with `bincode` as part of a module's compilation process. -use crate::{ - obj, DefinedFuncIndex, FuncIndex, FunctionLoc, MemoryInitialization, Module, ModuleTranslation, - PrimaryMap, Tunables, WasmFunctionInfo, -}; -use anyhow::{bail, Result}; -use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; -use object::SectionKind; +use crate::{DefinedFuncIndex, FilePos, FuncIndex, Module, PrimaryMap, StackMap}; use serde_derive::{Deserialize, Serialize}; +use std::fmt; use std::ops::Range; use std::str; use wasmtime_types::ModuleInternedTypeIndex; @@ -27,6 +22,38 @@ pub struct CompiledFunctionInfo { pub native_to_wasm_trampoline: Option, } +/// Information about a function, such as trap information, address map, +/// and stack maps. +#[derive(Serialize, Deserialize, Default)] +#[allow(missing_docs)] +pub struct WasmFunctionInfo { + pub start_srcloc: FilePos, + pub stack_maps: Box<[StackMapInformation]>, +} + +/// Description of where a function is located in the text section of a +/// compiled image. +#[derive(Copy, Clone, Serialize, Deserialize)] +pub struct FunctionLoc { + /// The byte offset from the start of the text section where this + /// function starts. + pub start: u32, + /// The byte length of this function's function body. + pub length: u32, +} + +/// The offset within a function of a GC safepoint, and its associated stack +/// map. +#[derive(Serialize, Deserialize, Debug)] +pub struct StackMapInformation { + /// The offset of the GC safepoint within the function's native code. It is + /// relative to the beginning of the function. + pub code_offset: u32, + + /// The stack map for identifying live GC refs at the GC safepoint. + pub stack_map: StackMap, +} + /// Secondary in-memory results of module compilation. /// /// This opaque structure can be optionally passed back to @@ -88,290 +115,31 @@ pub struct Metadata { pub dwarf: Vec<(u8, Range)>, } -/// Helper structure to create an ELF file as a compilation artifact. -/// -/// This structure exposes the process which Wasmtime will encode a core wasm -/// module into an ELF file, notably managing data sections and all that good -/// business going into the final file. -pub struct ObjectBuilder<'a> { - /// The `object`-crate-defined ELF file write we're using. - obj: Object<'a>, - - /// General compilation configuration. - tunables: &'a Tunables, - - /// The section identifier for "rodata" which is where wasm data segments - /// will go. - data: SectionId, - - /// The section identifier for function name information, or otherwise where - /// the `name` custom section of wasm is copied into. - /// - /// This is optional and lazily created on demand. - names: Option, - - /// The section identifier for dwarf information copied from the original - /// wasm files. - /// - /// This is optional and lazily created on demand. - dwarf: Option, +/// Value of a configured setting for a [`Compiler`] +#[derive(Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] +pub enum FlagValue<'a> { + /// Name of the value that has been configured for this setting. + Enum(&'a str), + /// The numerical value of the configured settings. + Num(u8), + /// Whether the setting is on or off. + Bool(bool), } -impl<'a> ObjectBuilder<'a> { - /// Creates a new builder for the `obj` specified. - pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { - let data = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_WASM_DATA.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - ObjectBuilder { - obj, - tunables, - data, - names: None, - dwarf: None, +impl fmt::Display for FlagValue<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Enum(v) => v.fmt(f), + Self::Num(v) => v.fmt(f), + Self::Bool(v) => v.fmt(f), } } - - /// Completes compilation of the `translation` specified, inserting - /// everything necessary into the `Object` being built. - /// - /// This function will consume the final results of compiling a wasm module - /// and finish the ELF image in-progress as part of `self.obj` by appending - /// any compiler-agnostic sections. - /// - /// The auxiliary `CompiledModuleInfo` structure returned here has also been - /// serialized into the object returned, but if the caller will quickly - /// turn-around and invoke `CompiledModule::from_artifacts` after this then - /// the information can be passed to that method to avoid extra - /// deserialization. This is done to avoid a serialize-then-deserialize for - /// API calls like `Module::new` where the compiled module is immediately - /// going to be used. - /// - /// The various arguments here are: - /// - /// * `translation` - the core wasm translation that's being completed. - /// - /// * `funcs` - compilation metadata about functions within the translation - /// as well as where the functions are located in the text section and any - /// associated trampolines. - /// - /// * `wasm_to_native_trampolines` - list of all trampolines necessary for - /// Wasm callers calling native callees (e.g. `Func::wrap`). One for each - /// function signature in the module. Must be sorted by `SignatureIndex`. - /// - /// Returns the `CompiledModuleInfo` corresponding to this core Wasm module - /// as a result of this append operation. This is then serialized into the - /// final artifact by the caller. - pub fn append( - &mut self, - translation: ModuleTranslation<'_>, - funcs: PrimaryMap, - wasm_to_native_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>, - ) -> Result { - let ModuleTranslation { - mut module, - debuginfo, - has_unparsed_debuginfo, - data, - data_align, - passive_data, - .. - } = translation; - - // Place all data from the wasm module into a section which will the - // source of the data later at runtime. This additionally keeps track of - // the offset of - let mut total_data_len = 0; - let data_offset = self - .obj - .append_section_data(self.data, &[], data_align.unwrap_or(1)); - for (i, data) in data.iter().enumerate() { - // The first data segment has its alignment specified as the alignment - // for the entire section, but everything afterwards is adjacent so it - // has alignment of 1. - let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; - self.obj.append_section_data(self.data, data, align); - total_data_len += data.len(); - } - for data in passive_data.iter() { - self.obj.append_section_data(self.data, data, 1); - } - - // If any names are present in the module then the `ELF_NAME_DATA` section - // is create and appended. - let mut func_names = Vec::new(); - if debuginfo.name_section.func_names.len() > 0 { - let name_id = *self.names.get_or_insert_with(|| { - self.obj.add_section( - self.obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_NAME_DATA.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ) - }); - let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); - sorted_names.sort_by_key(|(idx, _name)| *idx); - for (idx, name) in sorted_names { - let offset = self.obj.append_section_data(name_id, name.as_bytes(), 1); - let offset = match u32::try_from(offset) { - Ok(offset) => offset, - Err(_) => bail!("name section too large (> 4gb)"), - }; - let len = u32::try_from(name.len()).unwrap(); - func_names.push(FunctionName { - idx: *idx, - offset, - len, - }); - } - } - - // Data offsets in `MemoryInitialization` are offsets within the - // `translation.data` list concatenated which is now present in the data - // segment that's appended to the object. Increase the offsets by - // `self.data_size` to account for any previously added module. - let data_offset = u32::try_from(data_offset).unwrap(); - match &mut module.memory_initialization { - MemoryInitialization::Segmented(list) => { - for segment in list { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - MemoryInitialization::Static { map } => { - for (_, segment) in map { - if let Some(segment) = segment { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - } - } - - // Data offsets for passive data are relative to the start of - // `translation.passive_data` which was appended to the data segment - // of this object, after active data in `translation.data`. Update the - // offsets to account prior modules added in addition to active data. - let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); - for (_, range) in module.passive_data_map.iter_mut() { - range.start = range.start.checked_add(data_offset).unwrap(); - range.end = range.end.checked_add(data_offset).unwrap(); - } - - // Insert the wasm raw wasm-based debuginfo into the output, if - // requested. Note that this is distinct from the native debuginfo - // possibly generated by the native compiler, hence these sections - // getting wasm-specific names. - let mut dwarf = Vec::new(); - if self.tunables.parse_wasm_debuginfo { - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_abbrev); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_addr); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_aranges); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_info); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line_str); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str_offsets); - self.push_debug(&mut dwarf, &debuginfo.debug_ranges); - self.push_debug(&mut dwarf, &debuginfo.debug_rnglists); - } - // Sort this for binary-search-lookup later in `symbolize_context`. - dwarf.sort_by_key(|(id, _)| *id); - - Ok(CompiledModuleInfo { - module, - funcs, - wasm_to_native_trampolines, - func_names, - meta: Metadata { - native_debug_info_present: self.tunables.generate_native_debuginfo, - has_unparsed_debuginfo, - code_section_offset: debuginfo.wasm_file.code_section_offset, - has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, - dwarf, - }, - }) - } - - fn push_debug<'b, T>(&mut self, dwarf: &mut Vec<(u8, Range)>, section: &T) - where - T: gimli::Section>, - { - let data = section.reader().slice(); - if data.is_empty() { - return; - } - let section_id = *self.dwarf.get_or_insert_with(|| { - self.obj.add_section( - self.obj.segment_name(StandardSegment::Debug).to_vec(), - obj::ELF_WASMTIME_DWARF.as_bytes().to_vec(), - SectionKind::Debug, - ) - }); - let offset = self.obj.append_section_data(section_id, data, 1); - dwarf.push((T::id() as u8, offset..offset + data.len() as u64)); - } - - /// Creates the `ELF_WASMTIME_INFO` section from the given serializable data - /// structure. - pub fn serialize_info(&mut self, info: &T) - where - T: serde::Serialize, - { - let section = self.obj.add_section( - self.obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_WASMTIME_INFO.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - let data = bincode::serialize(info).unwrap(); - self.obj.set_section_data(section, data, 1); - } - - /// Serializes `self` into a buffer. This can be used for execution as well - /// as serialization. - pub fn finish(self, t: &mut T) -> Result<()> { - self.obj.emit(t).map_err(|e| e.into()) - } } -/// A type which can be the result of serializing an object. -pub trait FinishedObject: Sized { - /// Emit the object as `Self`. - fn finish_object(obj: ObjectBuilder<'_>) -> Result; -} - -impl FinishedObject for Vec { - fn finish_object(obj: ObjectBuilder<'_>) -> Result { - let mut result = ObjectVec::default(); - obj.finish(&mut result)?; - return Ok(result.0); - - #[derive(Default)] - struct ObjectVec(Vec); - - impl WritableBuffer for ObjectVec { - fn len(&self) -> usize { - self.0.len() - } - - fn reserve(&mut self, additional: usize) -> Result<(), ()> { - assert_eq!(self.0.len(), 0, "cannot reserve twice"); - self.0 = Vec::with_capacity(additional); - Ok(()) - } - - fn resize(&mut self, new_len: usize) { - if new_len <= self.0.len() { - self.0.truncate(new_len) - } else { - self.0.extend(vec![0; new_len - self.0.len()]) - } - } - - fn write_bytes(&mut self, val: &[u8]) { - self.0.extend(val); - } - } - } +/// Types of objects that can be created by `Compiler::object` +pub enum ObjectKind { + /// A core wasm compilation artifact + Module, + /// A component compilation artifact + Component, } diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index d14382a8be76..d59e1b000b97 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -1,21 +1,5 @@ -use crate::obj::ELF_WASMTIME_TRAPS; -use object::write::{Object, StandardSegment}; -use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; +use object::{Bytes, LittleEndian, U32Bytes}; use std::fmt; -use std::ops::Range; - -/// A helper structure to build the custom-encoded section of a wasmtime -/// compilation image which encodes trap information. -/// -/// This structure is incrementally fed the results of compiling individual -/// functions and handles all the encoding internally, allowing usage of -/// `lookup_trap_code` below with the resulting section. -#[derive(Default)] -pub struct TrapEncodingBuilder { - offsets: Vec>, - traps: Vec, - last_offset: u32, -} /// Information about trap. #[derive(Debug, PartialEq, Eq, Clone)] @@ -162,57 +146,6 @@ impl fmt::Display for Trap { impl std::error::Error for Trap {} -impl TrapEncodingBuilder { - /// Appends trap information about a function into this section. - /// - /// This function is called to describe traps for the `func` range - /// specified. The `func` offsets are specified relative to the text section - /// itself, and the `traps` offsets are specified relative to the start of - /// `func`. - /// - /// This is required to be called in-order for increasing ranges of `func` - /// to ensure the final array is properly sorted. Additionally `traps` must - /// be sorted. - pub fn push(&mut self, func: Range, traps: &[TrapInformation]) { - // NB: for now this only supports <=4GB text sections in object files. - // Alternative schemes will need to be created for >32-bit offsets to - // avoid making this section overly large. - let func_start = u32::try_from(func.start).unwrap(); - let func_end = u32::try_from(func.end).unwrap(); - - // Sanity-check to ensure that functions are pushed in-order, otherwise - // the `offsets` array won't be sorted which is our goal. - assert!(func_start >= self.last_offset); - - self.offsets.reserve(traps.len()); - self.traps.reserve(traps.len()); - for info in traps { - let pos = func_start + info.code_offset; - assert!(pos >= self.last_offset); - self.offsets.push(U32Bytes::new(LittleEndian, pos)); - self.traps.push(info.trap_code as u8); - self.last_offset = pos; - } - - self.last_offset = func_end; - } - - /// Encodes this section into the object provided. - pub fn append_to(self, obj: &mut Object) { - let section = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASMTIME_TRAPS.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - - // NB: this matches the encoding expected by `lookup` below. - let amt = u32::try_from(self.traps.len()).unwrap(); - obj.append_section_data(section, &amt.to_le_bytes(), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); - obj.append_section_data(section, &self.traps, 1); - } -} - /// Decodes the provided trap information section and attempts to find the trap /// code corresponding to the `offset` specified. /// diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 80f54d40e1b7..c2994850b427 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -41,6 +41,11 @@ use wasmtime_environ::{ mod code_builder; pub use self::code_builder::{CodeBuilder, HashedEngineCompileEnv}; +#[cfg(feature = "runtime")] +mod runtime; +#[cfg(feature = "runtime")] +pub use self::runtime::finish_object; + /// Converts an input binary-encoded WebAssembly module to compilation /// artifacts and type information. /// diff --git a/crates/wasmtime/src/compile/code_builder.rs b/crates/wasmtime/src/compile/code_builder.rs index 9393278edf1f..71d914588b81 100644 --- a/crates/wasmtime/src/compile/code_builder.rs +++ b/crates/wasmtime/src/compile/code_builder.rs @@ -1,15 +1,7 @@ -#[cfg(all(feature = "runtime", feature = "component-model"))] -use crate::component::Component; use crate::Engine; -#[cfg(feature = "runtime")] -use crate::{instantiate::MmapVecWrapper, CodeMemory, Module}; use anyhow::{anyhow, bail, Context, Result}; use std::borrow::Cow; use std::path::Path; -use std::sync::Arc; -use wasmtime_environ::ObjectKind; -#[cfg(feature = "runtime")] -use wasmtime_runtime::MmapVec; /// Builder-style structure used to create a [`Module`](crate::Module) or /// pre-compile a module to a serialized list of bytes. @@ -45,7 +37,7 @@ use wasmtime_runtime::MmapVec; /// [`wasm`]: CodeBuilder::wasm /// [`wasm_file`]: CodeBuilder::wasm_file pub struct CodeBuilder<'a> { - engine: &'a Engine, + pub(super) engine: &'a Engine, wasm: Option>, wasm_path: Option>, wat: bool, @@ -130,7 +122,7 @@ impl<'a> CodeBuilder<'a> { Ok(self) } - fn wasm_binary(&self) -> Result> { + pub(super) fn wasm_binary(&self) -> Result> { let wasm = self .wasm .as_ref() @@ -147,65 +139,6 @@ impl<'a> CodeBuilder<'a> { Ok((&wasm[..]).into()) } - #[cfg(feature = "runtime")] - fn compile_cached( - &self, - build_artifacts: fn(&Engine, &[u8]) -> Result<(MmapVecWrapper, Option)>, - ) -> Result<(Arc, Option)> { - let wasm = self.wasm_binary()?; - - self.engine - .check_compatible_with_native_host() - .context("compilation settings are not compatible with the native host")?; - - #[cfg(feature = "cache")] - { - let state = ( - HashedEngineCompileEnv(self.engine), - &wasm, - // Don't hash this as it's just its own "pure" function pointer. - NotHashed(build_artifacts), - ); - let (code, info_and_types) = - wasmtime_cache::ModuleCacheEntry::new("wasmtime", self.engine.cache_config()) - .get_data_raw( - &state, - // Cache miss, compute the actual artifacts - |(engine, wasm, build_artifacts)| -> Result<_> { - let (mmap, info) = (build_artifacts.0)(engine.0, wasm)?; - let code = publish_mmap(mmap.0)?; - Ok((code, info)) - }, - // Implementation of how to serialize artifacts - |(_engine, _wasm, _), (code, _info_and_types)| Some(code.mmap().to_vec()), - // Cache hit, deserialize the provided artifacts - |(engine, wasm, _), serialized_bytes| { - let kind = if wasmparser::Parser::is_component(&wasm) { - ObjectKind::Component - } else { - ObjectKind::Module - }; - let code = engine.0.load_code_bytes(&serialized_bytes, kind).ok()?; - Some((code, None)) - }, - )?; - return Ok((code, info_and_types)); - } - - #[cfg(not(feature = "cache"))] - { - let (mmap, info_and_types) = build_artifacts(self.engine, &wasm)?; - let code = publish_mmap(mmap.0)?; - return Ok((code, info_and_types)); - } - - struct NotHashed(T); - - impl std::hash::Hash for NotHashed { - fn hash(&self, _hasher: &mut H) {} - } - } - /// Finishes this compilation and produces a serialized list of bytes. /// /// This method requires that either [`CodeBuilder::wasm`] or @@ -228,18 +161,6 @@ impl<'a> CodeBuilder<'a> { Ok(v) } - /// Same as [`CodeBuilder::compile_module_serialized`] except that a - /// [`Module`](crate::Module) is produced instead. - /// - /// Note that this method will cache compilations if the `cache` feature is - /// enabled and turned on in [`Config`](crate::Config). - #[cfg(feature = "runtime")] - #[cfg_attr(docsrs, doc(cfg(feature = "runtime")))] - pub fn compile_module(&self) -> Result { - let (code, info_and_types) = self.compile_cached(super::build_artifacts)?; - Module::from_parts(self.engine, code, info_and_types) - } - /// Same as [`CodeBuilder::compile_module_serialized`] except that it /// compiles a serialized [`Component`] instead of a module. #[cfg(feature = "component-model")] @@ -249,18 +170,6 @@ impl<'a> CodeBuilder<'a> { let (v, _) = super::build_component_artifacts(self.engine, &bytes)?; Ok(v) } - - /// Same as [`CodeBuilder::compile_module`] except that it compiles a - /// [`Component`] instead of a module. - #[cfg(all(feature = "runtime", feature = "component-model"))] - #[cfg_attr( - docsrs, - doc(cfg(all(feature = "runtime", feature = "component-model"))) - )] - pub fn compile_component(&self) -> Result { - let (code, artifacts) = self.compile_cached(super::build_component_artifacts)?; - Component::from_parts(self.engine, code, artifacts) - } } /// This is a helper struct used when caching to hash the state of an `Engine` @@ -289,10 +198,3 @@ impl std::hash::Hash for HashedEngineCompileEnv<'_> { config.module_version.hash(hasher); } } - -#[cfg(feature = "runtime")] -fn publish_mmap(mmap: MmapVec) -> Result> { - let mut code = CodeMemory::new(mmap)?; - code.publish()?; - Ok(Arc::new(code)) -} diff --git a/crates/wasmtime/src/compile/runtime.rs b/crates/wasmtime/src/compile/runtime.rs new file mode 100644 index 000000000000..fde8dfcdfeaf --- /dev/null +++ b/crates/wasmtime/src/compile/runtime.rs @@ -0,0 +1,175 @@ +use crate::compile::HashedEngineCompileEnv; +#[cfg(feature = "component-model")] +use crate::component::Component; +use crate::{CodeBuilder, CodeMemory, Engine, Module}; +use anyhow::{Context, Error, Result}; +use object::write::WritableBuffer; +use std::sync::Arc; +use wasmtime_environ::{FinishedObject, ObjectBuilder, ObjectKind}; +use wasmtime_runtime::MmapVec; + +impl<'a> CodeBuilder<'a> { + fn compile_cached( + &self, + build_artifacts: fn(&Engine, &[u8]) -> Result<(MmapVecWrapper, Option)>, + ) -> Result<(Arc, Option)> { + let wasm = self.wasm_binary()?; + + self.engine + .check_compatible_with_native_host() + .context("compilation settings are not compatible with the native host")?; + + #[cfg(feature = "cache")] + { + let state = ( + HashedEngineCompileEnv(self.engine), + &wasm, + // Don't hash this as it's just its own "pure" function pointer. + NotHashed(build_artifacts), + ); + let (code, info_and_types) = + wasmtime_cache::ModuleCacheEntry::new("wasmtime", self.engine.cache_config()) + .get_data_raw( + &state, + // Cache miss, compute the actual artifacts + |(engine, wasm, build_artifacts)| -> Result<_> { + let (mmap, info) = (build_artifacts.0)(engine.0, wasm)?; + let code = publish_mmap(mmap.0)?; + Ok((code, info)) + }, + // Implementation of how to serialize artifacts + |(_engine, _wasm, _), (code, _info_and_types)| Some(code.mmap().to_vec()), + // Cache hit, deserialize the provided artifacts + |(engine, wasm, _), serialized_bytes| { + let kind = if wasmparser::Parser::is_component(&wasm) { + ObjectKind::Component + } else { + ObjectKind::Module + }; + let code = engine.0.load_code_bytes(&serialized_bytes, kind).ok()?; + Some((code, None)) + }, + )?; + return Ok((code, info_and_types)); + } + + #[cfg(not(feature = "cache"))] + { + let (mmap, info_and_types) = build_artifacts(self.engine, &wasm)?; + let code = publish_mmap(mmap.0)?; + return Ok((code, info_and_types)); + } + + struct NotHashed(T); + + impl std::hash::Hash for NotHashed { + fn hash(&self, _hasher: &mut H) {} + } + } + + /// Same as [`CodeBuilder::compile_module_serialized`] except that a + /// [`Module`](crate::Module) is produced instead. + /// + /// Note that this method will cache compilations if the `cache` feature is + /// enabled and turned on in [`Config`](crate::Config). + #[cfg_attr(docsrs, doc(cfg(feature = "runtime")))] + pub fn compile_module(&self) -> Result { + let (code, info_and_types) = self.compile_cached(super::build_artifacts)?; + Module::from_parts(self.engine, code, info_and_types) + } + + /// Same as [`CodeBuilder::compile_module`] except that it compiles a + /// [`Component`] instead of a module. + #[cfg(feature = "component-model")] + #[cfg_attr( + docsrs, + doc(cfg(all(feature = "runtime", feature = "component-model"))) + )] + pub fn compile_component(&self) -> Result { + let (code, artifacts) = self.compile_cached(super::build_component_artifacts)?; + Component::from_parts(self.engine, code, artifacts) + } +} + +fn publish_mmap(mmap: MmapVec) -> Result> { + let mut code = CodeMemory::new(mmap)?; + code.publish()?; + Ok(Arc::new(code)) +} + +/// Write an object out to an [`MmapVec`] so that it can be marked executable +/// before running. +/// +/// The returned `MmapVec` will contain the serialized version of `obj` +/// and is sized appropriately to the exact size of the object serialized. +pub fn finish_object(obj: ObjectBuilder<'_>) -> Result { + Ok(::finish_object(obj)?.0) +} + +pub(crate) struct MmapVecWrapper(pub MmapVec); + +impl FinishedObject for MmapVecWrapper { + fn finish_object(obj: ObjectBuilder<'_>) -> Result { + let mut result = ObjectMmap::default(); + return match obj.finish(&mut result) { + Ok(()) => { + assert!(result.mmap.is_some(), "no reserve"); + let mmap = result.mmap.expect("reserve not called"); + assert_eq!(mmap.len(), result.len); + Ok(MmapVecWrapper(mmap)) + } + Err(e) => match result.err.take() { + Some(original) => Err(original.context(e)), + None => Err(e.into()), + }, + }; + + /// Helper struct to implement the `WritableBuffer` trait from the `object` + /// crate. + /// + /// This enables writing an object directly into an mmap'd memory so it's + /// immediately usable for execution after compilation. This implementation + /// relies on a call to `reserve` happening once up front with all the needed + /// data, and the mmap internally does not attempt to grow afterwards. + #[derive(Default)] + struct ObjectMmap { + mmap: Option, + len: usize, + err: Option, + } + + impl WritableBuffer for ObjectMmap { + fn len(&self) -> usize { + self.len + } + + fn reserve(&mut self, additional: usize) -> Result<(), ()> { + assert!(self.mmap.is_none(), "cannot reserve twice"); + self.mmap = match MmapVec::with_capacity(additional) { + Ok(mmap) => Some(mmap), + Err(e) => { + self.err = Some(e); + return Err(()); + } + }; + Ok(()) + } + + fn resize(&mut self, new_len: usize) { + // Resizing always appends 0 bytes and since new mmaps start out as 0 + // bytes we don't actually need to do anything as part of this other + // than update our own length. + if new_len <= self.len { + return; + } + self.len = new_len; + } + + fn write_bytes(&mut self, val: &[u8]) { + let mmap = self.mmap.as_mut().expect("write before reserve"); + mmap[self.len..][..val.len()].copy_from_slice(val); + self.len += val.len(); + } + } + } +} diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 271c9bc25ff5..f4f2e091d525 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -26,6 +26,7 @@ use crate::stack::{StackCreator, StackCreatorProxy}; #[cfg(feature = "async")] use wasmtime_fiber::RuntimeFiberStackCreator; +#[cfg(all(feature = "incremental-cache", feature = "cranelift"))] pub use wasmtime_environ::CacheStore; #[cfg(feature = "pooling-allocator")] use wasmtime_runtime::mpk; @@ -158,7 +159,7 @@ struct CompilerConfig { target: Option, settings: HashMap, flags: HashSet, - #[cfg(any(feature = "cranelift", feature = "winch"))] + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] cache_store: Option>, clif_dir: Option, wmemcheck: bool, @@ -172,6 +173,7 @@ impl CompilerConfig { target: None, settings: HashMap::new(), flags: HashSet::new(), + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] cache_store: None, clif_dir: None, wmemcheck: false, @@ -1894,6 +1896,7 @@ impl Config { compiler.enable(flag)?; } + #[cfg(feature = "incremental-cache")] if let Some(cache_store) = &self.compiler_config.cache_store { compiler.enable_incremental_compilation(cache_store.clone())?; } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 0e6f16cf2ab8..ef79116a1e8e 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -2,6 +2,7 @@ use crate::runtime::type_registry::TypeRegistry; use crate::Config; use anyhow::{Context, Result}; +#[cfg(any(feature = "cranelift", feature = "winch"))] use object::write::{Object, StandardSegment}; use object::SectionKind; use once_cell::sync::OnceCell; diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs index 3f1473fc1fa8..07855c608afd 100644 --- a/crates/wasmtime/src/engine/serialization.rs +++ b/crates/wasmtime/src/engine/serialization.rs @@ -23,6 +23,7 @@ use crate::{Engine, ModuleVersionStrategy, Precompiled}; use anyhow::{anyhow, bail, ensure, Context, Result}; +#[cfg(any(feature = "cranelift", feature = "winch"))] use object::write::{Object, StandardSegment}; use object::{File, FileFlags, Object as _, ObjectSection, SectionKind}; use serde_derive::{Deserialize, Serialize}; @@ -112,6 +113,7 @@ pub fn check_compatible(engine: &Engine, mmap: &[u8], expected: ObjectKind) -> R bincode::deserialize::>(data)?.check_compatible(engine) } +#[cfg(any(feature = "cranelift", feature = "winch"))] pub fn append_compiler_info(engine: &Engine, obj: &mut Object<'_>, metadata: &Metadata<'_>) { let section = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs index 165663a37c06..c09b96d6b547 100644 --- a/crates/wasmtime/src/runtime/instantiate.rs +++ b/crates/wasmtime/src/runtime/instantiate.rs @@ -4,14 +4,13 @@ //! steps. use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent}; -use anyhow::{Error, Result}; -use object::write::WritableBuffer; +use anyhow::Result; use std::str; use std::sync::Arc; use wasmtime_environ::{ - CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FinishedObject, FuncIndex, - FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, ObjectBuilder, - PrimaryMap, StackMapInformation, WasmFunctionInfo, + CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FuncIndex, FunctionLoc, + FunctionName, Metadata, Module, ModuleInternedTypeIndex, PrimaryMap, StackMapInformation, + WasmFunctionInfo, }; use wasmtime_runtime::{CompiledModuleId, CompiledModuleIdAllocator, MmapVec}; @@ -344,80 +343,3 @@ impl<'a> SymbolizeContext<'a> { self.code_section_offset } } - -/// Write an object out to an [`MmapVec`] so that it can be marked executable -/// before running. -/// -/// The returned `MmapVec` will contain the serialized version of `obj` -/// and is sized appropriately to the exact size of the object serialized. -pub fn finish_object(obj: ObjectBuilder<'_>) -> Result { - Ok(::finish_object(obj)?.0) -} - -pub(crate) struct MmapVecWrapper(pub MmapVec); - -impl FinishedObject for MmapVecWrapper { - fn finish_object(obj: ObjectBuilder<'_>) -> Result { - let mut result = ObjectMmap::default(); - return match obj.finish(&mut result) { - Ok(()) => { - assert!(result.mmap.is_some(), "no reserve"); - let mmap = result.mmap.expect("reserve not called"); - assert_eq!(mmap.len(), result.len); - Ok(MmapVecWrapper(mmap)) - } - Err(e) => match result.err.take() { - Some(original) => Err(original.context(e)), - None => Err(e.into()), - }, - }; - - /// Helper struct to implement the `WritableBuffer` trait from the `object` - /// crate. - /// - /// This enables writing an object directly into an mmap'd memory so it's - /// immediately usable for execution after compilation. This implementation - /// relies on a call to `reserve` happening once up front with all the needed - /// data, and the mmap internally does not attempt to grow afterwards. - #[derive(Default)] - struct ObjectMmap { - mmap: Option, - len: usize, - err: Option, - } - - impl WritableBuffer for ObjectMmap { - fn len(&self) -> usize { - self.len - } - - fn reserve(&mut self, additional: usize) -> Result<(), ()> { - assert!(self.mmap.is_none(), "cannot reserve twice"); - self.mmap = match MmapVec::with_capacity(additional) { - Ok(mmap) => Some(mmap), - Err(e) => { - self.err = Some(e); - return Err(()); - } - }; - Ok(()) - } - - fn resize(&mut self, new_len: usize) { - // Resizing always appends 0 bytes and since new mmaps start out as 0 - // bytes we don't actually need to do anything as part of this other - // than update our own length. - if new_len <= self.len { - return; - } - self.len = new_len; - } - - fn write_bytes(&mut self, val: &[u8]) { - let mmap = self.mmap.as_mut().expect("write before reserve"); - mmap[self.len..][..val.len()].copy_from_slice(val); - self.len += val.len(); - } - } - } -} diff --git a/crates/wasmtime/src/runtime/trampoline/func.rs b/crates/wasmtime/src/runtime/trampoline/func.rs index 50c078e9d2a3..03aa908348d9 100644 --- a/crates/wasmtime/src/runtime/trampoline/func.rs +++ b/crates/wasmtime/src/runtime/trampoline/func.rs @@ -85,7 +85,7 @@ pub fn create_array_call_function( where F: Fn(*mut VMContext, &mut [ValRaw]) -> Result<()> + Send + Sync + 'static, { - use crate::instantiate::finish_object; + use crate::compile::finish_object; use std::ptr; let mut obj = engine From e8bfd230442e161ffd556df83667561dfa278295 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 27 Mar 2024 18:11:25 -0700 Subject: [PATCH 2/5] Fix compile issue --- crates/environ/src/component/info.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index 1f42330f0175..4c9abeb7221a 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -523,7 +523,10 @@ pub enum FixedEncoding { } impl FixedEncoding { - pub(crate) fn width(&self) -> u8 { + /// Returns the byte width of unit loads/stores for this encoding, for + /// example the unit length is multiplied by this return value to get the + /// byte width of a string. + pub fn width(&self) -> u8 { match self { FixedEncoding::Utf8 => 1, FixedEncoding::Utf16 => 2, From 12888cccd20dee9c8d14476169810718f0ec8243 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 27 Mar 2024 18:41:13 -0700 Subject: [PATCH 3/5] Gate `ModuleTranslation` and its methods on `compile` --- crates/environ/src/lib.rs | 6 +- crates/environ/src/module.rs | 344 +------------------------- crates/environ/src/module_environ.rs | 349 ++++++++++++++++++++++++++- 3 files changed, 351 insertions(+), 348 deletions(-) diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 97c748359353..0e473596d7da 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -11,7 +11,6 @@ mod builtin; mod demangling; mod module; mod module_artifacts; -mod module_environ; mod module_types; pub mod obj; mod ref_bits; @@ -26,7 +25,6 @@ pub use crate::builtin::*; pub use crate::demangling::*; pub use crate::module::*; pub use crate::module_artifacts::*; -pub use crate::module_environ::*; pub use crate::module_types::*; pub use crate::ref_bits::*; pub use crate::scopevec::ScopeVec; @@ -39,7 +37,11 @@ pub use object; #[cfg(feature = "compile")] mod compile; #[cfg(feature = "compile")] +mod module_environ; +#[cfg(feature = "compile")] pub use crate::compile::*; +#[cfg(feature = "compile")] +pub use crate::module_environ::*; #[cfg(feature = "component-model")] pub mod component; diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 24dec26af0ac..15ed0a325a70 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -1,11 +1,10 @@ //! Data structures for representing decoded wasm modules. -use crate::{ModuleTranslation, PrimaryMap, Tunables, WASM_PAGE_SIZE}; +use crate::{PrimaryMap, Tunables, WASM_PAGE_SIZE}; use cranelift_entity::{packed_option::ReservedValue, EntityRef}; use indexmap::IndexMap; use serde_derive::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::mem; use std::ops::Range; use wasmtime_types::*; @@ -179,347 +178,6 @@ pub enum MemoryInitialization { }, } -impl ModuleTranslation<'_> { - /// Attempts to convert segmented memory initialization into static - /// initialization for the module that this translation represents. - /// - /// If this module's memory initialization is not compatible with paged - /// initialization then this won't change anything. Otherwise if it is - /// compatible then the `memory_initialization` field will be updated. - /// - /// Takes a `page_size` argument in order to ensure that all - /// initialization is page-aligned for mmap-ability, and - /// `max_image_size_always_allowed` to control how we decide - /// whether to use static init. - /// - /// We will try to avoid generating very sparse images, which are - /// possible if e.g. a module has an initializer at offset 0 and a - /// very high offset (say, 1 GiB). To avoid this, we use a dual - /// condition: we always allow images less than - /// `max_image_size_always_allowed`, and the embedder of Wasmtime - /// can set this if desired to ensure that static init should - /// always be done if the size of the module or its heaps is - /// otherwise bounded by the system. We also allow images with - /// static init data bigger than that, but only if it is "dense", - /// defined as having at least half (50%) of its pages with some - /// data. - /// - /// We could do something slightly better by building a dense part - /// and keeping a sparse list of outlier/leftover segments (see - /// issue #3820). This would also allow mostly-static init of - /// modules that have some dynamically-placed data segments. But, - /// for now, this is sufficient to allow a system that "knows what - /// it's doing" to always get static init. - pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { - // This method only attempts to transform a `Segmented` memory init - // into a `Static` one, no other state. - if !self.module.memory_initialization.is_segmented() { - return; - } - - // First a dry run of memory initialization is performed. This - // collects information about the extent of memory initialized for each - // memory as well as the size of all data segments being copied in. - struct Memory { - data_size: u64, - min_addr: u64, - max_addr: u64, - // The `usize` here is a pointer into `self.data` which is the list - // of data segments corresponding to what was found in the original - // wasm module. - segments: Vec<(usize, StaticMemoryInitializer)>, - } - let mut info = PrimaryMap::with_capacity(self.module.memory_plans.len()); - for _ in 0..self.module.memory_plans.len() { - info.push(Memory { - data_size: 0, - min_addr: u64::MAX, - max_addr: 0, - segments: Vec::new(), - }); - } - let mut idx = 0; - let ok = self.module.memory_initialization.init_memory( - &mut (), - InitMemory::CompileTime(&self.module), - |(), memory, init| { - // Currently `Static` only applies to locally-defined memories, - // so if a data segment references an imported memory then - // transitioning to a `Static` memory initializer is not - // possible. - if self.module.defined_memory_index(memory).is_none() { - return false; - }; - let info = &mut info[memory]; - let data_len = u64::from(init.data.end - init.data.start); - if data_len > 0 { - info.data_size += data_len; - info.min_addr = info.min_addr.min(init.offset); - info.max_addr = info.max_addr.max(init.offset + data_len); - info.segments.push((idx, init.clone())); - } - idx += 1; - true - }, - ); - if !ok { - return; - } - - // Validate that the memory information collected is indeed valid for - // static memory initialization. - for info in info.values().filter(|i| i.data_size > 0) { - let image_size = info.max_addr - info.min_addr; - - // If the range of memory being initialized is less than twice the - // total size of the data itself then it's assumed that static - // initialization is ok. This means we'll at most double memory - // consumption during the memory image creation process, which is - // currently assumed to "probably be ok" but this will likely need - // tweaks over time. - if image_size < info.data_size.saturating_mul(2) { - continue; - } - - // If the memory initialization image is larger than the size of all - // data, then we still allow memory initialization if the image will - // be of a relatively modest size, such as 1MB here. - if image_size < max_image_size_always_allowed { - continue; - } - - // At this point memory initialization is concluded to be too - // expensive to do at compile time so it's entirely deferred to - // happen at runtime. - return; - } - - // Here's where we've now committed to changing to static memory. The - // memory initialization image is built here from the page data and then - // it's converted to a single initializer. - let data = mem::replace(&mut self.data, Vec::new()); - let mut map = PrimaryMap::with_capacity(info.len()); - let mut module_data_size = 0u32; - for (memory, info) in info.iter() { - // Create the in-memory `image` which is the initialized contents of - // this linear memory. - let extent = if info.segments.len() > 0 { - (info.max_addr - info.min_addr) as usize - } else { - 0 - }; - let mut image = Vec::with_capacity(extent); - for (idx, init) in info.segments.iter() { - let data = &data[*idx]; - assert_eq!(data.len(), init.data.len()); - let offset = usize::try_from(init.offset - info.min_addr).unwrap(); - if image.len() < offset { - image.resize(offset, 0u8); - image.extend_from_slice(data); - } else { - image.splice( - offset..(offset + data.len()).min(image.len()), - data.iter().copied(), - ); - } - } - assert_eq!(image.len(), extent); - assert_eq!(image.capacity(), extent); - let mut offset = if info.segments.len() > 0 { - info.min_addr - } else { - 0 - }; - - // Chop off trailing zeros from the image as memory is already - // zero-initialized. Note that `i` is the position of a nonzero - // entry here, so to not lose it we truncate to `i + 1`. - if let Some(i) = image.iter().rposition(|i| *i != 0) { - image.truncate(i + 1); - } - - // Also chop off leading zeros, if any. - if let Some(i) = image.iter().position(|i| *i != 0) { - offset += i as u64; - image.drain(..i); - } - let mut len = u64::try_from(image.len()).unwrap(); - - // The goal is to enable mapping this image directly into memory, so - // the offset into linear memory must be a multiple of the page - // size. If that's not already the case then the image is padded at - // the front and back with extra zeros as necessary - if offset % page_size != 0 { - let zero_padding = offset % page_size; - self.data.push(vec![0; zero_padding as usize].into()); - offset -= zero_padding; - len += zero_padding; - } - self.data.push(image.into()); - if len % page_size != 0 { - let zero_padding = page_size - (len % page_size); - self.data.push(vec![0; zero_padding as usize].into()); - len += zero_padding; - } - - // Offset/length should now always be page-aligned. - assert!(offset % page_size == 0); - assert!(len % page_size == 0); - - // Create the `StaticMemoryInitializer` which describes this image, - // only needed if the image is actually present and has a nonzero - // length. The `offset` has been calculates above, originally - // sourced from `info.min_addr`. The `data` field is the extent - // within the final data segment we'll emit to an ELF image, which - // is the concatenation of `self.data`, so here it's the size of - // the section-so-far plus the current segment we're appending. - let len = u32::try_from(len).unwrap(); - let init = if len > 0 { - Some(StaticMemoryInitializer { - offset, - data: module_data_size..module_data_size + len, - }) - } else { - None - }; - let idx = map.push(init); - assert_eq!(idx, memory); - module_data_size += len; - } - self.data_align = Some(page_size); - self.module.memory_initialization = MemoryInitialization::Static { map }; - } - - /// Attempts to convert the module's table initializers to - /// FuncTable form where possible. This enables lazy table - /// initialization later by providing a one-to-one map of initial - /// table values, without having to parse all segments. - pub fn try_func_table_init(&mut self) { - // This should be large enough to support very large Wasm - // modules with huge funcref tables, but small enough to avoid - // OOMs or DoS on truly sparse tables. - const MAX_FUNC_TABLE_SIZE: u32 = 1024 * 1024; - - // First convert any element-initialized tables to images of just that - // single function if the minimum size of the table allows doing so. - for ((_, init), (_, plan)) in self - .module - .table_initialization - .initial_values - .iter_mut() - .zip( - self.module - .table_plans - .iter() - .skip(self.module.num_imported_tables), - ) - { - let table_size = plan.table.minimum; - if table_size > MAX_FUNC_TABLE_SIZE { - continue; - } - if let TableInitialValue::FuncRef(val) = *init { - *init = TableInitialValue::Null { - precomputed: vec![val; table_size as usize], - }; - } - } - - let mut segments = mem::take(&mut self.module.table_initialization.segments) - .into_iter() - .peekable(); - - // The goal of this loop is to interpret a table segment and apply it - // "statically" to a local table. This will iterate over segments and - // apply them one-by-one to each table. - // - // If any segment can't be applied, however, then this loop exits and - // all remaining segments are placed back into the segment list. This is - // because segments are supposed to be initialized one-at-a-time which - // means that intermediate state is visible with respect to traps. If - // anything isn't statically known to not trap it's pessimistically - // assumed to trap meaning all further segment initializers must be - // applied manually at instantiation time. - while let Some(segment) = segments.peek() { - let defined_index = match self.module.defined_table_index(segment.table_index) { - Some(index) => index, - // Skip imported tables: we can't provide a preconstructed - // table for them, because their values depend on the - // imported table overlaid with whatever segments we have. - None => break, - }; - - // If the base of this segment is dynamic, then we can't - // include it in the statically-built array of initial - // contents. - if segment.base.is_some() { - break; - } - - // Get the end of this segment. If out-of-bounds, or too - // large for our dense table representation, then skip the - // segment. - let top = match segment.offset.checked_add(segment.elements.len()) { - Some(top) => top, - None => break, - }; - let table_size = self.module.table_plans[segment.table_index].table.minimum; - if top > table_size || top > MAX_FUNC_TABLE_SIZE { - break; - } - - match self.module.table_plans[segment.table_index] - .table - .wasm_ty - .heap_type - { - WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {} - // If this is not a funcref table, then we can't support a - // pre-computed table of function indices. Technically this - // initializer won't trap so we could continue processing - // segments, but that's left as a future optimization if - // necessary. - WasmHeapType::Extern => break, - } - - // Function indices can be optimized here, but fully general - // expressions are deferred to get evaluated at runtime. - let function_elements = match &segment.elements { - TableSegmentElements::Functions(indices) => indices, - TableSegmentElements::Expressions(_) => break, - }; - - let precomputed = - match &mut self.module.table_initialization.initial_values[defined_index] { - TableInitialValue::Null { precomputed } => precomputed, - - // If this table is still listed as an initial value here - // then that means the initial size of the table doesn't - // support a precomputed function list, so skip this. - // Technically this won't trap so it's possible to process - // further initializers, but that's left as a future - // optimization. - TableInitialValue::FuncRef(_) | TableInitialValue::GlobalGet(_) => break, - }; - - // At this point we're committing to pre-initializing the table - // with the `segment` that's being iterated over. This segment is - // applied to the `precomputed` list for the table by ensuring - // it's large enough to hold the segment and then copying the - // segment into the precomputed list. - if precomputed.len() < top as usize { - precomputed.resize(top as usize, FuncIndex::reserved_value()); - } - let dst = &mut precomputed[(segment.offset as usize)..(top as usize)]; - dst.copy_from_slice(&function_elements); - - // advance the iterator to see the next segment - let _ = segments.next(); - } - self.module.table_initialization.segments = segments.collect(); - } -} - impl Default for MemoryInitialization { fn default() -> Self { Self::Segmented(Vec::new()) diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index c96c74b5c4da..1e3e7de9c754 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -4,13 +4,15 @@ use crate::module::{ }; use crate::{ DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, FuncIndex, GlobalIndex, - GlobalInit, MemoryIndex, ModuleTypesBuilder, PrimaryMap, TableIndex, TableInitialValue, - Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, WasmHeapType, WasmResult, WasmValType, - WasmparserTypeConverter, + GlobalInit, InitMemory, MemoryIndex, ModuleTypesBuilder, PrimaryMap, StaticMemoryInitializer, + TableIndex, TableInitialValue, Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, + WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter, }; use anyhow::{bail, Result}; +use cranelift_entity::packed_option::ReservedValue; use std::borrow::Cow; use std::collections::HashMap; +use std::mem; use std::path::PathBuf; use std::sync::Arc; use wasmparser::types::{CoreTypeId, Types}; @@ -901,3 +903,344 @@ impl TypeConvert for ModuleEnvironment<'_, '_> { .lookup_heap_type(index) } } + +impl ModuleTranslation<'_> { + /// Attempts to convert segmented memory initialization into static + /// initialization for the module that this translation represents. + /// + /// If this module's memory initialization is not compatible with paged + /// initialization then this won't change anything. Otherwise if it is + /// compatible then the `memory_initialization` field will be updated. + /// + /// Takes a `page_size` argument in order to ensure that all + /// initialization is page-aligned for mmap-ability, and + /// `max_image_size_always_allowed` to control how we decide + /// whether to use static init. + /// + /// We will try to avoid generating very sparse images, which are + /// possible if e.g. a module has an initializer at offset 0 and a + /// very high offset (say, 1 GiB). To avoid this, we use a dual + /// condition: we always allow images less than + /// `max_image_size_always_allowed`, and the embedder of Wasmtime + /// can set this if desired to ensure that static init should + /// always be done if the size of the module or its heaps is + /// otherwise bounded by the system. We also allow images with + /// static init data bigger than that, but only if it is "dense", + /// defined as having at least half (50%) of its pages with some + /// data. + /// + /// We could do something slightly better by building a dense part + /// and keeping a sparse list of outlier/leftover segments (see + /// issue #3820). This would also allow mostly-static init of + /// modules that have some dynamically-placed data segments. But, + /// for now, this is sufficient to allow a system that "knows what + /// it's doing" to always get static init. + pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { + // This method only attempts to transform a `Segmented` memory init + // into a `Static` one, no other state. + if !self.module.memory_initialization.is_segmented() { + return; + } + + // First a dry run of memory initialization is performed. This + // collects information about the extent of memory initialized for each + // memory as well as the size of all data segments being copied in. + struct Memory { + data_size: u64, + min_addr: u64, + max_addr: u64, + // The `usize` here is a pointer into `self.data` which is the list + // of data segments corresponding to what was found in the original + // wasm module. + segments: Vec<(usize, StaticMemoryInitializer)>, + } + let mut info = PrimaryMap::with_capacity(self.module.memory_plans.len()); + for _ in 0..self.module.memory_plans.len() { + info.push(Memory { + data_size: 0, + min_addr: u64::MAX, + max_addr: 0, + segments: Vec::new(), + }); + } + let mut idx = 0; + let ok = self.module.memory_initialization.init_memory( + &mut (), + InitMemory::CompileTime(&self.module), + |(), memory, init| { + // Currently `Static` only applies to locally-defined memories, + // so if a data segment references an imported memory then + // transitioning to a `Static` memory initializer is not + // possible. + if self.module.defined_memory_index(memory).is_none() { + return false; + }; + let info = &mut info[memory]; + let data_len = u64::from(init.data.end - init.data.start); + if data_len > 0 { + info.data_size += data_len; + info.min_addr = info.min_addr.min(init.offset); + info.max_addr = info.max_addr.max(init.offset + data_len); + info.segments.push((idx, init.clone())); + } + idx += 1; + true + }, + ); + if !ok { + return; + } + + // Validate that the memory information collected is indeed valid for + // static memory initialization. + for info in info.values().filter(|i| i.data_size > 0) { + let image_size = info.max_addr - info.min_addr; + + // If the range of memory being initialized is less than twice the + // total size of the data itself then it's assumed that static + // initialization is ok. This means we'll at most double memory + // consumption during the memory image creation process, which is + // currently assumed to "probably be ok" but this will likely need + // tweaks over time. + if image_size < info.data_size.saturating_mul(2) { + continue; + } + + // If the memory initialization image is larger than the size of all + // data, then we still allow memory initialization if the image will + // be of a relatively modest size, such as 1MB here. + if image_size < max_image_size_always_allowed { + continue; + } + + // At this point memory initialization is concluded to be too + // expensive to do at compile time so it's entirely deferred to + // happen at runtime. + return; + } + + // Here's where we've now committed to changing to static memory. The + // memory initialization image is built here from the page data and then + // it's converted to a single initializer. + let data = mem::replace(&mut self.data, Vec::new()); + let mut map = PrimaryMap::with_capacity(info.len()); + let mut module_data_size = 0u32; + for (memory, info) in info.iter() { + // Create the in-memory `image` which is the initialized contents of + // this linear memory. + let extent = if info.segments.len() > 0 { + (info.max_addr - info.min_addr) as usize + } else { + 0 + }; + let mut image = Vec::with_capacity(extent); + for (idx, init) in info.segments.iter() { + let data = &data[*idx]; + assert_eq!(data.len(), init.data.len()); + let offset = usize::try_from(init.offset - info.min_addr).unwrap(); + if image.len() < offset { + image.resize(offset, 0u8); + image.extend_from_slice(data); + } else { + image.splice( + offset..(offset + data.len()).min(image.len()), + data.iter().copied(), + ); + } + } + assert_eq!(image.len(), extent); + assert_eq!(image.capacity(), extent); + let mut offset = if info.segments.len() > 0 { + info.min_addr + } else { + 0 + }; + + // Chop off trailing zeros from the image as memory is already + // zero-initialized. Note that `i` is the position of a nonzero + // entry here, so to not lose it we truncate to `i + 1`. + if let Some(i) = image.iter().rposition(|i| *i != 0) { + image.truncate(i + 1); + } + + // Also chop off leading zeros, if any. + if let Some(i) = image.iter().position(|i| *i != 0) { + offset += i as u64; + image.drain(..i); + } + let mut len = u64::try_from(image.len()).unwrap(); + + // The goal is to enable mapping this image directly into memory, so + // the offset into linear memory must be a multiple of the page + // size. If that's not already the case then the image is padded at + // the front and back with extra zeros as necessary + if offset % page_size != 0 { + let zero_padding = offset % page_size; + self.data.push(vec![0; zero_padding as usize].into()); + offset -= zero_padding; + len += zero_padding; + } + self.data.push(image.into()); + if len % page_size != 0 { + let zero_padding = page_size - (len % page_size); + self.data.push(vec![0; zero_padding as usize].into()); + len += zero_padding; + } + + // Offset/length should now always be page-aligned. + assert!(offset % page_size == 0); + assert!(len % page_size == 0); + + // Create the `StaticMemoryInitializer` which describes this image, + // only needed if the image is actually present and has a nonzero + // length. The `offset` has been calculates above, originally + // sourced from `info.min_addr`. The `data` field is the extent + // within the final data segment we'll emit to an ELF image, which + // is the concatenation of `self.data`, so here it's the size of + // the section-so-far plus the current segment we're appending. + let len = u32::try_from(len).unwrap(); + let init = if len > 0 { + Some(StaticMemoryInitializer { + offset, + data: module_data_size..module_data_size + len, + }) + } else { + None + }; + let idx = map.push(init); + assert_eq!(idx, memory); + module_data_size += len; + } + self.data_align = Some(page_size); + self.module.memory_initialization = MemoryInitialization::Static { map }; + } + + /// Attempts to convert the module's table initializers to + /// FuncTable form where possible. This enables lazy table + /// initialization later by providing a one-to-one map of initial + /// table values, without having to parse all segments. + pub fn try_func_table_init(&mut self) { + // This should be large enough to support very large Wasm + // modules with huge funcref tables, but small enough to avoid + // OOMs or DoS on truly sparse tables. + const MAX_FUNC_TABLE_SIZE: u32 = 1024 * 1024; + + // First convert any element-initialized tables to images of just that + // single function if the minimum size of the table allows doing so. + for ((_, init), (_, plan)) in self + .module + .table_initialization + .initial_values + .iter_mut() + .zip( + self.module + .table_plans + .iter() + .skip(self.module.num_imported_tables), + ) + { + let table_size = plan.table.minimum; + if table_size > MAX_FUNC_TABLE_SIZE { + continue; + } + if let TableInitialValue::FuncRef(val) = *init { + *init = TableInitialValue::Null { + precomputed: vec![val; table_size as usize], + }; + } + } + + let mut segments = mem::take(&mut self.module.table_initialization.segments) + .into_iter() + .peekable(); + + // The goal of this loop is to interpret a table segment and apply it + // "statically" to a local table. This will iterate over segments and + // apply them one-by-one to each table. + // + // If any segment can't be applied, however, then this loop exits and + // all remaining segments are placed back into the segment list. This is + // because segments are supposed to be initialized one-at-a-time which + // means that intermediate state is visible with respect to traps. If + // anything isn't statically known to not trap it's pessimistically + // assumed to trap meaning all further segment initializers must be + // applied manually at instantiation time. + while let Some(segment) = segments.peek() { + let defined_index = match self.module.defined_table_index(segment.table_index) { + Some(index) => index, + // Skip imported tables: we can't provide a preconstructed + // table for them, because their values depend on the + // imported table overlaid with whatever segments we have. + None => break, + }; + + // If the base of this segment is dynamic, then we can't + // include it in the statically-built array of initial + // contents. + if segment.base.is_some() { + break; + } + + // Get the end of this segment. If out-of-bounds, or too + // large for our dense table representation, then skip the + // segment. + let top = match segment.offset.checked_add(segment.elements.len()) { + Some(top) => top, + None => break, + }; + let table_size = self.module.table_plans[segment.table_index].table.minimum; + if top > table_size || top > MAX_FUNC_TABLE_SIZE { + break; + } + + match self.module.table_plans[segment.table_index] + .table + .wasm_ty + .heap_type + { + WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {} + // If this is not a funcref table, then we can't support a + // pre-computed table of function indices. Technically this + // initializer won't trap so we could continue processing + // segments, but that's left as a future optimization if + // necessary. + WasmHeapType::Extern => break, + } + + // Function indices can be optimized here, but fully general + // expressions are deferred to get evaluated at runtime. + let function_elements = match &segment.elements { + TableSegmentElements::Functions(indices) => indices, + TableSegmentElements::Expressions(_) => break, + }; + + let precomputed = + match &mut self.module.table_initialization.initial_values[defined_index] { + TableInitialValue::Null { precomputed } => precomputed, + + // If this table is still listed as an initial value here + // then that means the initial size of the table doesn't + // support a precomputed function list, so skip this. + // Technically this won't trap so it's possible to process + // further initializers, but that's left as a future + // optimization. + TableInitialValue::FuncRef(_) | TableInitialValue::GlobalGet(_) => break, + }; + + // At this point we're committing to pre-initializing the table + // with the `segment` that's being iterated over. This segment is + // applied to the `precomputed` list for the table by ensuring + // it's large enough to hold the segment and then copying the + // segment into the precomputed list. + if precomputed.len() < top as usize { + precomputed.resize(top as usize, FuncIndex::reserved_value()); + } + let dst = &mut precomputed[(segment.offset as usize)..(top as usize)]; + dst.copy_from_slice(&function_elements); + + // advance the iterator to see the next segment + let _ = segments.next(); + } + self.module.table_initialization.segments = segments.collect(); + } +} From 7c914be62f74beb14803854f4b2445bc4681db9d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 28 Mar 2024 08:22:55 -0700 Subject: [PATCH 4/5] Fix doc link --- crates/environ/src/module_artifacts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index aa1e739519b9..b35479b10e88 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -115,7 +115,7 @@ pub struct Metadata { pub dwarf: Vec<(u8, Range)>, } -/// Value of a configured setting for a [`Compiler`] +/// Value of a configured setting for a [`Compiler`](crate::Compiler) #[derive(Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] pub enum FlagValue<'a> { /// Name of the value that has been configured for this setting. From c938d4de8f58d2331f3ce881bbe818832e66c8e2 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 28 Mar 2024 08:49:26 -0700 Subject: [PATCH 5/5] Fix doc link --- crates/wasmtime/src/compile/code_builder.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/wasmtime/src/compile/code_builder.rs b/crates/wasmtime/src/compile/code_builder.rs index 71d914588b81..35d3990ab769 100644 --- a/crates/wasmtime/src/compile/code_builder.rs +++ b/crates/wasmtime/src/compile/code_builder.rs @@ -162,7 +162,8 @@ impl<'a> CodeBuilder<'a> { } /// Same as [`CodeBuilder::compile_module_serialized`] except that it - /// compiles a serialized [`Component`] instead of a module. + /// compiles a serialized [`Component`](crate::component::Component) + /// instead of a module. #[cfg(feature = "component-model")] #[cfg_attr(docsrs, doc(cfg(feature = "component-model")))] pub fn compile_component_serialized(&self) -> Result> {