diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index ad2aa057a195..47a1a86360cd 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -16,7 +16,7 @@ workspace = true [dependencies] anyhow = { workspace = true } log = { workspace = true } -wasmtime-environ = { workspace = true } +wasmtime-environ = { workspace = true, features = ['compile'] } cranelift-wasm = { workspace = true } cranelift-codegen = { workspace = true, features = ["host-arch"] } cranelift-frontend = { workspace = true } diff --git a/crates/environ/Cargo.toml b/crates/environ/Cargo.toml index 50093112220b..39e536fc0c07 100644 --- a/crates/environ/Cargo.toml +++ b/crates/environ/Cargo.toml @@ -25,8 +25,8 @@ thiserror = { workspace = true } serde = "1.0.188" serde_derive = "1.0.188" log = { workspace = true } -gimli = { workspace = true, features = ["write"] } -object = { workspace = true, features = ['write_core'] } +gimli = { workspace = true } +object = { workspace = true } rustc-demangle = { version = "0.1.16", optional = true } target-lexicon = { workspace = true } wasm-encoder = { workspace = true, optional = true } @@ -50,3 +50,4 @@ component-model = [ ] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] gc = [] +compile = ['gimli/write', 'object/write_core'] diff --git a/crates/environ/src/address_map.rs b/crates/environ/src/address_map.rs index d8faba78a973..a707d69b4708 100644 --- a/crates/environ/src/address_map.rs +++ b/crates/environ/src/address_map.rs @@ -1,10 +1,7 @@ //! Data structures to provide transformation of the source -use crate::obj::ELF_WASMTIME_ADDRMAP; -use object::write::{Object, StandardSegment}; -use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; +use object::{Bytes, LittleEndian, U32Bytes}; use serde_derive::{Deserialize, Serialize}; -use std::ops::Range; /// Single source location to generated address mapping. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] @@ -52,69 +49,6 @@ impl Default for FilePos { } } -/// Builder for the address map section of a wasmtime compilation image. -/// -/// This builder is used to conveniently built the `ELF_WASMTIME_ADDRMAP` -/// section by compilers, and provides utilities to directly insert the results -/// into an `Object`. -#[derive(Default)] -pub struct AddressMapSection { - offsets: Vec>, - positions: Vec>, - last_offset: u32, -} - -impl AddressMapSection { - /// Pushes a new set of instruction mapping information for a function added - /// in the exectuable. - /// - /// The `func` argument here is the range of the function, relative to the - /// start of the text section in the executable. The `instrs` provided are - /// the descriptors for instructions in the function and their various - /// mappings back to original source positions. - /// - /// This is required to be called for `func` values that are strictly - /// increasing in addresses (e.g. as the object is built). Additionally the - /// `instrs` map must be sorted based on code offset in the native text - /// section. - pub fn push(&mut self, func: Range, instrs: &[InstructionAddressMap]) { - // NB: for now this only supports <=4GB text sections in object files. - // Alternative schemes will need to be created for >32-bit offsets to - // avoid making this section overly large. - let func_start = u32::try_from(func.start).unwrap(); - let func_end = u32::try_from(func.end).unwrap(); - - self.offsets.reserve(instrs.len()); - self.positions.reserve(instrs.len()); - for map in instrs { - // Sanity-check to ensure that functions are pushed in-order, otherwise - // the `offsets` array won't be sorted which is our goal. - let pos = func_start + map.code_offset; - assert!(pos >= self.last_offset); - self.offsets.push(U32Bytes::new(LittleEndian, pos)); - self.positions - .push(U32Bytes::new(LittleEndian, map.srcloc.0)); - self.last_offset = pos; - } - self.last_offset = func_end; - } - - /// Finishes encoding this section into the `Object` provided. - pub fn append_to(self, obj: &mut Object) { - let section = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASMTIME_ADDRMAP.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - - // NB: this matches the encoding expected by `lookup` below. - let amt = u32::try_from(self.offsets.len()).unwrap(); - obj.append_section_data(section, &amt.to_le_bytes(), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.positions), 1); - } -} - /// Parse an `ELF_WASMTIME_ADDRMAP` section, returning the slice of code offsets /// and the slice of associated file positions for each offset. fn parse_address_map( diff --git a/crates/environ/src/compile/address_map.rs b/crates/environ/src/compile/address_map.rs new file mode 100644 index 000000000000..614ccc3cbc94 --- /dev/null +++ b/crates/environ/src/compile/address_map.rs @@ -0,0 +1,72 @@ +//! Data structures to provide transformation of the source + +use crate::obj::ELF_WASMTIME_ADDRMAP; +use crate::InstructionAddressMap; +use object::write::{Object, StandardSegment}; +use object::{LittleEndian, SectionKind, U32Bytes}; +use std::ops::Range; + +/// Builder for the address map section of a wasmtime compilation image. +/// +/// This builder is used to conveniently built the `ELF_WASMTIME_ADDRMAP` +/// section by compilers, and provides utilities to directly insert the results +/// into an `Object`. +#[derive(Default)] +pub struct AddressMapSection { + offsets: Vec>, + positions: Vec>, + last_offset: u32, +} + +impl AddressMapSection { + /// Pushes a new set of instruction mapping information for a function added + /// in the exectuable. + /// + /// The `func` argument here is the range of the function, relative to the + /// start of the text section in the executable. The `instrs` provided are + /// the descriptors for instructions in the function and their various + /// mappings back to original source positions. + /// + /// This is required to be called for `func` values that are strictly + /// increasing in addresses (e.g. as the object is built). Additionally the + /// `instrs` map must be sorted based on code offset in the native text + /// section. + pub fn push(&mut self, func: Range, instrs: &[InstructionAddressMap]) { + // NB: for now this only supports <=4GB text sections in object files. + // Alternative schemes will need to be created for >32-bit offsets to + // avoid making this section overly large. + let func_start = u32::try_from(func.start).unwrap(); + let func_end = u32::try_from(func.end).unwrap(); + + self.offsets.reserve(instrs.len()); + self.positions.reserve(instrs.len()); + for map in instrs { + // Sanity-check to ensure that functions are pushed in-order, otherwise + // the `offsets` array won't be sorted which is our goal. + let pos = func_start + map.code_offset; + assert!(pos >= self.last_offset); + self.offsets.push(U32Bytes::new(LittleEndian, pos)); + self.positions.push(U32Bytes::new( + LittleEndian, + map.srcloc.file_offset().unwrap_or(u32::MAX), + )); + self.last_offset = pos; + } + self.last_offset = func_end; + } + + /// Finishes encoding this section into the `Object` provided. + pub fn append_to(self, obj: &mut Object) { + let section = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + ELF_WASMTIME_ADDRMAP.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + + // NB: this matches the encoding expected by `lookup` below. + let amt = u32::try_from(self.offsets.len()).unwrap(); + obj.append_section_data(section, &amt.to_le_bytes(), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.positions), 1); + } +} diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compile/mod.rs similarity index 88% rename from crates/environ/src/compilation.rs rename to crates/environ/src/compile/mod.rs index b3c5863fa68c..993a6c6cddd1 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compile/mod.rs @@ -3,13 +3,13 @@ use crate::{obj, Tunables}; use crate::{ - BuiltinFunctionIndex, DefinedFuncIndex, FilePos, FuncIndex, FunctionBodyData, - ModuleTranslation, ModuleTypesBuilder, PrimaryMap, StackMap, WasmError, WasmFuncType, + BuiltinFunctionIndex, DefinedFuncIndex, FlagValue, FuncIndex, FunctionBodyData, FunctionLoc, + ModuleTranslation, ModuleTypesBuilder, ObjectKind, PrimaryMap, WasmError, WasmFuncType, + WasmFunctionInfo, }; use anyhow::Result; use object::write::{Object, SymbolId}; use object::{Architecture, BinaryFormat, FileFlags}; -use serde_derive::{Deserialize, Serialize}; use std::any::Any; use std::borrow::Cow; use std::fmt; @@ -17,37 +17,13 @@ use std::path; use std::sync::Arc; use thiserror::Error; -/// Information about a function, such as trap information, address map, -/// and stack maps. -#[derive(Serialize, Deserialize, Default)] -#[allow(missing_docs)] -pub struct WasmFunctionInfo { - pub start_srcloc: FilePos, - pub stack_maps: Box<[StackMapInformation]>, -} - -/// Description of where a function is located in the text section of a -/// compiled image. -#[derive(Copy, Clone, Serialize, Deserialize)] -pub struct FunctionLoc { - /// The byte offset from the start of the text section where this - /// function starts. - pub start: u32, - /// The byte length of this function's function body. - pub length: u32, -} +mod address_map; +mod module_artifacts; +mod trap_encoding; -/// The offset within a function of a GC safepoint, and its associated stack -/// map. -#[derive(Serialize, Deserialize, Debug)] -pub struct StackMapInformation { - /// The offset of the GC safepoint within the function's native code. It is - /// relative to the beginning of the function. - pub code_offset: u32, - - /// The stack map for identifying live GC refs at the GC safepoint. - pub stack_map: StackMap, -} +pub use self::address_map::*; +pub use self::module_artifacts::*; +pub use self::trap_encoding::*; /// An error while compiling WebAssembly to machine code. #[derive(Error, Debug)] @@ -171,14 +147,6 @@ pub enum SettingKind { Preset, } -/// Types of objects that can be created by `Compiler::object` -pub enum ObjectKind { - /// A core wasm compilation artifact - Module, - /// A component compilation artifact - Component, -} - /// An implementation of a compiler which can compile WebAssembly functions to /// machine code and perform other miscellaneous tasks needed by the JIT runtime. pub trait Compiler: Send + Sync { @@ -419,24 +387,3 @@ pub trait Compiler: Send + Sync { None } } - -/// Value of a configured setting for a [`Compiler`] -#[derive(Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] -pub enum FlagValue<'a> { - /// Name of the value that has been configured for this setting. - Enum(&'a str), - /// The numerical value of the configured settings. - Num(u8), - /// Whether the setting is on or off. - Bool(bool), -} - -impl fmt::Display for FlagValue<'_> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Self::Enum(v) => v.fmt(f), - Self::Num(v) => v.fmt(f), - Self::Bool(v) => v.fmt(f), - } - } -} diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs new file mode 100644 index 000000000000..7210f9139c62 --- /dev/null +++ b/crates/environ/src/compile/module_artifacts.rs @@ -0,0 +1,300 @@ +//! Definitions of runtime structures and metadata which are serialized into ELF +//! with `bincode` as part of a module's compilation process. + +use crate::{ + obj, CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FunctionLoc, FunctionName, + MemoryInitialization, Metadata, ModuleTranslation, PrimaryMap, Tunables, +}; +use anyhow::{bail, Result}; +use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; +use object::SectionKind; +use std::ops::Range; +use wasmtime_types::ModuleInternedTypeIndex; + +/// Helper structure to create an ELF file as a compilation artifact. +/// +/// This structure exposes the process which Wasmtime will encode a core wasm +/// module into an ELF file, notably managing data sections and all that good +/// business going into the final file. +pub struct ObjectBuilder<'a> { + /// The `object`-crate-defined ELF file write we're using. + obj: Object<'a>, + + /// General compilation configuration. + tunables: &'a Tunables, + + /// The section identifier for "rodata" which is where wasm data segments + /// will go. + data: SectionId, + + /// The section identifier for function name information, or otherwise where + /// the `name` custom section of wasm is copied into. + /// + /// This is optional and lazily created on demand. + names: Option, + + /// The section identifier for dwarf information copied from the original + /// wasm files. + /// + /// This is optional and lazily created on demand. + dwarf: Option, +} + +impl<'a> ObjectBuilder<'a> { + /// Creates a new builder for the `obj` specified. + pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { + let data = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_WASM_DATA.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + ObjectBuilder { + obj, + tunables, + data, + names: None, + dwarf: None, + } + } + + /// Completes compilation of the `translation` specified, inserting + /// everything necessary into the `Object` being built. + /// + /// This function will consume the final results of compiling a wasm module + /// and finish the ELF image in-progress as part of `self.obj` by appending + /// any compiler-agnostic sections. + /// + /// The auxiliary `CompiledModuleInfo` structure returned here has also been + /// serialized into the object returned, but if the caller will quickly + /// turn-around and invoke `CompiledModule::from_artifacts` after this then + /// the information can be passed to that method to avoid extra + /// deserialization. This is done to avoid a serialize-then-deserialize for + /// API calls like `Module::new` where the compiled module is immediately + /// going to be used. + /// + /// The various arguments here are: + /// + /// * `translation` - the core wasm translation that's being completed. + /// + /// * `funcs` - compilation metadata about functions within the translation + /// as well as where the functions are located in the text section and any + /// associated trampolines. + /// + /// * `wasm_to_native_trampolines` - list of all trampolines necessary for + /// Wasm callers calling native callees (e.g. `Func::wrap`). One for each + /// function signature in the module. Must be sorted by `SignatureIndex`. + /// + /// Returns the `CompiledModuleInfo` corresponding to this core Wasm module + /// as a result of this append operation. This is then serialized into the + /// final artifact by the caller. + pub fn append( + &mut self, + translation: ModuleTranslation<'_>, + funcs: PrimaryMap, + wasm_to_native_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>, + ) -> Result { + let ModuleTranslation { + mut module, + debuginfo, + has_unparsed_debuginfo, + data, + data_align, + passive_data, + .. + } = translation; + + // Place all data from the wasm module into a section which will the + // source of the data later at runtime. This additionally keeps track of + // the offset of + let mut total_data_len = 0; + let data_offset = self + .obj + .append_section_data(self.data, &[], data_align.unwrap_or(1)); + for (i, data) in data.iter().enumerate() { + // The first data segment has its alignment specified as the alignment + // for the entire section, but everything afterwards is adjacent so it + // has alignment of 1. + let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; + self.obj.append_section_data(self.data, data, align); + total_data_len += data.len(); + } + for data in passive_data.iter() { + self.obj.append_section_data(self.data, data, 1); + } + + // If any names are present in the module then the `ELF_NAME_DATA` section + // is create and appended. + let mut func_names = Vec::new(); + if debuginfo.name_section.func_names.len() > 0 { + let name_id = *self.names.get_or_insert_with(|| { + self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_NAME_DATA.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ) + }); + let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); + sorted_names.sort_by_key(|(idx, _name)| *idx); + for (idx, name) in sorted_names { + let offset = self.obj.append_section_data(name_id, name.as_bytes(), 1); + let offset = match u32::try_from(offset) { + Ok(offset) => offset, + Err(_) => bail!("name section too large (> 4gb)"), + }; + let len = u32::try_from(name.len()).unwrap(); + func_names.push(FunctionName { + idx: *idx, + offset, + len, + }); + } + } + + // Data offsets in `MemoryInitialization` are offsets within the + // `translation.data` list concatenated which is now present in the data + // segment that's appended to the object. Increase the offsets by + // `self.data_size` to account for any previously added module. + let data_offset = u32::try_from(data_offset).unwrap(); + match &mut module.memory_initialization { + MemoryInitialization::Segmented(list) => { + for segment in list { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + MemoryInitialization::Static { map } => { + for (_, segment) in map { + if let Some(segment) = segment { + segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); + segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); + } + } + } + } + + // Data offsets for passive data are relative to the start of + // `translation.passive_data` which was appended to the data segment + // of this object, after active data in `translation.data`. Update the + // offsets to account prior modules added in addition to active data. + let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); + for (_, range) in module.passive_data_map.iter_mut() { + range.start = range.start.checked_add(data_offset).unwrap(); + range.end = range.end.checked_add(data_offset).unwrap(); + } + + // Insert the wasm raw wasm-based debuginfo into the output, if + // requested. Note that this is distinct from the native debuginfo + // possibly generated by the native compiler, hence these sections + // getting wasm-specific names. + let mut dwarf = Vec::new(); + if self.tunables.parse_wasm_debuginfo { + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_abbrev); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_addr); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_aranges); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_info); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line_str); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str); + self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str_offsets); + self.push_debug(&mut dwarf, &debuginfo.debug_ranges); + self.push_debug(&mut dwarf, &debuginfo.debug_rnglists); + } + // Sort this for binary-search-lookup later in `symbolize_context`. + dwarf.sort_by_key(|(id, _)| *id); + + Ok(CompiledModuleInfo { + module, + funcs, + wasm_to_native_trampolines, + func_names, + meta: Metadata { + native_debug_info_present: self.tunables.generate_native_debuginfo, + has_unparsed_debuginfo, + code_section_offset: debuginfo.wasm_file.code_section_offset, + has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, + dwarf, + }, + }) + } + + fn push_debug<'b, T>(&mut self, dwarf: &mut Vec<(u8, Range)>, section: &T) + where + T: gimli::Section>, + { + let data = section.reader().slice(); + if data.is_empty() { + return; + } + let section_id = *self.dwarf.get_or_insert_with(|| { + self.obj.add_section( + self.obj.segment_name(StandardSegment::Debug).to_vec(), + obj::ELF_WASMTIME_DWARF.as_bytes().to_vec(), + SectionKind::Debug, + ) + }); + let offset = self.obj.append_section_data(section_id, data, 1); + dwarf.push((T::id() as u8, offset..offset + data.len() as u64)); + } + + /// Creates the `ELF_WASMTIME_INFO` section from the given serializable data + /// structure. + pub fn serialize_info(&mut self, info: &T) + where + T: serde::Serialize, + { + let section = self.obj.add_section( + self.obj.segment_name(StandardSegment::Data).to_vec(), + obj::ELF_WASMTIME_INFO.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + let data = bincode::serialize(info).unwrap(); + self.obj.set_section_data(section, data, 1); + } + + /// Serializes `self` into a buffer. This can be used for execution as well + /// as serialization. + pub fn finish(self, t: &mut T) -> Result<()> { + self.obj.emit(t).map_err(|e| e.into()) + } +} + +/// A type which can be the result of serializing an object. +pub trait FinishedObject: Sized { + /// Emit the object as `Self`. + fn finish_object(obj: ObjectBuilder<'_>) -> Result; +} + +impl FinishedObject for Vec { + fn finish_object(obj: ObjectBuilder<'_>) -> Result { + let mut result = ObjectVec::default(); + obj.finish(&mut result)?; + return Ok(result.0); + + #[derive(Default)] + struct ObjectVec(Vec); + + impl WritableBuffer for ObjectVec { + fn len(&self) -> usize { + self.0.len() + } + + fn reserve(&mut self, additional: usize) -> Result<(), ()> { + assert_eq!(self.0.len(), 0, "cannot reserve twice"); + self.0 = Vec::with_capacity(additional); + Ok(()) + } + + fn resize(&mut self, new_len: usize) { + if new_len <= self.0.len() { + self.0.truncate(new_len) + } else { + self.0.extend(vec![0; new_len - self.0.len()]) + } + } + + fn write_bytes(&mut self, val: &[u8]) { + self.0.extend(val); + } + } + } +} diff --git a/crates/environ/src/compile/trap_encoding.rs b/crates/environ/src/compile/trap_encoding.rs new file mode 100644 index 000000000000..7f38523f1721 --- /dev/null +++ b/crates/environ/src/compile/trap_encoding.rs @@ -0,0 +1,69 @@ +use crate::obj::ELF_WASMTIME_TRAPS; +use crate::TrapInformation; +use object::write::{Object, StandardSegment}; +use object::{LittleEndian, SectionKind, U32Bytes}; +use std::ops::Range; + +/// A helper structure to build the custom-encoded section of a wasmtime +/// compilation image which encodes trap information. +/// +/// This structure is incrementally fed the results of compiling individual +/// functions and handles all the encoding internally, allowing usage of +/// `lookup_trap_code` below with the resulting section. +#[derive(Default)] +pub struct TrapEncodingBuilder { + offsets: Vec>, + traps: Vec, + last_offset: u32, +} + +impl TrapEncodingBuilder { + /// Appends trap information about a function into this section. + /// + /// This function is called to describe traps for the `func` range + /// specified. The `func` offsets are specified relative to the text section + /// itself, and the `traps` offsets are specified relative to the start of + /// `func`. + /// + /// This is required to be called in-order for increasing ranges of `func` + /// to ensure the final array is properly sorted. Additionally `traps` must + /// be sorted. + pub fn push(&mut self, func: Range, traps: &[TrapInformation]) { + // NB: for now this only supports <=4GB text sections in object files. + // Alternative schemes will need to be created for >32-bit offsets to + // avoid making this section overly large. + let func_start = u32::try_from(func.start).unwrap(); + let func_end = u32::try_from(func.end).unwrap(); + + // Sanity-check to ensure that functions are pushed in-order, otherwise + // the `offsets` array won't be sorted which is our goal. + assert!(func_start >= self.last_offset); + + self.offsets.reserve(traps.len()); + self.traps.reserve(traps.len()); + for info in traps { + let pos = func_start + info.code_offset; + assert!(pos >= self.last_offset); + self.offsets.push(U32Bytes::new(LittleEndian, pos)); + self.traps.push(info.trap_code as u8); + self.last_offset = pos; + } + + self.last_offset = func_end; + } + + /// Encodes this section into the object provided. + pub fn append_to(self, obj: &mut Object) { + let section = obj.add_section( + obj.segment_name(StandardSegment::Data).to_vec(), + ELF_WASMTIME_TRAPS.as_bytes().to_vec(), + SectionKind::ReadOnlyData, + ); + + // NB: this matches the encoding expected by `lookup` below. + let amt = u32::try_from(self.traps.len()).unwrap(); + obj.append_section_data(section, &amt.to_le_bytes(), 1); + obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); + obj.append_section_data(section, &self.traps, 1); + } +} diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rs index 4ec045a1e55c..67dce36814c0 100644 --- a/crates/environ/src/component.rs +++ b/crates/environ/src/component.rs @@ -38,19 +38,25 @@ pub const MAX_FLAT_PARAMS: usize = 16; pub const MAX_FLAT_RESULTS: usize = 1; mod artifacts; -mod compiler; -pub mod dfg; mod info; -mod translate; mod types; mod vmcomponent_offsets; pub use self::artifacts::*; -pub use self::compiler::*; pub use self::info::*; -pub use self::translate::*; pub use self::types::*; pub use self::vmcomponent_offsets::*; +#[cfg(feature = "compile")] +mod compiler; +#[cfg(feature = "compile")] +pub mod dfg; +#[cfg(feature = "compile")] +mod translate; +#[cfg(feature = "compile")] +pub use self::compiler::*; +#[cfg(feature = "compile")] +pub use self::translate::*; + /// Helper macro to iterate over the transcoders that the host will provide /// adapter modules through libcalls. #[macro_export] diff --git a/crates/environ/src/component/artifacts.rs b/crates/environ/src/component/artifacts.rs index 3acafbd30a1b..d6cd79b73b4d 100644 --- a/crates/environ/src/component/artifacts.rs +++ b/crates/environ/src/component/artifacts.rs @@ -2,7 +2,7 @@ //! which are serialized with `bincode` into output ELF files. use crate::{ - component::{AllCallFunc, Component, ComponentTypes, TrampolineIndex, TypeComponentIndex}, + component::{Component, ComponentTypes, TrampolineIndex, TypeComponentIndex}, CompiledModuleInfo, FunctionLoc, PrimaryMap, StaticModuleIndex, }; use serde_derive::{Deserialize, Serialize}; @@ -43,3 +43,30 @@ pub struct CompiledComponentInfo { /// intrinsic. pub resource_drop_wasm_to_native_trampoline: Option, } + +/// A triple of related functions/trampolines variants with differing calling +/// conventions: `{wasm,array,native}_call`. +/// +/// Generic so we can use this with either the `Box`s that +/// implementations of the compiler trait return or with `FunctionLoc`s inside +/// an object file, for example. +#[derive(Clone, Copy, Default, Serialize, Deserialize)] +pub struct AllCallFunc { + /// The function exposing the Wasm calling convention. + pub wasm_call: T, + /// The function exposing the array calling convention. + pub array_call: T, + /// The function exposing the native calling convention. + pub native_call: T, +} + +impl AllCallFunc { + /// Map an `AllCallFunc` into an `AllCallFunc`. + pub fn map(self, mut f: impl FnMut(T) -> U) -> AllCallFunc { + AllCallFunc { + wasm_call: f(self.wasm_call), + array_call: f(self.array_call), + native_call: f(self.native_call), + } + } +} diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs index b89f67a77fc3..ee0fe41a6eeb 100644 --- a/crates/environ/src/component/compiler.rs +++ b/crates/environ/src/component/compiler.rs @@ -1,35 +1,7 @@ -use crate::component::{ComponentTranslation, ComponentTypesBuilder, TrampolineIndex}; +use crate::component::{AllCallFunc, ComponentTranslation, ComponentTypesBuilder, TrampolineIndex}; use anyhow::Result; -use serde_derive::{Deserialize, Serialize}; use std::any::Any; -/// A triple of related functions/trampolines variants with differing calling -/// conventions: `{wasm,array,native}_call`. -/// -/// Generic so we can use this with either the `Box`s that -/// implementations of the compiler trait return or with `FunctionLoc`s inside -/// an object file, for example. -#[derive(Clone, Copy, Default, Serialize, Deserialize)] -pub struct AllCallFunc { - /// The function exposing the Wasm calling convention. - pub wasm_call: T, - /// The function exposing the array calling convention. - pub array_call: T, - /// The function exposing the native calling convention. - pub native_call: T, -} - -impl AllCallFunc { - /// Map an `AllCallFunc` into an `AllCallFunc`. - pub fn map(self, mut f: impl FnMut(T) -> U) -> AllCallFunc { - AllCallFunc { - wasm_call: f(self.wasm_call), - array_call: f(self.array_call), - native_call: f(self.native_call), - } - } -} - /// Compilation support necessary for components. pub trait ComponentCompiler: Send + Sync { /// Compiles the pieces necessary to create a `VMFuncRef` for the diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index fc47dbc3082c..4c9abeb7221a 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -455,7 +455,85 @@ pub enum StringEncoding { CompactUtf16, } -pub use crate::fact::{FixedEncoding, Transcode}; +/// Possible transcoding operations that must be provided by the host. +/// +/// Note that each transcoding operation may have a unique signature depending +/// on the precise operation. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] +pub enum Transcode { + Copy(FixedEncoding), + Latin1ToUtf16, + Latin1ToUtf8, + Utf16ToCompactProbablyUtf16, + Utf16ToCompactUtf16, + Utf16ToLatin1, + Utf16ToUtf8, + Utf8ToCompactUtf16, + Utf8ToLatin1, + Utf8ToUtf16, +} + +impl Transcode { + /// Get this transcoding's symbol fragment. + pub fn symbol_fragment(&self) -> &'static str { + match self { + Transcode::Copy(x) => match x { + FixedEncoding::Utf8 => "copy_utf8", + FixedEncoding::Utf16 => "copy_utf16", + FixedEncoding::Latin1 => "copy_latin1", + }, + Transcode::Latin1ToUtf16 => "latin1_to_utf16", + Transcode::Latin1ToUtf8 => "latin1_to_utf8", + Transcode::Utf16ToCompactProbablyUtf16 => "utf16_to_compact_probably_utf16", + Transcode::Utf16ToCompactUtf16 => "utf16_to_compact_utf16", + Transcode::Utf16ToLatin1 => "utf16_to_latin1", + Transcode::Utf16ToUtf8 => "utf16_to_utf8", + Transcode::Utf8ToCompactUtf16 => "utf8_to_compact_utf16", + Transcode::Utf8ToLatin1 => "utf8_to_latin1", + Transcode::Utf8ToUtf16 => "utf8_to_utf16", + } + } + + /// Returns a human-readable description for this transcoding operation. + pub fn desc(&self) -> &'static str { + match self { + Transcode::Copy(FixedEncoding::Utf8) => "utf8-to-utf8", + Transcode::Copy(FixedEncoding::Utf16) => "utf16-to-utf16", + Transcode::Copy(FixedEncoding::Latin1) => "latin1-to-latin1", + Transcode::Latin1ToUtf16 => "latin1-to-utf16", + Transcode::Latin1ToUtf8 => "latin1-to-utf8", + Transcode::Utf16ToCompactProbablyUtf16 => "utf16-to-compact-probably-utf16", + Transcode::Utf16ToCompactUtf16 => "utf16-to-compact-utf16", + Transcode::Utf16ToLatin1 => "utf16-to-latin1", + Transcode::Utf16ToUtf8 => "utf16-to-utf8", + Transcode::Utf8ToCompactUtf16 => "utf8-to-compact-utf16", + Transcode::Utf8ToLatin1 => "utf8-to-latin1", + Transcode::Utf8ToUtf16 => "utf8-to-utf16", + } + } +} + +#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] +#[allow(missing_docs)] +pub enum FixedEncoding { + Utf8, + Utf16, + Latin1, +} + +impl FixedEncoding { + /// Returns the byte width of unit loads/stores for this encoding, for + /// example the unit length is multiplied by this return value to get the + /// byte width of a string. + pub fn width(&self) -> u8 { + match self { + FixedEncoding::Utf8 => 1, + FixedEncoding::Utf16 => 2, + FixedEncoding::Latin1 => 1, + } + } +} /// Description of a new resource declared in a `GlobalInitializer::Resource` /// variant. diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rs index 70437996c996..0ad0b119d899 100644 --- a/crates/environ/src/fact.rs +++ b/crates/environ/src/fact.rs @@ -21,7 +21,7 @@ use crate::component::dfg::CoreDef; use crate::component::{ Adapter, AdapterOptions as AdapterOptionsDfg, ComponentTypesBuilder, FlatType, InterfaceType, - StringEncoding, TypeFuncIndex, + StringEncoding, Transcode, TypeFuncIndex, }; use crate::fact::transcode::Transcoder; use crate::{EntityRef, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap}; @@ -35,8 +35,6 @@ mod trampoline; mod transcode; mod traps; -pub use self::transcode::{FixedEncoding, Transcode}; - /// Representation of an adapter module. pub struct Module<'a> { /// Whether or not debug code is inserted into the adapters themselves. diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs index f85a60c6dc45..a71ffacb1e5c 100644 --- a/crates/environ/src/fact/trampoline.rs +++ b/crates/environ/src/fact/trampoline.rs @@ -16,13 +16,13 @@ //! can be somewhat arbitrary, an intentional decision. use crate::component::{ - CanonicalAbiInfo, ComponentTypesBuilder, FlatType, InterfaceType, StringEncoding, - TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex, - TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, VariantInfo, - FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, + CanonicalAbiInfo, ComponentTypesBuilder, FixedEncoding as FE, FlatType, InterfaceType, + StringEncoding, Transcode, TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, + TypeRecordIndex, TypeResourceTableIndex, TypeResultIndex, TypeTupleIndex, TypeVariantIndex, + VariantInfo, FLAG_MAY_ENTER, FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, }; use crate::fact::signature::Signature; -use crate::fact::transcode::{FixedEncoding as FE, Transcode, Transcoder}; +use crate::fact::transcode::Transcoder; use crate::fact::traps::Trap; use crate::fact::{ AdapterData, Body, Context, Function, FunctionId, Helper, HelperLocation, HelperType, Module, diff --git a/crates/environ/src/fact/transcode.rs b/crates/environ/src/fact/transcode.rs index ca34a8583156..e41b45c1ec91 100644 --- a/crates/environ/src/fact/transcode.rs +++ b/crates/environ/src/fact/transcode.rs @@ -1,6 +1,6 @@ +use crate::component::Transcode; use crate::fact::core_types::CoreTypes; use crate::MemoryIndex; -use serde_derive::{Deserialize, Serialize}; use wasm_encoder::{EntityType, ValType}; #[derive(Copy, Clone, Hash, Eq, PartialEq)] @@ -12,55 +12,6 @@ pub struct Transcoder { pub op: Transcode, } -/// Possible transcoding operations that must be provided by the host. -/// -/// Note that each transcoding operation may have a unique signature depending -/// on the precise operation. -#[allow(missing_docs)] -#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] -pub enum Transcode { - Copy(FixedEncoding), - Latin1ToUtf16, - Latin1ToUtf8, - Utf16ToCompactProbablyUtf16, - Utf16ToCompactUtf16, - Utf16ToLatin1, - Utf16ToUtf8, - Utf8ToCompactUtf16, - Utf8ToLatin1, - Utf8ToUtf16, -} - -impl Transcode { - /// Get this transcoding's symbol fragment. - pub fn symbol_fragment(&self) -> &'static str { - match self { - Transcode::Copy(x) => match x { - FixedEncoding::Utf8 => "copy_utf8", - FixedEncoding::Utf16 => "copy_utf16", - FixedEncoding::Latin1 => "copy_latin1", - }, - Transcode::Latin1ToUtf16 => "latin1_to_utf16", - Transcode::Latin1ToUtf8 => "latin1_to_utf8", - Transcode::Utf16ToCompactProbablyUtf16 => "utf16_to_compact_probably_utf16", - Transcode::Utf16ToCompactUtf16 => "utf16_to_compact_utf16", - Transcode::Utf16ToLatin1 => "utf16_to_latin1", - Transcode::Utf16ToUtf8 => "utf16_to_utf8", - Transcode::Utf8ToCompactUtf16 => "utf8_to_compact_utf16", - Transcode::Utf8ToLatin1 => "utf8_to_latin1", - Transcode::Utf8ToUtf16 => "utf8_to_utf16", - } - } -} - -#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] -#[allow(missing_docs)] -pub enum FixedEncoding { - Utf8, - Utf16, - Latin1, -} - impl Transcoder { pub fn name(&self) -> String { format!( @@ -136,33 +87,3 @@ impl Transcoder { EntityType::Function(ty) } } - -impl Transcode { - /// Returns a human-readable description for this transcoding operation. - pub fn desc(&self) -> &'static str { - match self { - Transcode::Copy(FixedEncoding::Utf8) => "utf8-to-utf8", - Transcode::Copy(FixedEncoding::Utf16) => "utf16-to-utf16", - Transcode::Copy(FixedEncoding::Latin1) => "latin1-to-latin1", - Transcode::Latin1ToUtf16 => "latin1-to-utf16", - Transcode::Latin1ToUtf8 => "latin1-to-utf8", - Transcode::Utf16ToCompactProbablyUtf16 => "utf16-to-compact-probably-utf16", - Transcode::Utf16ToCompactUtf16 => "utf16-to-compact-utf16", - Transcode::Utf16ToLatin1 => "utf16-to-latin1", - Transcode::Utf16ToUtf8 => "utf16-to-utf8", - Transcode::Utf8ToCompactUtf16 => "utf8-to-compact-utf16", - Transcode::Utf8ToLatin1 => "utf8-to-latin1", - Transcode::Utf8ToUtf16 => "utf8-to-utf16", - } - } -} - -impl FixedEncoding { - pub(crate) fn width(&self) -> u8 { - match self { - FixedEncoding::Utf8 => 1, - FixedEncoding::Utf16 => 2, - FixedEncoding::Latin1 => 1, - } - } -} diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 4df17658a11f..0e473596d7da 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -8,11 +8,9 @@ mod address_map; mod builtin; -mod compilation; mod demangling; mod module; mod module_artifacts; -mod module_environ; mod module_types; pub mod obj; mod ref_bits; @@ -24,10 +22,9 @@ mod vmoffsets; pub use crate::address_map::*; pub use crate::builtin::*; -pub use crate::compilation::*; pub use crate::demangling::*; pub use crate::module::*; -pub use crate::module_environ::*; +pub use crate::module_artifacts::*; pub use crate::module_types::*; pub use crate::ref_bits::*; pub use crate::scopevec::ScopeVec; @@ -37,13 +34,18 @@ pub use crate::tunables::*; pub use crate::vmoffsets::*; pub use object; -pub use crate::module_artifacts::{ - CompiledFunctionInfo, CompiledModuleInfo, FinishedObject, FunctionName, Metadata, ObjectBuilder, -}; +#[cfg(feature = "compile")] +mod compile; +#[cfg(feature = "compile")] +mod module_environ; +#[cfg(feature = "compile")] +pub use crate::compile::*; +#[cfg(feature = "compile")] +pub use crate::module_environ::*; #[cfg(feature = "component-model")] pub mod component; -#[cfg(feature = "component-model")] +#[cfg(all(feature = "component-model", feature = "compile"))] pub mod fact; // Reexport all of these type-level since they're quite commonly used and it's diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 24dec26af0ac..15ed0a325a70 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -1,11 +1,10 @@ //! Data structures for representing decoded wasm modules. -use crate::{ModuleTranslation, PrimaryMap, Tunables, WASM_PAGE_SIZE}; +use crate::{PrimaryMap, Tunables, WASM_PAGE_SIZE}; use cranelift_entity::{packed_option::ReservedValue, EntityRef}; use indexmap::IndexMap; use serde_derive::{Deserialize, Serialize}; use std::collections::BTreeMap; -use std::mem; use std::ops::Range; use wasmtime_types::*; @@ -179,347 +178,6 @@ pub enum MemoryInitialization { }, } -impl ModuleTranslation<'_> { - /// Attempts to convert segmented memory initialization into static - /// initialization for the module that this translation represents. - /// - /// If this module's memory initialization is not compatible with paged - /// initialization then this won't change anything. Otherwise if it is - /// compatible then the `memory_initialization` field will be updated. - /// - /// Takes a `page_size` argument in order to ensure that all - /// initialization is page-aligned for mmap-ability, and - /// `max_image_size_always_allowed` to control how we decide - /// whether to use static init. - /// - /// We will try to avoid generating very sparse images, which are - /// possible if e.g. a module has an initializer at offset 0 and a - /// very high offset (say, 1 GiB). To avoid this, we use a dual - /// condition: we always allow images less than - /// `max_image_size_always_allowed`, and the embedder of Wasmtime - /// can set this if desired to ensure that static init should - /// always be done if the size of the module or its heaps is - /// otherwise bounded by the system. We also allow images with - /// static init data bigger than that, but only if it is "dense", - /// defined as having at least half (50%) of its pages with some - /// data. - /// - /// We could do something slightly better by building a dense part - /// and keeping a sparse list of outlier/leftover segments (see - /// issue #3820). This would also allow mostly-static init of - /// modules that have some dynamically-placed data segments. But, - /// for now, this is sufficient to allow a system that "knows what - /// it's doing" to always get static init. - pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { - // This method only attempts to transform a `Segmented` memory init - // into a `Static` one, no other state. - if !self.module.memory_initialization.is_segmented() { - return; - } - - // First a dry run of memory initialization is performed. This - // collects information about the extent of memory initialized for each - // memory as well as the size of all data segments being copied in. - struct Memory { - data_size: u64, - min_addr: u64, - max_addr: u64, - // The `usize` here is a pointer into `self.data` which is the list - // of data segments corresponding to what was found in the original - // wasm module. - segments: Vec<(usize, StaticMemoryInitializer)>, - } - let mut info = PrimaryMap::with_capacity(self.module.memory_plans.len()); - for _ in 0..self.module.memory_plans.len() { - info.push(Memory { - data_size: 0, - min_addr: u64::MAX, - max_addr: 0, - segments: Vec::new(), - }); - } - let mut idx = 0; - let ok = self.module.memory_initialization.init_memory( - &mut (), - InitMemory::CompileTime(&self.module), - |(), memory, init| { - // Currently `Static` only applies to locally-defined memories, - // so if a data segment references an imported memory then - // transitioning to a `Static` memory initializer is not - // possible. - if self.module.defined_memory_index(memory).is_none() { - return false; - }; - let info = &mut info[memory]; - let data_len = u64::from(init.data.end - init.data.start); - if data_len > 0 { - info.data_size += data_len; - info.min_addr = info.min_addr.min(init.offset); - info.max_addr = info.max_addr.max(init.offset + data_len); - info.segments.push((idx, init.clone())); - } - idx += 1; - true - }, - ); - if !ok { - return; - } - - // Validate that the memory information collected is indeed valid for - // static memory initialization. - for info in info.values().filter(|i| i.data_size > 0) { - let image_size = info.max_addr - info.min_addr; - - // If the range of memory being initialized is less than twice the - // total size of the data itself then it's assumed that static - // initialization is ok. This means we'll at most double memory - // consumption during the memory image creation process, which is - // currently assumed to "probably be ok" but this will likely need - // tweaks over time. - if image_size < info.data_size.saturating_mul(2) { - continue; - } - - // If the memory initialization image is larger than the size of all - // data, then we still allow memory initialization if the image will - // be of a relatively modest size, such as 1MB here. - if image_size < max_image_size_always_allowed { - continue; - } - - // At this point memory initialization is concluded to be too - // expensive to do at compile time so it's entirely deferred to - // happen at runtime. - return; - } - - // Here's where we've now committed to changing to static memory. The - // memory initialization image is built here from the page data and then - // it's converted to a single initializer. - let data = mem::replace(&mut self.data, Vec::new()); - let mut map = PrimaryMap::with_capacity(info.len()); - let mut module_data_size = 0u32; - for (memory, info) in info.iter() { - // Create the in-memory `image` which is the initialized contents of - // this linear memory. - let extent = if info.segments.len() > 0 { - (info.max_addr - info.min_addr) as usize - } else { - 0 - }; - let mut image = Vec::with_capacity(extent); - for (idx, init) in info.segments.iter() { - let data = &data[*idx]; - assert_eq!(data.len(), init.data.len()); - let offset = usize::try_from(init.offset - info.min_addr).unwrap(); - if image.len() < offset { - image.resize(offset, 0u8); - image.extend_from_slice(data); - } else { - image.splice( - offset..(offset + data.len()).min(image.len()), - data.iter().copied(), - ); - } - } - assert_eq!(image.len(), extent); - assert_eq!(image.capacity(), extent); - let mut offset = if info.segments.len() > 0 { - info.min_addr - } else { - 0 - }; - - // Chop off trailing zeros from the image as memory is already - // zero-initialized. Note that `i` is the position of a nonzero - // entry here, so to not lose it we truncate to `i + 1`. - if let Some(i) = image.iter().rposition(|i| *i != 0) { - image.truncate(i + 1); - } - - // Also chop off leading zeros, if any. - if let Some(i) = image.iter().position(|i| *i != 0) { - offset += i as u64; - image.drain(..i); - } - let mut len = u64::try_from(image.len()).unwrap(); - - // The goal is to enable mapping this image directly into memory, so - // the offset into linear memory must be a multiple of the page - // size. If that's not already the case then the image is padded at - // the front and back with extra zeros as necessary - if offset % page_size != 0 { - let zero_padding = offset % page_size; - self.data.push(vec![0; zero_padding as usize].into()); - offset -= zero_padding; - len += zero_padding; - } - self.data.push(image.into()); - if len % page_size != 0 { - let zero_padding = page_size - (len % page_size); - self.data.push(vec![0; zero_padding as usize].into()); - len += zero_padding; - } - - // Offset/length should now always be page-aligned. - assert!(offset % page_size == 0); - assert!(len % page_size == 0); - - // Create the `StaticMemoryInitializer` which describes this image, - // only needed if the image is actually present and has a nonzero - // length. The `offset` has been calculates above, originally - // sourced from `info.min_addr`. The `data` field is the extent - // within the final data segment we'll emit to an ELF image, which - // is the concatenation of `self.data`, so here it's the size of - // the section-so-far plus the current segment we're appending. - let len = u32::try_from(len).unwrap(); - let init = if len > 0 { - Some(StaticMemoryInitializer { - offset, - data: module_data_size..module_data_size + len, - }) - } else { - None - }; - let idx = map.push(init); - assert_eq!(idx, memory); - module_data_size += len; - } - self.data_align = Some(page_size); - self.module.memory_initialization = MemoryInitialization::Static { map }; - } - - /// Attempts to convert the module's table initializers to - /// FuncTable form where possible. This enables lazy table - /// initialization later by providing a one-to-one map of initial - /// table values, without having to parse all segments. - pub fn try_func_table_init(&mut self) { - // This should be large enough to support very large Wasm - // modules with huge funcref tables, but small enough to avoid - // OOMs or DoS on truly sparse tables. - const MAX_FUNC_TABLE_SIZE: u32 = 1024 * 1024; - - // First convert any element-initialized tables to images of just that - // single function if the minimum size of the table allows doing so. - for ((_, init), (_, plan)) in self - .module - .table_initialization - .initial_values - .iter_mut() - .zip( - self.module - .table_plans - .iter() - .skip(self.module.num_imported_tables), - ) - { - let table_size = plan.table.minimum; - if table_size > MAX_FUNC_TABLE_SIZE { - continue; - } - if let TableInitialValue::FuncRef(val) = *init { - *init = TableInitialValue::Null { - precomputed: vec![val; table_size as usize], - }; - } - } - - let mut segments = mem::take(&mut self.module.table_initialization.segments) - .into_iter() - .peekable(); - - // The goal of this loop is to interpret a table segment and apply it - // "statically" to a local table. This will iterate over segments and - // apply them one-by-one to each table. - // - // If any segment can't be applied, however, then this loop exits and - // all remaining segments are placed back into the segment list. This is - // because segments are supposed to be initialized one-at-a-time which - // means that intermediate state is visible with respect to traps. If - // anything isn't statically known to not trap it's pessimistically - // assumed to trap meaning all further segment initializers must be - // applied manually at instantiation time. - while let Some(segment) = segments.peek() { - let defined_index = match self.module.defined_table_index(segment.table_index) { - Some(index) => index, - // Skip imported tables: we can't provide a preconstructed - // table for them, because their values depend on the - // imported table overlaid with whatever segments we have. - None => break, - }; - - // If the base of this segment is dynamic, then we can't - // include it in the statically-built array of initial - // contents. - if segment.base.is_some() { - break; - } - - // Get the end of this segment. If out-of-bounds, or too - // large for our dense table representation, then skip the - // segment. - let top = match segment.offset.checked_add(segment.elements.len()) { - Some(top) => top, - None => break, - }; - let table_size = self.module.table_plans[segment.table_index].table.minimum; - if top > table_size || top > MAX_FUNC_TABLE_SIZE { - break; - } - - match self.module.table_plans[segment.table_index] - .table - .wasm_ty - .heap_type - { - WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {} - // If this is not a funcref table, then we can't support a - // pre-computed table of function indices. Technically this - // initializer won't trap so we could continue processing - // segments, but that's left as a future optimization if - // necessary. - WasmHeapType::Extern => break, - } - - // Function indices can be optimized here, but fully general - // expressions are deferred to get evaluated at runtime. - let function_elements = match &segment.elements { - TableSegmentElements::Functions(indices) => indices, - TableSegmentElements::Expressions(_) => break, - }; - - let precomputed = - match &mut self.module.table_initialization.initial_values[defined_index] { - TableInitialValue::Null { precomputed } => precomputed, - - // If this table is still listed as an initial value here - // then that means the initial size of the table doesn't - // support a precomputed function list, so skip this. - // Technically this won't trap so it's possible to process - // further initializers, but that's left as a future - // optimization. - TableInitialValue::FuncRef(_) | TableInitialValue::GlobalGet(_) => break, - }; - - // At this point we're committing to pre-initializing the table - // with the `segment` that's being iterated over. This segment is - // applied to the `precomputed` list for the table by ensuring - // it's large enough to hold the segment and then copying the - // segment into the precomputed list. - if precomputed.len() < top as usize { - precomputed.resize(top as usize, FuncIndex::reserved_value()); - } - let dst = &mut precomputed[(segment.offset as usize)..(top as usize)]; - dst.copy_from_slice(&function_elements); - - // advance the iterator to see the next segment - let _ = segments.next(); - } - self.module.table_initialization.segments = segments.collect(); - } -} - impl Default for MemoryInitialization { fn default() -> Self { Self::Segmented(Vec::new()) diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 65b38b2de030..b35479b10e88 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -1,14 +1,9 @@ //! Definitions of runtime structures and metadata which are serialized into ELF //! with `bincode` as part of a module's compilation process. -use crate::{ - obj, DefinedFuncIndex, FuncIndex, FunctionLoc, MemoryInitialization, Module, ModuleTranslation, - PrimaryMap, Tunables, WasmFunctionInfo, -}; -use anyhow::{bail, Result}; -use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; -use object::SectionKind; +use crate::{DefinedFuncIndex, FilePos, FuncIndex, Module, PrimaryMap, StackMap}; use serde_derive::{Deserialize, Serialize}; +use std::fmt; use std::ops::Range; use std::str; use wasmtime_types::ModuleInternedTypeIndex; @@ -27,6 +22,38 @@ pub struct CompiledFunctionInfo { pub native_to_wasm_trampoline: Option, } +/// Information about a function, such as trap information, address map, +/// and stack maps. +#[derive(Serialize, Deserialize, Default)] +#[allow(missing_docs)] +pub struct WasmFunctionInfo { + pub start_srcloc: FilePos, + pub stack_maps: Box<[StackMapInformation]>, +} + +/// Description of where a function is located in the text section of a +/// compiled image. +#[derive(Copy, Clone, Serialize, Deserialize)] +pub struct FunctionLoc { + /// The byte offset from the start of the text section where this + /// function starts. + pub start: u32, + /// The byte length of this function's function body. + pub length: u32, +} + +/// The offset within a function of a GC safepoint, and its associated stack +/// map. +#[derive(Serialize, Deserialize, Debug)] +pub struct StackMapInformation { + /// The offset of the GC safepoint within the function's native code. It is + /// relative to the beginning of the function. + pub code_offset: u32, + + /// The stack map for identifying live GC refs at the GC safepoint. + pub stack_map: StackMap, +} + /// Secondary in-memory results of module compilation. /// /// This opaque structure can be optionally passed back to @@ -88,290 +115,31 @@ pub struct Metadata { pub dwarf: Vec<(u8, Range)>, } -/// Helper structure to create an ELF file as a compilation artifact. -/// -/// This structure exposes the process which Wasmtime will encode a core wasm -/// module into an ELF file, notably managing data sections and all that good -/// business going into the final file. -pub struct ObjectBuilder<'a> { - /// The `object`-crate-defined ELF file write we're using. - obj: Object<'a>, - - /// General compilation configuration. - tunables: &'a Tunables, - - /// The section identifier for "rodata" which is where wasm data segments - /// will go. - data: SectionId, - - /// The section identifier for function name information, or otherwise where - /// the `name` custom section of wasm is copied into. - /// - /// This is optional and lazily created on demand. - names: Option, - - /// The section identifier for dwarf information copied from the original - /// wasm files. - /// - /// This is optional and lazily created on demand. - dwarf: Option, +/// Value of a configured setting for a [`Compiler`](crate::Compiler) +#[derive(Serialize, Deserialize, Hash, Eq, PartialEq, Debug)] +pub enum FlagValue<'a> { + /// Name of the value that has been configured for this setting. + Enum(&'a str), + /// The numerical value of the configured settings. + Num(u8), + /// Whether the setting is on or off. + Bool(bool), } -impl<'a> ObjectBuilder<'a> { - /// Creates a new builder for the `obj` specified. - pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { - let data = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_WASM_DATA.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - ObjectBuilder { - obj, - tunables, - data, - names: None, - dwarf: None, +impl fmt::Display for FlagValue<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Enum(v) => v.fmt(f), + Self::Num(v) => v.fmt(f), + Self::Bool(v) => v.fmt(f), } } - - /// Completes compilation of the `translation` specified, inserting - /// everything necessary into the `Object` being built. - /// - /// This function will consume the final results of compiling a wasm module - /// and finish the ELF image in-progress as part of `self.obj` by appending - /// any compiler-agnostic sections. - /// - /// The auxiliary `CompiledModuleInfo` structure returned here has also been - /// serialized into the object returned, but if the caller will quickly - /// turn-around and invoke `CompiledModule::from_artifacts` after this then - /// the information can be passed to that method to avoid extra - /// deserialization. This is done to avoid a serialize-then-deserialize for - /// API calls like `Module::new` where the compiled module is immediately - /// going to be used. - /// - /// The various arguments here are: - /// - /// * `translation` - the core wasm translation that's being completed. - /// - /// * `funcs` - compilation metadata about functions within the translation - /// as well as where the functions are located in the text section and any - /// associated trampolines. - /// - /// * `wasm_to_native_trampolines` - list of all trampolines necessary for - /// Wasm callers calling native callees (e.g. `Func::wrap`). One for each - /// function signature in the module. Must be sorted by `SignatureIndex`. - /// - /// Returns the `CompiledModuleInfo` corresponding to this core Wasm module - /// as a result of this append operation. This is then serialized into the - /// final artifact by the caller. - pub fn append( - &mut self, - translation: ModuleTranslation<'_>, - funcs: PrimaryMap, - wasm_to_native_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>, - ) -> Result { - let ModuleTranslation { - mut module, - debuginfo, - has_unparsed_debuginfo, - data, - data_align, - passive_data, - .. - } = translation; - - // Place all data from the wasm module into a section which will the - // source of the data later at runtime. This additionally keeps track of - // the offset of - let mut total_data_len = 0; - let data_offset = self - .obj - .append_section_data(self.data, &[], data_align.unwrap_or(1)); - for (i, data) in data.iter().enumerate() { - // The first data segment has its alignment specified as the alignment - // for the entire section, but everything afterwards is adjacent so it - // has alignment of 1. - let align = if i == 0 { data_align.unwrap_or(1) } else { 1 }; - self.obj.append_section_data(self.data, data, align); - total_data_len += data.len(); - } - for data in passive_data.iter() { - self.obj.append_section_data(self.data, data, 1); - } - - // If any names are present in the module then the `ELF_NAME_DATA` section - // is create and appended. - let mut func_names = Vec::new(); - if debuginfo.name_section.func_names.len() > 0 { - let name_id = *self.names.get_or_insert_with(|| { - self.obj.add_section( - self.obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_NAME_DATA.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ) - }); - let mut sorted_names = debuginfo.name_section.func_names.iter().collect::>(); - sorted_names.sort_by_key(|(idx, _name)| *idx); - for (idx, name) in sorted_names { - let offset = self.obj.append_section_data(name_id, name.as_bytes(), 1); - let offset = match u32::try_from(offset) { - Ok(offset) => offset, - Err(_) => bail!("name section too large (> 4gb)"), - }; - let len = u32::try_from(name.len()).unwrap(); - func_names.push(FunctionName { - idx: *idx, - offset, - len, - }); - } - } - - // Data offsets in `MemoryInitialization` are offsets within the - // `translation.data` list concatenated which is now present in the data - // segment that's appended to the object. Increase the offsets by - // `self.data_size` to account for any previously added module. - let data_offset = u32::try_from(data_offset).unwrap(); - match &mut module.memory_initialization { - MemoryInitialization::Segmented(list) => { - for segment in list { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - MemoryInitialization::Static { map } => { - for (_, segment) in map { - if let Some(segment) = segment { - segment.data.start = segment.data.start.checked_add(data_offset).unwrap(); - segment.data.end = segment.data.end.checked_add(data_offset).unwrap(); - } - } - } - } - - // Data offsets for passive data are relative to the start of - // `translation.passive_data` which was appended to the data segment - // of this object, after active data in `translation.data`. Update the - // offsets to account prior modules added in addition to active data. - let data_offset = data_offset + u32::try_from(total_data_len).unwrap(); - for (_, range) in module.passive_data_map.iter_mut() { - range.start = range.start.checked_add(data_offset).unwrap(); - range.end = range.end.checked_add(data_offset).unwrap(); - } - - // Insert the wasm raw wasm-based debuginfo into the output, if - // requested. Note that this is distinct from the native debuginfo - // possibly generated by the native compiler, hence these sections - // getting wasm-specific names. - let mut dwarf = Vec::new(); - if self.tunables.parse_wasm_debuginfo { - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_abbrev); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_addr); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_aranges); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_info); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_line_str); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str); - self.push_debug(&mut dwarf, &debuginfo.dwarf.debug_str_offsets); - self.push_debug(&mut dwarf, &debuginfo.debug_ranges); - self.push_debug(&mut dwarf, &debuginfo.debug_rnglists); - } - // Sort this for binary-search-lookup later in `symbolize_context`. - dwarf.sort_by_key(|(id, _)| *id); - - Ok(CompiledModuleInfo { - module, - funcs, - wasm_to_native_trampolines, - func_names, - meta: Metadata { - native_debug_info_present: self.tunables.generate_native_debuginfo, - has_unparsed_debuginfo, - code_section_offset: debuginfo.wasm_file.code_section_offset, - has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, - dwarf, - }, - }) - } - - fn push_debug<'b, T>(&mut self, dwarf: &mut Vec<(u8, Range)>, section: &T) - where - T: gimli::Section>, - { - let data = section.reader().slice(); - if data.is_empty() { - return; - } - let section_id = *self.dwarf.get_or_insert_with(|| { - self.obj.add_section( - self.obj.segment_name(StandardSegment::Debug).to_vec(), - obj::ELF_WASMTIME_DWARF.as_bytes().to_vec(), - SectionKind::Debug, - ) - }); - let offset = self.obj.append_section_data(section_id, data, 1); - dwarf.push((T::id() as u8, offset..offset + data.len() as u64)); - } - - /// Creates the `ELF_WASMTIME_INFO` section from the given serializable data - /// structure. - pub fn serialize_info(&mut self, info: &T) - where - T: serde::Serialize, - { - let section = self.obj.add_section( - self.obj.segment_name(StandardSegment::Data).to_vec(), - obj::ELF_WASMTIME_INFO.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - let data = bincode::serialize(info).unwrap(); - self.obj.set_section_data(section, data, 1); - } - - /// Serializes `self` into a buffer. This can be used for execution as well - /// as serialization. - pub fn finish(self, t: &mut T) -> Result<()> { - self.obj.emit(t).map_err(|e| e.into()) - } } -/// A type which can be the result of serializing an object. -pub trait FinishedObject: Sized { - /// Emit the object as `Self`. - fn finish_object(obj: ObjectBuilder<'_>) -> Result; -} - -impl FinishedObject for Vec { - fn finish_object(obj: ObjectBuilder<'_>) -> Result { - let mut result = ObjectVec::default(); - obj.finish(&mut result)?; - return Ok(result.0); - - #[derive(Default)] - struct ObjectVec(Vec); - - impl WritableBuffer for ObjectVec { - fn len(&self) -> usize { - self.0.len() - } - - fn reserve(&mut self, additional: usize) -> Result<(), ()> { - assert_eq!(self.0.len(), 0, "cannot reserve twice"); - self.0 = Vec::with_capacity(additional); - Ok(()) - } - - fn resize(&mut self, new_len: usize) { - if new_len <= self.0.len() { - self.0.truncate(new_len) - } else { - self.0.extend(vec![0; new_len - self.0.len()]) - } - } - - fn write_bytes(&mut self, val: &[u8]) { - self.0.extend(val); - } - } - } +/// Types of objects that can be created by `Compiler::object` +pub enum ObjectKind { + /// A core wasm compilation artifact + Module, + /// A component compilation artifact + Component, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index c96c74b5c4da..1e3e7de9c754 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -4,13 +4,15 @@ use crate::module::{ }; use crate::{ DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, FuncIndex, GlobalIndex, - GlobalInit, MemoryIndex, ModuleTypesBuilder, PrimaryMap, TableIndex, TableInitialValue, - Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, WasmHeapType, WasmResult, WasmValType, - WasmparserTypeConverter, + GlobalInit, InitMemory, MemoryIndex, ModuleTypesBuilder, PrimaryMap, StaticMemoryInitializer, + TableIndex, TableInitialValue, Tunables, TypeConvert, TypeIndex, Unsigned, WasmError, + WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter, }; use anyhow::{bail, Result}; +use cranelift_entity::packed_option::ReservedValue; use std::borrow::Cow; use std::collections::HashMap; +use std::mem; use std::path::PathBuf; use std::sync::Arc; use wasmparser::types::{CoreTypeId, Types}; @@ -901,3 +903,344 @@ impl TypeConvert for ModuleEnvironment<'_, '_> { .lookup_heap_type(index) } } + +impl ModuleTranslation<'_> { + /// Attempts to convert segmented memory initialization into static + /// initialization for the module that this translation represents. + /// + /// If this module's memory initialization is not compatible with paged + /// initialization then this won't change anything. Otherwise if it is + /// compatible then the `memory_initialization` field will be updated. + /// + /// Takes a `page_size` argument in order to ensure that all + /// initialization is page-aligned for mmap-ability, and + /// `max_image_size_always_allowed` to control how we decide + /// whether to use static init. + /// + /// We will try to avoid generating very sparse images, which are + /// possible if e.g. a module has an initializer at offset 0 and a + /// very high offset (say, 1 GiB). To avoid this, we use a dual + /// condition: we always allow images less than + /// `max_image_size_always_allowed`, and the embedder of Wasmtime + /// can set this if desired to ensure that static init should + /// always be done if the size of the module or its heaps is + /// otherwise bounded by the system. We also allow images with + /// static init data bigger than that, but only if it is "dense", + /// defined as having at least half (50%) of its pages with some + /// data. + /// + /// We could do something slightly better by building a dense part + /// and keeping a sparse list of outlier/leftover segments (see + /// issue #3820). This would also allow mostly-static init of + /// modules that have some dynamically-placed data segments. But, + /// for now, this is sufficient to allow a system that "knows what + /// it's doing" to always get static init. + pub fn try_static_init(&mut self, page_size: u64, max_image_size_always_allowed: u64) { + // This method only attempts to transform a `Segmented` memory init + // into a `Static` one, no other state. + if !self.module.memory_initialization.is_segmented() { + return; + } + + // First a dry run of memory initialization is performed. This + // collects information about the extent of memory initialized for each + // memory as well as the size of all data segments being copied in. + struct Memory { + data_size: u64, + min_addr: u64, + max_addr: u64, + // The `usize` here is a pointer into `self.data` which is the list + // of data segments corresponding to what was found in the original + // wasm module. + segments: Vec<(usize, StaticMemoryInitializer)>, + } + let mut info = PrimaryMap::with_capacity(self.module.memory_plans.len()); + for _ in 0..self.module.memory_plans.len() { + info.push(Memory { + data_size: 0, + min_addr: u64::MAX, + max_addr: 0, + segments: Vec::new(), + }); + } + let mut idx = 0; + let ok = self.module.memory_initialization.init_memory( + &mut (), + InitMemory::CompileTime(&self.module), + |(), memory, init| { + // Currently `Static` only applies to locally-defined memories, + // so if a data segment references an imported memory then + // transitioning to a `Static` memory initializer is not + // possible. + if self.module.defined_memory_index(memory).is_none() { + return false; + }; + let info = &mut info[memory]; + let data_len = u64::from(init.data.end - init.data.start); + if data_len > 0 { + info.data_size += data_len; + info.min_addr = info.min_addr.min(init.offset); + info.max_addr = info.max_addr.max(init.offset + data_len); + info.segments.push((idx, init.clone())); + } + idx += 1; + true + }, + ); + if !ok { + return; + } + + // Validate that the memory information collected is indeed valid for + // static memory initialization. + for info in info.values().filter(|i| i.data_size > 0) { + let image_size = info.max_addr - info.min_addr; + + // If the range of memory being initialized is less than twice the + // total size of the data itself then it's assumed that static + // initialization is ok. This means we'll at most double memory + // consumption during the memory image creation process, which is + // currently assumed to "probably be ok" but this will likely need + // tweaks over time. + if image_size < info.data_size.saturating_mul(2) { + continue; + } + + // If the memory initialization image is larger than the size of all + // data, then we still allow memory initialization if the image will + // be of a relatively modest size, such as 1MB here. + if image_size < max_image_size_always_allowed { + continue; + } + + // At this point memory initialization is concluded to be too + // expensive to do at compile time so it's entirely deferred to + // happen at runtime. + return; + } + + // Here's where we've now committed to changing to static memory. The + // memory initialization image is built here from the page data and then + // it's converted to a single initializer. + let data = mem::replace(&mut self.data, Vec::new()); + let mut map = PrimaryMap::with_capacity(info.len()); + let mut module_data_size = 0u32; + for (memory, info) in info.iter() { + // Create the in-memory `image` which is the initialized contents of + // this linear memory. + let extent = if info.segments.len() > 0 { + (info.max_addr - info.min_addr) as usize + } else { + 0 + }; + let mut image = Vec::with_capacity(extent); + for (idx, init) in info.segments.iter() { + let data = &data[*idx]; + assert_eq!(data.len(), init.data.len()); + let offset = usize::try_from(init.offset - info.min_addr).unwrap(); + if image.len() < offset { + image.resize(offset, 0u8); + image.extend_from_slice(data); + } else { + image.splice( + offset..(offset + data.len()).min(image.len()), + data.iter().copied(), + ); + } + } + assert_eq!(image.len(), extent); + assert_eq!(image.capacity(), extent); + let mut offset = if info.segments.len() > 0 { + info.min_addr + } else { + 0 + }; + + // Chop off trailing zeros from the image as memory is already + // zero-initialized. Note that `i` is the position of a nonzero + // entry here, so to not lose it we truncate to `i + 1`. + if let Some(i) = image.iter().rposition(|i| *i != 0) { + image.truncate(i + 1); + } + + // Also chop off leading zeros, if any. + if let Some(i) = image.iter().position(|i| *i != 0) { + offset += i as u64; + image.drain(..i); + } + let mut len = u64::try_from(image.len()).unwrap(); + + // The goal is to enable mapping this image directly into memory, so + // the offset into linear memory must be a multiple of the page + // size. If that's not already the case then the image is padded at + // the front and back with extra zeros as necessary + if offset % page_size != 0 { + let zero_padding = offset % page_size; + self.data.push(vec![0; zero_padding as usize].into()); + offset -= zero_padding; + len += zero_padding; + } + self.data.push(image.into()); + if len % page_size != 0 { + let zero_padding = page_size - (len % page_size); + self.data.push(vec![0; zero_padding as usize].into()); + len += zero_padding; + } + + // Offset/length should now always be page-aligned. + assert!(offset % page_size == 0); + assert!(len % page_size == 0); + + // Create the `StaticMemoryInitializer` which describes this image, + // only needed if the image is actually present and has a nonzero + // length. The `offset` has been calculates above, originally + // sourced from `info.min_addr`. The `data` field is the extent + // within the final data segment we'll emit to an ELF image, which + // is the concatenation of `self.data`, so here it's the size of + // the section-so-far plus the current segment we're appending. + let len = u32::try_from(len).unwrap(); + let init = if len > 0 { + Some(StaticMemoryInitializer { + offset, + data: module_data_size..module_data_size + len, + }) + } else { + None + }; + let idx = map.push(init); + assert_eq!(idx, memory); + module_data_size += len; + } + self.data_align = Some(page_size); + self.module.memory_initialization = MemoryInitialization::Static { map }; + } + + /// Attempts to convert the module's table initializers to + /// FuncTable form where possible. This enables lazy table + /// initialization later by providing a one-to-one map of initial + /// table values, without having to parse all segments. + pub fn try_func_table_init(&mut self) { + // This should be large enough to support very large Wasm + // modules with huge funcref tables, but small enough to avoid + // OOMs or DoS on truly sparse tables. + const MAX_FUNC_TABLE_SIZE: u32 = 1024 * 1024; + + // First convert any element-initialized tables to images of just that + // single function if the minimum size of the table allows doing so. + for ((_, init), (_, plan)) in self + .module + .table_initialization + .initial_values + .iter_mut() + .zip( + self.module + .table_plans + .iter() + .skip(self.module.num_imported_tables), + ) + { + let table_size = plan.table.minimum; + if table_size > MAX_FUNC_TABLE_SIZE { + continue; + } + if let TableInitialValue::FuncRef(val) = *init { + *init = TableInitialValue::Null { + precomputed: vec![val; table_size as usize], + }; + } + } + + let mut segments = mem::take(&mut self.module.table_initialization.segments) + .into_iter() + .peekable(); + + // The goal of this loop is to interpret a table segment and apply it + // "statically" to a local table. This will iterate over segments and + // apply them one-by-one to each table. + // + // If any segment can't be applied, however, then this loop exits and + // all remaining segments are placed back into the segment list. This is + // because segments are supposed to be initialized one-at-a-time which + // means that intermediate state is visible with respect to traps. If + // anything isn't statically known to not trap it's pessimistically + // assumed to trap meaning all further segment initializers must be + // applied manually at instantiation time. + while let Some(segment) = segments.peek() { + let defined_index = match self.module.defined_table_index(segment.table_index) { + Some(index) => index, + // Skip imported tables: we can't provide a preconstructed + // table for them, because their values depend on the + // imported table overlaid with whatever segments we have. + None => break, + }; + + // If the base of this segment is dynamic, then we can't + // include it in the statically-built array of initial + // contents. + if segment.base.is_some() { + break; + } + + // Get the end of this segment. If out-of-bounds, or too + // large for our dense table representation, then skip the + // segment. + let top = match segment.offset.checked_add(segment.elements.len()) { + Some(top) => top, + None => break, + }; + let table_size = self.module.table_plans[segment.table_index].table.minimum; + if top > table_size || top > MAX_FUNC_TABLE_SIZE { + break; + } + + match self.module.table_plans[segment.table_index] + .table + .wasm_ty + .heap_type + { + WasmHeapType::Func | WasmHeapType::Concrete(_) | WasmHeapType::NoFunc => {} + // If this is not a funcref table, then we can't support a + // pre-computed table of function indices. Technically this + // initializer won't trap so we could continue processing + // segments, but that's left as a future optimization if + // necessary. + WasmHeapType::Extern => break, + } + + // Function indices can be optimized here, but fully general + // expressions are deferred to get evaluated at runtime. + let function_elements = match &segment.elements { + TableSegmentElements::Functions(indices) => indices, + TableSegmentElements::Expressions(_) => break, + }; + + let precomputed = + match &mut self.module.table_initialization.initial_values[defined_index] { + TableInitialValue::Null { precomputed } => precomputed, + + // If this table is still listed as an initial value here + // then that means the initial size of the table doesn't + // support a precomputed function list, so skip this. + // Technically this won't trap so it's possible to process + // further initializers, but that's left as a future + // optimization. + TableInitialValue::FuncRef(_) | TableInitialValue::GlobalGet(_) => break, + }; + + // At this point we're committing to pre-initializing the table + // with the `segment` that's being iterated over. This segment is + // applied to the `precomputed` list for the table by ensuring + // it's large enough to hold the segment and then copying the + // segment into the precomputed list. + if precomputed.len() < top as usize { + precomputed.resize(top as usize, FuncIndex::reserved_value()); + } + let dst = &mut precomputed[(segment.offset as usize)..(top as usize)]; + dst.copy_from_slice(&function_elements); + + // advance the iterator to see the next segment + let _ = segments.next(); + } + self.module.table_initialization.segments = segments.collect(); + } +} diff --git a/crates/environ/src/trap_encoding.rs b/crates/environ/src/trap_encoding.rs index d14382a8be76..d59e1b000b97 100644 --- a/crates/environ/src/trap_encoding.rs +++ b/crates/environ/src/trap_encoding.rs @@ -1,21 +1,5 @@ -use crate::obj::ELF_WASMTIME_TRAPS; -use object::write::{Object, StandardSegment}; -use object::{Bytes, LittleEndian, SectionKind, U32Bytes}; +use object::{Bytes, LittleEndian, U32Bytes}; use std::fmt; -use std::ops::Range; - -/// A helper structure to build the custom-encoded section of a wasmtime -/// compilation image which encodes trap information. -/// -/// This structure is incrementally fed the results of compiling individual -/// functions and handles all the encoding internally, allowing usage of -/// `lookup_trap_code` below with the resulting section. -#[derive(Default)] -pub struct TrapEncodingBuilder { - offsets: Vec>, - traps: Vec, - last_offset: u32, -} /// Information about trap. #[derive(Debug, PartialEq, Eq, Clone)] @@ -162,57 +146,6 @@ impl fmt::Display for Trap { impl std::error::Error for Trap {} -impl TrapEncodingBuilder { - /// Appends trap information about a function into this section. - /// - /// This function is called to describe traps for the `func` range - /// specified. The `func` offsets are specified relative to the text section - /// itself, and the `traps` offsets are specified relative to the start of - /// `func`. - /// - /// This is required to be called in-order for increasing ranges of `func` - /// to ensure the final array is properly sorted. Additionally `traps` must - /// be sorted. - pub fn push(&mut self, func: Range, traps: &[TrapInformation]) { - // NB: for now this only supports <=4GB text sections in object files. - // Alternative schemes will need to be created for >32-bit offsets to - // avoid making this section overly large. - let func_start = u32::try_from(func.start).unwrap(); - let func_end = u32::try_from(func.end).unwrap(); - - // Sanity-check to ensure that functions are pushed in-order, otherwise - // the `offsets` array won't be sorted which is our goal. - assert!(func_start >= self.last_offset); - - self.offsets.reserve(traps.len()); - self.traps.reserve(traps.len()); - for info in traps { - let pos = func_start + info.code_offset; - assert!(pos >= self.last_offset); - self.offsets.push(U32Bytes::new(LittleEndian, pos)); - self.traps.push(info.trap_code as u8); - self.last_offset = pos; - } - - self.last_offset = func_end; - } - - /// Encodes this section into the object provided. - pub fn append_to(self, obj: &mut Object) { - let section = obj.add_section( - obj.segment_name(StandardSegment::Data).to_vec(), - ELF_WASMTIME_TRAPS.as_bytes().to_vec(), - SectionKind::ReadOnlyData, - ); - - // NB: this matches the encoding expected by `lookup` below. - let amt = u32::try_from(self.traps.len()).unwrap(); - obj.append_section_data(section, &amt.to_le_bytes(), 1); - obj.append_section_data(section, object::bytes_of_slice(&self.offsets), 1); - obj.append_section_data(section, &self.traps, 1); - } -} - /// Decodes the provided trap information section and attempts to find the trap /// code corresponding to the `offset` specified. /// diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 80f54d40e1b7..c2994850b427 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -41,6 +41,11 @@ use wasmtime_environ::{ mod code_builder; pub use self::code_builder::{CodeBuilder, HashedEngineCompileEnv}; +#[cfg(feature = "runtime")] +mod runtime; +#[cfg(feature = "runtime")] +pub use self::runtime::finish_object; + /// Converts an input binary-encoded WebAssembly module to compilation /// artifacts and type information. /// diff --git a/crates/wasmtime/src/compile/code_builder.rs b/crates/wasmtime/src/compile/code_builder.rs index 9393278edf1f..35d3990ab769 100644 --- a/crates/wasmtime/src/compile/code_builder.rs +++ b/crates/wasmtime/src/compile/code_builder.rs @@ -1,15 +1,7 @@ -#[cfg(all(feature = "runtime", feature = "component-model"))] -use crate::component::Component; use crate::Engine; -#[cfg(feature = "runtime")] -use crate::{instantiate::MmapVecWrapper, CodeMemory, Module}; use anyhow::{anyhow, bail, Context, Result}; use std::borrow::Cow; use std::path::Path; -use std::sync::Arc; -use wasmtime_environ::ObjectKind; -#[cfg(feature = "runtime")] -use wasmtime_runtime::MmapVec; /// Builder-style structure used to create a [`Module`](crate::Module) or /// pre-compile a module to a serialized list of bytes. @@ -45,7 +37,7 @@ use wasmtime_runtime::MmapVec; /// [`wasm`]: CodeBuilder::wasm /// [`wasm_file`]: CodeBuilder::wasm_file pub struct CodeBuilder<'a> { - engine: &'a Engine, + pub(super) engine: &'a Engine, wasm: Option>, wasm_path: Option>, wat: bool, @@ -130,7 +122,7 @@ impl<'a> CodeBuilder<'a> { Ok(self) } - fn wasm_binary(&self) -> Result> { + pub(super) fn wasm_binary(&self) -> Result> { let wasm = self .wasm .as_ref() @@ -147,65 +139,6 @@ impl<'a> CodeBuilder<'a> { Ok((&wasm[..]).into()) } - #[cfg(feature = "runtime")] - fn compile_cached( - &self, - build_artifacts: fn(&Engine, &[u8]) -> Result<(MmapVecWrapper, Option)>, - ) -> Result<(Arc, Option)> { - let wasm = self.wasm_binary()?; - - self.engine - .check_compatible_with_native_host() - .context("compilation settings are not compatible with the native host")?; - - #[cfg(feature = "cache")] - { - let state = ( - HashedEngineCompileEnv(self.engine), - &wasm, - // Don't hash this as it's just its own "pure" function pointer. - NotHashed(build_artifacts), - ); - let (code, info_and_types) = - wasmtime_cache::ModuleCacheEntry::new("wasmtime", self.engine.cache_config()) - .get_data_raw( - &state, - // Cache miss, compute the actual artifacts - |(engine, wasm, build_artifacts)| -> Result<_> { - let (mmap, info) = (build_artifacts.0)(engine.0, wasm)?; - let code = publish_mmap(mmap.0)?; - Ok((code, info)) - }, - // Implementation of how to serialize artifacts - |(_engine, _wasm, _), (code, _info_and_types)| Some(code.mmap().to_vec()), - // Cache hit, deserialize the provided artifacts - |(engine, wasm, _), serialized_bytes| { - let kind = if wasmparser::Parser::is_component(&wasm) { - ObjectKind::Component - } else { - ObjectKind::Module - }; - let code = engine.0.load_code_bytes(&serialized_bytes, kind).ok()?; - Some((code, None)) - }, - )?; - return Ok((code, info_and_types)); - } - - #[cfg(not(feature = "cache"))] - { - let (mmap, info_and_types) = build_artifacts(self.engine, &wasm)?; - let code = publish_mmap(mmap.0)?; - return Ok((code, info_and_types)); - } - - struct NotHashed(T); - - impl std::hash::Hash for NotHashed { - fn hash(&self, _hasher: &mut H) {} - } - } - /// Finishes this compilation and produces a serialized list of bytes. /// /// This method requires that either [`CodeBuilder::wasm`] or @@ -228,20 +161,9 @@ impl<'a> CodeBuilder<'a> { Ok(v) } - /// Same as [`CodeBuilder::compile_module_serialized`] except that a - /// [`Module`](crate::Module) is produced instead. - /// - /// Note that this method will cache compilations if the `cache` feature is - /// enabled and turned on in [`Config`](crate::Config). - #[cfg(feature = "runtime")] - #[cfg_attr(docsrs, doc(cfg(feature = "runtime")))] - pub fn compile_module(&self) -> Result { - let (code, info_and_types) = self.compile_cached(super::build_artifacts)?; - Module::from_parts(self.engine, code, info_and_types) - } - /// Same as [`CodeBuilder::compile_module_serialized`] except that it - /// compiles a serialized [`Component`] instead of a module. + /// compiles a serialized [`Component`](crate::component::Component) + /// instead of a module. #[cfg(feature = "component-model")] #[cfg_attr(docsrs, doc(cfg(feature = "component-model")))] pub fn compile_component_serialized(&self) -> Result> { @@ -249,18 +171,6 @@ impl<'a> CodeBuilder<'a> { let (v, _) = super::build_component_artifacts(self.engine, &bytes)?; Ok(v) } - - /// Same as [`CodeBuilder::compile_module`] except that it compiles a - /// [`Component`] instead of a module. - #[cfg(all(feature = "runtime", feature = "component-model"))] - #[cfg_attr( - docsrs, - doc(cfg(all(feature = "runtime", feature = "component-model"))) - )] - pub fn compile_component(&self) -> Result { - let (code, artifacts) = self.compile_cached(super::build_component_artifacts)?; - Component::from_parts(self.engine, code, artifacts) - } } /// This is a helper struct used when caching to hash the state of an `Engine` @@ -289,10 +199,3 @@ impl std::hash::Hash for HashedEngineCompileEnv<'_> { config.module_version.hash(hasher); } } - -#[cfg(feature = "runtime")] -fn publish_mmap(mmap: MmapVec) -> Result> { - let mut code = CodeMemory::new(mmap)?; - code.publish()?; - Ok(Arc::new(code)) -} diff --git a/crates/wasmtime/src/compile/runtime.rs b/crates/wasmtime/src/compile/runtime.rs new file mode 100644 index 000000000000..fde8dfcdfeaf --- /dev/null +++ b/crates/wasmtime/src/compile/runtime.rs @@ -0,0 +1,175 @@ +use crate::compile::HashedEngineCompileEnv; +#[cfg(feature = "component-model")] +use crate::component::Component; +use crate::{CodeBuilder, CodeMemory, Engine, Module}; +use anyhow::{Context, Error, Result}; +use object::write::WritableBuffer; +use std::sync::Arc; +use wasmtime_environ::{FinishedObject, ObjectBuilder, ObjectKind}; +use wasmtime_runtime::MmapVec; + +impl<'a> CodeBuilder<'a> { + fn compile_cached( + &self, + build_artifacts: fn(&Engine, &[u8]) -> Result<(MmapVecWrapper, Option)>, + ) -> Result<(Arc, Option)> { + let wasm = self.wasm_binary()?; + + self.engine + .check_compatible_with_native_host() + .context("compilation settings are not compatible with the native host")?; + + #[cfg(feature = "cache")] + { + let state = ( + HashedEngineCompileEnv(self.engine), + &wasm, + // Don't hash this as it's just its own "pure" function pointer. + NotHashed(build_artifacts), + ); + let (code, info_and_types) = + wasmtime_cache::ModuleCacheEntry::new("wasmtime", self.engine.cache_config()) + .get_data_raw( + &state, + // Cache miss, compute the actual artifacts + |(engine, wasm, build_artifacts)| -> Result<_> { + let (mmap, info) = (build_artifacts.0)(engine.0, wasm)?; + let code = publish_mmap(mmap.0)?; + Ok((code, info)) + }, + // Implementation of how to serialize artifacts + |(_engine, _wasm, _), (code, _info_and_types)| Some(code.mmap().to_vec()), + // Cache hit, deserialize the provided artifacts + |(engine, wasm, _), serialized_bytes| { + let kind = if wasmparser::Parser::is_component(&wasm) { + ObjectKind::Component + } else { + ObjectKind::Module + }; + let code = engine.0.load_code_bytes(&serialized_bytes, kind).ok()?; + Some((code, None)) + }, + )?; + return Ok((code, info_and_types)); + } + + #[cfg(not(feature = "cache"))] + { + let (mmap, info_and_types) = build_artifacts(self.engine, &wasm)?; + let code = publish_mmap(mmap.0)?; + return Ok((code, info_and_types)); + } + + struct NotHashed(T); + + impl std::hash::Hash for NotHashed { + fn hash(&self, _hasher: &mut H) {} + } + } + + /// Same as [`CodeBuilder::compile_module_serialized`] except that a + /// [`Module`](crate::Module) is produced instead. + /// + /// Note that this method will cache compilations if the `cache` feature is + /// enabled and turned on in [`Config`](crate::Config). + #[cfg_attr(docsrs, doc(cfg(feature = "runtime")))] + pub fn compile_module(&self) -> Result { + let (code, info_and_types) = self.compile_cached(super::build_artifacts)?; + Module::from_parts(self.engine, code, info_and_types) + } + + /// Same as [`CodeBuilder::compile_module`] except that it compiles a + /// [`Component`] instead of a module. + #[cfg(feature = "component-model")] + #[cfg_attr( + docsrs, + doc(cfg(all(feature = "runtime", feature = "component-model"))) + )] + pub fn compile_component(&self) -> Result { + let (code, artifacts) = self.compile_cached(super::build_component_artifacts)?; + Component::from_parts(self.engine, code, artifacts) + } +} + +fn publish_mmap(mmap: MmapVec) -> Result> { + let mut code = CodeMemory::new(mmap)?; + code.publish()?; + Ok(Arc::new(code)) +} + +/// Write an object out to an [`MmapVec`] so that it can be marked executable +/// before running. +/// +/// The returned `MmapVec` will contain the serialized version of `obj` +/// and is sized appropriately to the exact size of the object serialized. +pub fn finish_object(obj: ObjectBuilder<'_>) -> Result { + Ok(::finish_object(obj)?.0) +} + +pub(crate) struct MmapVecWrapper(pub MmapVec); + +impl FinishedObject for MmapVecWrapper { + fn finish_object(obj: ObjectBuilder<'_>) -> Result { + let mut result = ObjectMmap::default(); + return match obj.finish(&mut result) { + Ok(()) => { + assert!(result.mmap.is_some(), "no reserve"); + let mmap = result.mmap.expect("reserve not called"); + assert_eq!(mmap.len(), result.len); + Ok(MmapVecWrapper(mmap)) + } + Err(e) => match result.err.take() { + Some(original) => Err(original.context(e)), + None => Err(e.into()), + }, + }; + + /// Helper struct to implement the `WritableBuffer` trait from the `object` + /// crate. + /// + /// This enables writing an object directly into an mmap'd memory so it's + /// immediately usable for execution after compilation. This implementation + /// relies on a call to `reserve` happening once up front with all the needed + /// data, and the mmap internally does not attempt to grow afterwards. + #[derive(Default)] + struct ObjectMmap { + mmap: Option, + len: usize, + err: Option, + } + + impl WritableBuffer for ObjectMmap { + fn len(&self) -> usize { + self.len + } + + fn reserve(&mut self, additional: usize) -> Result<(), ()> { + assert!(self.mmap.is_none(), "cannot reserve twice"); + self.mmap = match MmapVec::with_capacity(additional) { + Ok(mmap) => Some(mmap), + Err(e) => { + self.err = Some(e); + return Err(()); + } + }; + Ok(()) + } + + fn resize(&mut self, new_len: usize) { + // Resizing always appends 0 bytes and since new mmaps start out as 0 + // bytes we don't actually need to do anything as part of this other + // than update our own length. + if new_len <= self.len { + return; + } + self.len = new_len; + } + + fn write_bytes(&mut self, val: &[u8]) { + let mmap = self.mmap.as_mut().expect("write before reserve"); + mmap[self.len..][..val.len()].copy_from_slice(val); + self.len += val.len(); + } + } + } +} diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 271c9bc25ff5..f4f2e091d525 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -26,6 +26,7 @@ use crate::stack::{StackCreator, StackCreatorProxy}; #[cfg(feature = "async")] use wasmtime_fiber::RuntimeFiberStackCreator; +#[cfg(all(feature = "incremental-cache", feature = "cranelift"))] pub use wasmtime_environ::CacheStore; #[cfg(feature = "pooling-allocator")] use wasmtime_runtime::mpk; @@ -158,7 +159,7 @@ struct CompilerConfig { target: Option, settings: HashMap, flags: HashSet, - #[cfg(any(feature = "cranelift", feature = "winch"))] + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] cache_store: Option>, clif_dir: Option, wmemcheck: bool, @@ -172,6 +173,7 @@ impl CompilerConfig { target: None, settings: HashMap::new(), flags: HashSet::new(), + #[cfg(all(feature = "incremental-cache", feature = "cranelift"))] cache_store: None, clif_dir: None, wmemcheck: false, @@ -1894,6 +1896,7 @@ impl Config { compiler.enable(flag)?; } + #[cfg(feature = "incremental-cache")] if let Some(cache_store) = &self.compiler_config.cache_store { compiler.enable_incremental_compilation(cache_store.clone())?; } diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 0e6f16cf2ab8..ef79116a1e8e 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -2,6 +2,7 @@ use crate::runtime::type_registry::TypeRegistry; use crate::Config; use anyhow::{Context, Result}; +#[cfg(any(feature = "cranelift", feature = "winch"))] use object::write::{Object, StandardSegment}; use object::SectionKind; use once_cell::sync::OnceCell; diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs index 3f1473fc1fa8..07855c608afd 100644 --- a/crates/wasmtime/src/engine/serialization.rs +++ b/crates/wasmtime/src/engine/serialization.rs @@ -23,6 +23,7 @@ use crate::{Engine, ModuleVersionStrategy, Precompiled}; use anyhow::{anyhow, bail, ensure, Context, Result}; +#[cfg(any(feature = "cranelift", feature = "winch"))] use object::write::{Object, StandardSegment}; use object::{File, FileFlags, Object as _, ObjectSection, SectionKind}; use serde_derive::{Deserialize, Serialize}; @@ -112,6 +113,7 @@ pub fn check_compatible(engine: &Engine, mmap: &[u8], expected: ObjectKind) -> R bincode::deserialize::>(data)?.check_compatible(engine) } +#[cfg(any(feature = "cranelift", feature = "winch"))] pub fn append_compiler_info(engine: &Engine, obj: &mut Object<'_>, metadata: &Metadata<'_>) { let section = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs index 165663a37c06..c09b96d6b547 100644 --- a/crates/wasmtime/src/runtime/instantiate.rs +++ b/crates/wasmtime/src/runtime/instantiate.rs @@ -4,14 +4,13 @@ //! steps. use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent}; -use anyhow::{Error, Result}; -use object::write::WritableBuffer; +use anyhow::Result; use std::str; use std::sync::Arc; use wasmtime_environ::{ - CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FinishedObject, FuncIndex, - FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex, ObjectBuilder, - PrimaryMap, StackMapInformation, WasmFunctionInfo, + CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FuncIndex, FunctionLoc, + FunctionName, Metadata, Module, ModuleInternedTypeIndex, PrimaryMap, StackMapInformation, + WasmFunctionInfo, }; use wasmtime_runtime::{CompiledModuleId, CompiledModuleIdAllocator, MmapVec}; @@ -344,80 +343,3 @@ impl<'a> SymbolizeContext<'a> { self.code_section_offset } } - -/// Write an object out to an [`MmapVec`] so that it can be marked executable -/// before running. -/// -/// The returned `MmapVec` will contain the serialized version of `obj` -/// and is sized appropriately to the exact size of the object serialized. -pub fn finish_object(obj: ObjectBuilder<'_>) -> Result { - Ok(::finish_object(obj)?.0) -} - -pub(crate) struct MmapVecWrapper(pub MmapVec); - -impl FinishedObject for MmapVecWrapper { - fn finish_object(obj: ObjectBuilder<'_>) -> Result { - let mut result = ObjectMmap::default(); - return match obj.finish(&mut result) { - Ok(()) => { - assert!(result.mmap.is_some(), "no reserve"); - let mmap = result.mmap.expect("reserve not called"); - assert_eq!(mmap.len(), result.len); - Ok(MmapVecWrapper(mmap)) - } - Err(e) => match result.err.take() { - Some(original) => Err(original.context(e)), - None => Err(e.into()), - }, - }; - - /// Helper struct to implement the `WritableBuffer` trait from the `object` - /// crate. - /// - /// This enables writing an object directly into an mmap'd memory so it's - /// immediately usable for execution after compilation. This implementation - /// relies on a call to `reserve` happening once up front with all the needed - /// data, and the mmap internally does not attempt to grow afterwards. - #[derive(Default)] - struct ObjectMmap { - mmap: Option, - len: usize, - err: Option, - } - - impl WritableBuffer for ObjectMmap { - fn len(&self) -> usize { - self.len - } - - fn reserve(&mut self, additional: usize) -> Result<(), ()> { - assert!(self.mmap.is_none(), "cannot reserve twice"); - self.mmap = match MmapVec::with_capacity(additional) { - Ok(mmap) => Some(mmap), - Err(e) => { - self.err = Some(e); - return Err(()); - } - }; - Ok(()) - } - - fn resize(&mut self, new_len: usize) { - // Resizing always appends 0 bytes and since new mmaps start out as 0 - // bytes we don't actually need to do anything as part of this other - // than update our own length. - if new_len <= self.len { - return; - } - self.len = new_len; - } - - fn write_bytes(&mut self, val: &[u8]) { - let mmap = self.mmap.as_mut().expect("write before reserve"); - mmap[self.len..][..val.len()].copy_from_slice(val); - self.len += val.len(); - } - } - } -} diff --git a/crates/wasmtime/src/runtime/trampoline/func.rs b/crates/wasmtime/src/runtime/trampoline/func.rs index 50c078e9d2a3..03aa908348d9 100644 --- a/crates/wasmtime/src/runtime/trampoline/func.rs +++ b/crates/wasmtime/src/runtime/trampoline/func.rs @@ -85,7 +85,7 @@ pub fn create_array_call_function( where F: Fn(*mut VMContext, &mut [ValRaw]) -> Result<()> + Send + Sync + 'static, { - use crate::instantiate::finish_object; + use crate::compile::finish_object; use std::ptr; let mut obj = engine