,
+ /// Whether or not to enable Spectre mitigation on heap bounds checks.
+ heap_access_spectre_mitigation: bool,
}
pub fn ptr_type_from_ptr_size(size: u8) -> WasmValType {
@@ -93,6 +126,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
vmoffsets: &'a VMOffsets,
translation: &'translation ModuleTranslation<'data>,
types: &'translation ModuleTypesBuilder,
+ isa: &dyn TargetIsa,
) -> Self {
Self {
vmoffsets,
@@ -100,6 +134,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
types,
resolved_tables: HashMap::new(),
resolved_heaps: HashMap::new(),
+ heap_access_spectre_mitigation: isa.flags().enable_heap_access_spectre_mitigation(),
}
}
@@ -198,7 +233,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
}
}
- /// Resolved a [HeapData] from a [MemoryIndex].
+ /// Resolve a [HeapData] from a [MemoryIndex].
// TODO: (@saulecabrera)
// Handle shared memories when implementing support for Wasm Threads.
pub fn resolve_heap(&mut self, index: MemoryIndex) -> HeapData {
@@ -226,15 +261,24 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
),
};
+ let plan = &self.translation.module.memory_plans[index];
+ let (min_size, max_size) = heap_limits(&plan);
+ let (style, offset_guard_size) = heap_style_and_offset_guard_size(&plan);
+
*entry.insert(HeapData {
offset: base_offset,
import_from,
current_length_offset,
- ty: if self.translation.module.memory_plans[index].memory.memory64 {
- WasmValType::I64
+ style,
+ ty: if plan.memory.memory64 {
+ // TODO: Add support for 64-bit memories.
+ unimplemented!("memory64")
} else {
WasmValType::I32
},
+ min_size,
+ max_size,
+ offset_guard_size,
})
}
}
@@ -244,6 +288,11 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
pub fn table_plan(&mut self, index: TableIndex) -> &TablePlan {
&self.translation.module.table_plans[index]
}
+
+ /// Returns true if Spectre mitigations are enabled for heap bounds check.
+ pub fn heap_access_spectre_mitigation(&self) -> bool {
+ self.heap_access_spectre_mitigation
+ }
}
impl TypeConvert for FuncEnv<'_, '_, '_, P> {
@@ -255,3 +304,40 @@ impl TypeConvert for FuncEnv<'_, '_, '_, P> {
.lookup_heap_type(idx)
}
}
+
+fn heap_style_and_offset_guard_size(plan: &MemoryPlan) -> (HeapStyle, u64) {
+ match plan {
+ MemoryPlan {
+ style: MemoryStyle::Static { bound },
+ offset_guard_size,
+ ..
+ } => (
+ HeapStyle::Static {
+ bound: bound * u64::from(WASM_PAGE_SIZE),
+ },
+ *offset_guard_size,
+ ),
+
+ MemoryPlan {
+ style: MemoryStyle::Dynamic { .. },
+ offset_guard_size,
+ ..
+ } => (HeapStyle::Dynamic, *offset_guard_size),
+ }
+}
+
+fn heap_limits(plan: &MemoryPlan) -> (u64, Option) {
+ (
+ plan.memory
+ .minimum
+ .checked_mul(u64::from(WASM_PAGE_SIZE))
+ .unwrap_or_else(|| {
+ // 2^64 as a minimum doesn't fin in a 64 bit integer.
+ // So in this case, the minimum is clamped to u64::MAX.
+ u64::MAX
+ }),
+ plan.memory
+ .maximum
+ .and_then(|max| max.checked_mul(u64::from(WASM_PAGE_SIZE))),
+ )
+}
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 93b83fa67f54..210179bc146e 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -2,13 +2,17 @@ use crate::{
abi::{ABIOperand, ABISig, RetArea, ABI},
codegen::BlockSig,
isa::reg::Reg,
- masm::{IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, TrapCode},
+ masm::{ExtendKind, IntCmpKind, MacroAssembler, OperandSize, RegImm, SPOffset, TrapCode},
stack::TypedReg,
};
use anyhow::Result;
use smallvec::SmallVec;
-use wasmparser::{BinaryReader, FuncValidator, Operator, ValidatorResources, VisitOperator};
-use wasmtime_environ::{PtrSize, TableIndex, TypeIndex, WasmHeapType, WasmValType, FUNCREF_MASK};
+use wasmparser::{
+ BinaryReader, FuncValidator, MemArg, Operator, ValidatorResources, VisitOperator,
+};
+use wasmtime_environ::{
+ MemoryIndex, PtrSize, TableIndex, TypeIndex, WasmHeapType, WasmValType, FUNCREF_MASK,
+};
mod context;
pub(crate) use context::*;
@@ -20,6 +24,9 @@ mod control;
pub(crate) use control::*;
mod builtin;
pub use builtin::*;
+pub(crate) mod bounds;
+
+use bounds::{Bounds, ImmOffset, Index};
/// The code generation abstraction.
pub(crate) struct CodeGen<'a, 'translation: 'a, 'data: 'translation, M>
@@ -430,6 +437,204 @@ where
self.masm.bind(cont);
}
+
+ /// Emits a series of instructions to bounds check and calculate the address
+ /// of the given WebAssembly memory.
+ /// This function returns a register containing the requested address.
+ ///
+ /// In essence, when computing the heap address for a WebAssembly load or
+ /// store instruction the objective is to ensure that such access is safe,
+ /// but also to perform the least amount of checks, and rely on the system to
+ /// detect illegal memory accesses where applicable.
+ ///
+ /// Winch follows almost the same principles as Cranelift when it comes to
+ /// bounds checks, for a more detailed explanation refer to
+ /// [cranelift_wasm::code_translator::prepare_addr].
+ ///
+ /// Winch implementation differs in that, it defaults to the general case
+ /// for dynamic heaps rather than optimizing for doing the least amount of
+ /// work possible at runtime, this is done to align with Winch's principle
+ /// of doing the least amount of work possible at compile time. For static
+ /// heaps, Winch does a bit more of work, given that some of the cases that
+ /// are checked against, can benefit compilation times, like for example,
+ /// detecting an out of bouds access at compile time.
+ pub fn emit_compute_heap_address(
+ &mut self,
+ memarg: &MemArg,
+ access_size: OperandSize,
+ ) -> Option {
+ let ptr_size: OperandSize = self.env.ptr_type().into();
+ let enable_spectre_mitigation = self.env.heap_access_spectre_mitigation();
+ let add_offset_and_access_size = |offset: ImmOffset, access_size: OperandSize| {
+ (access_size.bytes() as u64) + (offset.as_u32() as u64)
+ };
+
+ let memory_index = MemoryIndex::from_u32(memarg.memory);
+ let heap = self.env.resolve_heap(memory_index);
+ let index = Index::from_typed_reg(self.context.pop_to_reg(self.masm, None));
+ let offset = bounds::ensure_index_and_offset(self.masm, index, memarg.offset, ptr_size);
+ let offset_with_access_size = add_offset_and_access_size(offset, access_size);
+
+ let addr = match heap.style {
+ // == Dynamic Heaps ==
+
+ // Account for the general case for dynamic memories. The access is
+ // out of bounds if:
+ // * index + offset + access_size overflows
+ // OR
+ // * index + offset + access_size > bound
+ HeapStyle::Dynamic => {
+ let bounds =
+ bounds::load_dynamic_heap_bounds(&mut self.context, self.masm, &heap, ptr_size);
+
+ let index_reg = index.as_typed_reg().reg;
+ // Perform
+ // index = index + offset + access_size, trapping if the
+ // addition overflows.
+ self.masm.checked_uadd(
+ index_reg,
+ index_reg,
+ RegImm::i64(offset_with_access_size as i64),
+ ptr_size,
+ TrapCode::HeapOutOfBounds,
+ );
+
+ let addr = bounds::load_heap_addr_checked(
+ self.masm,
+ &mut self.context,
+ ptr_size,
+ &heap,
+ enable_spectre_mitigation,
+ bounds,
+ index,
+ offset,
+ |masm, bounds, index| {
+ let index_reg = index.as_typed_reg().reg;
+ let bounds_reg = bounds.as_typed_reg().reg;
+ masm.cmp(bounds_reg.into(), index_reg.into(), heap.ty.into());
+ IntCmpKind::GtU
+ },
+ );
+ self.context.free_reg(bounds.as_typed_reg().reg);
+ Some(addr)
+ }
+
+ // == Static Heaps ==
+
+ // Detect at compile time if the access is out of bounds.
+ // Doing so will put the compiler in an unreachable code state,
+ // optimizing the work that the compiler has to do until the
+ // reachability is restored or when reaching the end of the
+ // function.
+ HeapStyle::Static { bound } if offset_with_access_size > bound => {
+ self.masm.trap(TrapCode::HeapOutOfBounds);
+ self.context.reachable = false;
+ None
+ }
+
+ // Account for the case in which we can completely elide the bounds
+ // checks.
+ //
+ // This case, makes use of the fact that if a memory access uses
+ // a 32-bit index, then we be certain that
+ //
+ // index <= u32::MAX
+ //
+ // Therfore if any 32-bit index access occurs in the region
+ // represented by
+ //
+ // bound + guard_size - (offset + access_size)
+ //
+ // We are certain that it's in bounds or that the underlying virtual
+ // memory subsystem will report an illegal access at runtime.
+ //
+ // Note:
+ //
+ // * bound - (offset + access_size) cannot wrap, because it's checked
+ // in the condition above.
+ // * bound + heap.offset_guard_size is guaranteed to not overflow if
+ // the heap configuration is correct, given that it's address must
+ // fit in 64-bits.
+ // * If the heap type is 32-bits, the offset is at most u32::MAX, so
+ // no adjustment is needed as part of
+ // [bounds::ensure_index_and_offset].
+ HeapStyle::Static { bound }
+ if heap.ty == WasmValType::I32
+ && u64::from(u32::MAX)
+ <= u64::from(bound) + u64::from(heap.offset_guard_size)
+ - offset_with_access_size =>
+ {
+ let addr = self.context.any_gpr(self.masm);
+ bounds::load_heap_addr_unchecked(self.masm, &heap, index, offset, addr, ptr_size);
+ Some(addr)
+ }
+
+ // Account for the general case of static memories. The access is out
+ // of bounds if:
+ //
+ // index > bound - (offset + access_size)
+ //
+ // bound - (offset + access_size) cannot wrap, because we already
+ // checked that (offset + access_size) > bound, above.
+ HeapStyle::Static { bound } => {
+ let bounds = Bounds::from_u64(bound);
+ let addr = bounds::load_heap_addr_checked(
+ self.masm,
+ &mut self.context,
+ ptr_size,
+ &heap,
+ enable_spectre_mitigation,
+ bounds,
+ index,
+ offset,
+ |masm, bounds, index| {
+ let adjusted_bounds = bounds.as_u64() - offset_with_access_size;
+ let index_reg = index.as_typed_reg().reg;
+ masm.cmp(RegImm::i64(adjusted_bounds as i64), index_reg, ptr_size);
+ IntCmpKind::GtU
+ },
+ );
+ Some(addr)
+ }
+ };
+
+ self.context.free_reg(index.as_typed_reg().reg);
+ addr
+ }
+
+ /// Emit a WebAssembly load.
+ pub fn emit_wasm_load(
+ &mut self,
+ arg: &MemArg,
+ ty: WasmValType,
+ size: OperandSize,
+ sextend: Option,
+ ) {
+ if let Some(addr) = self.emit_compute_heap_address(&arg, size) {
+ let dst = match ty {
+ WasmValType::I32 | WasmValType::I64 => self.context.any_gpr(self.masm),
+ WasmValType::F32 | WasmValType::F64 => self.context.any_fpr(self.masm),
+ _ => unreachable!(),
+ };
+
+ let src = self.masm.address_at_reg(addr, 0);
+ self.masm.wasm_load(src, dst, size, sextend);
+ self.context.stack.push(TypedReg::new(ty, dst).into());
+ self.context.free_reg(addr);
+ }
+ }
+
+ /// Emit a WebAssembly store.
+ pub fn emit_wasm_store(&mut self, arg: &MemArg, size: OperandSize) {
+ let src = self.context.pop_to_reg(self.masm, None);
+ if let Some(addr) = self.emit_compute_heap_address(&arg, size) {
+ self.masm
+ .wasm_store(src.reg.into(), self.masm.address_at_reg(addr, 0), size);
+
+ self.context.free_reg(addr);
+ self.context.free_reg(src);
+ }
+ }
}
/// Returns the index of the [`ControlStackFrame`] for the given
diff --git a/winch/codegen/src/frame/mod.rs b/winch/codegen/src/frame/mod.rs
index e44666cb675b..2c1b931375de 100644
--- a/winch/codegen/src/frame/mod.rs
+++ b/winch/codegen/src/frame/mod.rs
@@ -40,7 +40,7 @@ impl DefinedLocals {
reader: &mut BinaryReader<'_>,
validator: &mut FuncValidator,
) -> Result {
- let mut next_stack = 0;
+ let mut next_stack: u32 = 0;
// The first 32 bits of a Wasm binary function describe the number of locals.
let local_count = reader.read_var_u32()?;
let mut slots: Locals = Default::default();
@@ -54,7 +54,7 @@ impl DefinedLocals {
let ty = types.convert_valtype(ty);
for _ in 0..count {
let ty_size = ::sizeof(&ty);
- next_stack = align_to(next_stack, ty_size) + ty_size;
+ next_stack = align_to(next_stack, ty_size as u32) + (ty_size as u32);
slots.push(LocalSlot::new(ty, next_stack));
}
}
@@ -102,7 +102,7 @@ impl Frame {
);
// Align the locals to add a slot for the VMContext pointer.
- let ptr_size = ::word_bytes();
+ let ptr_size = ::word_bytes() as u32;
let vmctx_offset =
align_to(defined_locals_start + defined_locals.stack_size, ptr_size) + ptr_size;
diff --git a/winch/codegen/src/isa/aarch64/abi.rs b/winch/codegen/src/isa/aarch64/abi.rs
index 0afcaaf9d5d3..3e4bdea17a05 100644
--- a/winch/codegen/src/isa/aarch64/abi.rs
+++ b/winch/codegen/src/isa/aarch64/abi.rs
@@ -79,7 +79,7 @@ impl ABI for Aarch64ABI {
8
}
- fn word_bits() -> u32 {
+ fn word_bits() -> u8 {
64
}
@@ -147,11 +147,11 @@ impl ABI for Aarch64ABI {
regs::callee_saved()
}
- fn stack_slot_size() -> u32 {
+ fn stack_slot_size() -> u8 {
Self::word_bytes()
}
- fn sizeof(ty: &WasmValType) -> u32 {
+ fn sizeof(ty: &WasmValType) -> u8 {
match ty {
WasmValType::Ref(rt) => match rt.heap_type {
WasmHeapType::Func => Self::word_bytes(),
@@ -162,6 +162,10 @@ impl ABI for Aarch64ABI {
ty => unimplemented!("Support for WasmType: {ty}"),
}
}
+
+ fn sizeof_bits(ty: &WasmValType) -> u8 {
+ Self::sizeof(ty) * 8
+ }
}
impl Aarch64ABI {
@@ -185,20 +189,20 @@ impl Aarch64ABI {
let ty_size = ::sizeof(wasm_arg);
let default = || {
- let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size);
+ let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size as u32);
let slot_size = Self::stack_slot_size();
// Stack slots for parameters are aligned to a fixed slot size,
// in the case of Aarch64, 8 bytes.
// Stack slots for returns are type-size aligned.
let next_stack = if params_or_returns == ParamsOrReturns::Params {
- align_to(stack_offset, slot_size) + slot_size
+ align_to(stack_offset, slot_size as u32) + (slot_size as u32)
} else {
- align_to(stack_offset, ty_size) + ty_size
+ align_to(stack_offset, ty_size as u32) + (ty_size as u32)
};
(arg, next_stack)
};
reg.map_or_else(default, |reg| {
- (ABIOperand::reg(reg, *ty, ty_size), stack_offset)
+ (ABIOperand::reg(reg, *ty, ty_size as u32), stack_offset)
})
}
}
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 85e7b7538da3..596d39940940 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -50,7 +50,7 @@ impl Masm for MacroAssembler {
}
fn check_stack(&mut self) {
- // TODO: implement when we have more complete assembler support
+ // TODO: Implement when we have more complete assembler support.
}
fn epilogue(&mut self, locals_size: u32) {
@@ -158,6 +158,10 @@ impl Masm for MacroAssembler {
self.asm.str(src, dst, size);
}
+ fn wasm_store(&mut self, _src: Reg, _dst: Self::Address, _size: OperandSize) {
+ todo!()
+ }
+
fn call(
&mut self,
_stack_args_size: u32,
@@ -174,6 +178,16 @@ impl Masm for MacroAssembler {
todo!()
}
+ fn wasm_load(
+ &mut self,
+ _src: Self::Address,
+ _dst: Reg,
+ _size: OperandSize,
+ _kind: Option,
+ ) {
+ todo!()
+ }
+
fn load_addr(&mut self, _src: Self::Address, _dst: Reg, _size: OperandSize) {
todo!()
}
@@ -231,6 +245,17 @@ impl Masm for MacroAssembler {
}
}
+ fn checked_uadd(
+ &mut self,
+ _dst: Reg,
+ _lhs: Reg,
+ _rhs: RegImm,
+ _size: OperandSize,
+ _trap: TrapCode,
+ ) {
+ todo!()
+ }
+
fn sub(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize) {
match (rhs, lhs, dst) {
(RegImm::Imm(v), rn, rd) => {
@@ -408,13 +433,13 @@ impl Masm for MacroAssembler {
fn push(&mut self, reg: Reg, _size: OperandSize) -> StackSlot {
let size = ::word_bytes();
- self.reserve_stack(size);
+ self.reserve_stack(size as u32);
let address = Address::from_shadow_sp(size as i64);
self.asm.str(reg, address, OperandSize::S64);
StackSlot {
offset: SPOffset::from_u32(self.sp_offset),
- size,
+ size: size as u32,
}
}
@@ -489,6 +514,10 @@ impl Masm for MacroAssembler {
todo!()
}
+ fn trap(&mut self, _code: TrapCode) {
+ todo!()
+ }
+
fn trapz(&mut self, _src: Reg, _code: TrapCode) {
todo!()
}
diff --git a/winch/codegen/src/isa/aarch64/mod.rs b/winch/codegen/src/isa/aarch64/mod.rs
index a17e8fbdde07..f66215107f52 100644
--- a/winch/codegen/src/isa/aarch64/mod.rs
+++ b/winch/codegen/src/isa/aarch64/mod.rs
@@ -98,7 +98,7 @@ impl TargetIsa for Aarch64 {
let stack = Stack::new();
let abi_sig = abi::Aarch64ABI::sig(sig, &CallingConvention::Default);
- let env = FuncEnv::new(&vmoffsets, translation, types);
+ let env = FuncEnv::new(&vmoffsets, translation, types, self);
let defined_locals = DefinedLocals::new::(&env, &mut body, validator)?;
let frame = Frame::new::(&abi_sig, &defined_locals)?;
let gpr = RegBitSet::int(
diff --git a/winch/codegen/src/isa/x64/abi.rs b/winch/codegen/src/isa/x64/abi.rs
index ac5e6672a1e7..81c37b8845ac 100644
--- a/winch/codegen/src/isa/x64/abi.rs
+++ b/winch/codegen/src/isa/x64/abi.rs
@@ -93,7 +93,7 @@ impl ABI for X64ABI {
8
}
- fn word_bits() -> u32 {
+ fn word_bits() -> u8 {
64
}
@@ -178,11 +178,11 @@ impl ABI for X64ABI {
regs::callee_saved(call_conv)
}
- fn stack_slot_size() -> u32 {
+ fn stack_slot_size() -> u8 {
Self::word_bytes()
}
- fn sizeof(ty: &WasmValType) -> u32 {
+ fn sizeof(ty: &WasmValType) -> u8 {
match ty {
WasmValType::Ref(rt) => match rt.heap_type {
WasmHeapType::Func => Self::word_bytes(),
@@ -193,6 +193,10 @@ impl ABI for X64ABI {
ty => unimplemented!("Support for WasmType: {ty}"),
}
}
+
+ fn sizeof_bits(ty: &WasmValType) -> u8 {
+ Self::sizeof(ty) * 8
+ }
}
impl X64ABI {
@@ -227,28 +231,28 @@ impl X64ABI {
let ty_size = ::sizeof(wasm_arg);
let default = || {
- let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size);
+ let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size as u32);
let slot_size = Self::stack_slot_size();
// Stack slots for parameters are aligned to a fixed slot size,
// in the case of x64, 8 bytes.
// Stack slots for returns are type-size aligned.
let next_stack = if params_or_returns == ParamsOrReturns::Params {
- align_to(stack_offset, slot_size) + slot_size
+ align_to(stack_offset, slot_size as u32) + (slot_size as u32)
} else {
// For the default calling convention, we don't type-size align,
// given that results on the stack must match spills generated
// from within the compiler, which are not type-size aligned.
if call_conv.is_default() {
- stack_offset + ty_size
+ stack_offset + (ty_size as u32)
} else {
- align_to(stack_offset, ty_size) + ty_size
+ align_to(stack_offset, ty_size as u32) + (ty_size as u32)
}
};
(arg, next_stack)
};
reg.map_or_else(default, |reg| {
- (ABIOperand::reg(reg, *ty, ty_size), stack_offset)
+ (ABIOperand::reg(reg, *ty, ty_size as u32), stack_offset)
})
}
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 27acea761e80..89f1a5b82e22 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -6,7 +6,9 @@ use crate::{
};
use cranelift_codegen::{
entity::EntityRef,
- ir::{types, ConstantPool, ExternalName, LibCall, Opcode, TrapCode, UserExternalNameRef},
+ ir::{
+ types, ConstantPool, ExternalName, LibCall, MemFlags, Opcode, TrapCode, UserExternalNameRef,
+ },
isa::{
unwind::UnwindInst,
x64::{
@@ -88,6 +90,8 @@ impl From for Xmm {
impl From for args::OperandSize {
fn from(size: OperandSize) -> Self {
match size {
+ OperandSize::S8 => Self::Size8,
+ OperandSize::S16 => Self::Size16,
OperandSize::S32 => Self::Size32,
OperandSize::S64 => Self::Size64,
s => panic!("Invalid operand size {:?}", s),
@@ -147,6 +151,20 @@ impl From for ExtMode {
}
}
+impl From for Option {
+ // Helper for cases in which it's known that the widening must be
+ // to quadword.
+ fn from(value: OperandSize) -> Self {
+ use OperandSize::*;
+ match value {
+ S128 | S64 => None,
+ S8 => Some(ExtMode::BQ),
+ S16 => Some(ExtMode::WQ),
+ S32 => Some(ExtMode::LQ),
+ }
+ }
+}
+
/// Low level assembler implementation for x64.
pub(crate) struct Assembler {
/// The machine instruction buffer.
@@ -205,19 +223,21 @@ impl Assembler {
pool: &mut ConstantPool,
constants: &mut VCodeConstants,
buffer: &mut MachBuffer,
+ memflags: MemFlags,
) -> SyntheticAmode {
match addr {
Address::Offset { base, offset } => {
- SyntheticAmode::real(Amode::imm_reg(*offset as i32, (*base).into()))
+ let amode = Amode::imm_reg(*offset as i32, (*base).into()).with_flags(memflags);
+ SyntheticAmode::real(amode)
}
Address::Const(c) => {
// Defer the creation of the
// `SyntheticAmode::ConstantOffset` addressing mode
// until the address is referenced by an actual
- // instrunction.
+ // instruction.
let constant_data = pool.get(*c);
let data = VCodeConstantData::Pool(*c, constant_data.clone());
- // If the constaant data is not marked as used, it will be
+ // If the constant data is not marked as used, it will be
// inserted, therefore, it needs to be registered.
let needs_registration = !constants.pool_uses(&data);
let constant = constants.insert(VCodeConstantData::Pool(*c, constant_data.clone()));
@@ -264,10 +284,15 @@ impl Assembler {
}
/// Register-to-memory move.
- pub fn mov_rm(&mut self, src: Reg, addr: &Address, size: OperandSize) {
+ pub fn mov_rm(&mut self, src: Reg, addr: &Address, size: OperandSize, flags: MemFlags) {
assert!(addr.is_offset());
- let dst =
- Self::to_synthetic_amode(addr, &mut self.pool, &mut self.constants, &mut self.buffer);
+ let dst = Self::to_synthetic_amode(
+ addr,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ flags,
+ );
self.emit(Inst::MovRM {
size: size.into(),
src: src.into(),
@@ -276,10 +301,15 @@ impl Assembler {
}
/// Immediate-to-memory move.
- pub fn mov_im(&mut self, src: i32, addr: &Address, size: OperandSize) {
+ pub fn mov_im(&mut self, src: i32, addr: &Address, size: OperandSize, flags: MemFlags) {
assert!(addr.is_offset());
- let dst =
- Self::to_synthetic_amode(addr, &mut self.pool, &mut self.constants, &mut self.buffer);
+ let dst = Self::to_synthetic_amode(
+ addr,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ flags,
+ );
self.emit(Inst::MovImmM {
size: size.into(),
simm32: src,
@@ -299,28 +329,55 @@ impl Assembler {
});
}
- /// Memory-to-register load.
- pub fn mov_mr(&mut self, addr: &Address, dst: Reg, size: OperandSize) {
- use OperandSize::S64;
-
- let src =
- Self::to_synthetic_amode(addr, &mut self.pool, &mut self.constants, &mut self.buffer);
+ /// Zero-extend memory-to-register load.
+ pub fn movzx_mr(&mut self, addr: &Address, dst: Reg, ext: Option, memflags: MemFlags) {
+ let src = Self::to_synthetic_amode(
+ addr,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ memflags,
+ );
- if size == S64 {
- self.emit(Inst::Mov64MR {
- src,
- dst: dst.into(),
- });
- } else {
+ if let Some(ext) = ext {
let reg_mem = RegMem::mem(src);
self.emit(Inst::MovzxRmR {
- ext_mode: ExtMode::LQ,
+ ext_mode: ext,
src: GprMem::new(reg_mem).expect("valid memory address"),
dst: dst.into(),
});
+ } else {
+ self.emit(Inst::Mov64MR {
+ src,
+ dst: dst.into(),
+ });
}
}
+ // Sign-extend memory-to-register load.
+ pub fn movsx_mr(
+ &mut self,
+ addr: &Address,
+ dst: Reg,
+ ext: impl Into,
+ memflags: MemFlags,
+ ) {
+ let src = Self::to_synthetic_amode(
+ addr,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ memflags,
+ );
+
+ let reg_mem = RegMem::mem(src);
+ self.emit(Inst::MovsxRmR {
+ ext_mode: ext.into(),
+ src: GprMem::new(reg_mem).expect("valid memory address"),
+ dst: dst.into(),
+ })
+ }
+
/// Register-to-register move with zero extension.
pub fn movzx_rr(&mut self, src: Reg, dst: Reg, kind: ExtendKind) {
self.emit(Inst::MovzxRmR {
@@ -359,6 +416,7 @@ impl Assembler {
S32 => SseOpcode::Movaps,
S64 => SseOpcode::Movapd,
S128 => SseOpcode::Movdqa,
+ S8 | S16 => unreachable!(),
};
self.emit(Inst::XmmUnaryRmRUnaligned {
@@ -369,7 +427,7 @@ impl Assembler {
}
/// Single and double precision floating point load.
- pub fn xmm_mov_mr(&mut self, src: &Address, dst: Reg, size: OperandSize) {
+ pub fn xmm_mov_mr(&mut self, src: &Address, dst: Reg, size: OperandSize, flags: MemFlags) {
use OperandSize::*;
assert!(dst.is_float());
@@ -377,10 +435,16 @@ impl Assembler {
S32 => SseOpcode::Movss,
S64 => SseOpcode::Movsd,
S128 => SseOpcode::Movdqu,
+ S16 | S8 => unreachable!(),
};
- let src =
- Self::to_synthetic_amode(src, &mut self.pool, &mut self.constants, &mut self.buffer);
+ let src = Self::to_synthetic_amode(
+ src,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ flags,
+ );
self.emit(Inst::XmmUnaryRmRUnaligned {
op,
src: XmmMem::new(RegMem::mem(src)).expect("valid xmm unaligned"),
@@ -389,7 +453,7 @@ impl Assembler {
}
/// Single and double precision floating point store.
- pub fn xmm_mov_rm(&mut self, src: Reg, dst: &Address, size: OperandSize) {
+ pub fn xmm_mov_rm(&mut self, src: Reg, dst: &Address, size: OperandSize, flags: MemFlags) {
use OperandSize::*;
assert!(src.is_float());
@@ -398,10 +462,16 @@ impl Assembler {
S32 => SseOpcode::Movss,
S64 => SseOpcode::Movsd,
S128 => SseOpcode::Movdqu,
+ S16 | S8 => unreachable!(),
};
- let dst =
- Self::to_synthetic_amode(dst, &mut self.pool, &mut self.constants, &mut self.buffer);
+ let dst = Self::to_synthetic_amode(
+ dst,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ flags,
+ );
self.emit(Inst::XmmMovRM {
op,
src: src.into(),
@@ -416,6 +486,7 @@ impl Assembler {
OperandSize::S64 => types::F64,
// Move the entire 128 bits via movdqa.
OperandSize::S128 => types::I128,
+ OperandSize::S8 | OperandSize::S16 => unreachable!(),
};
self.emit(Inst::XmmCmove {
@@ -478,7 +549,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Andps,
OperandSize::S64 => SseOpcode::Andpd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmR {
@@ -494,7 +565,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Andnps,
OperandSize::S64 => SseOpcode::Andnpd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmR {
@@ -509,7 +580,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Movd,
OperandSize::S64 => SseOpcode::Movq,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::GprToXmm {
@@ -524,7 +595,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Movd,
OperandSize::S64 => SseOpcode::Movq,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmToGpr {
@@ -590,7 +661,7 @@ impl Assembler {
let op = match dst_size {
OperandSize::S32 => SseOpcode::Cvtsi2ss,
OperandSize::S64 => SseOpcode::Cvtsi2sd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S16 | OperandSize::S8 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::CvtIntToFloat {
op,
@@ -667,7 +738,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Orps,
OperandSize::S64 => SseOpcode::Orpd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmR {
@@ -706,7 +777,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Xorps,
OperandSize::S64 => SseOpcode::Xorpd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmR {
@@ -927,7 +998,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Ucomiss,
OperandSize::S64 => SseOpcode::Ucomisd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmCmpRmR {
@@ -1053,7 +1124,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Addss,
OperandSize::S64 => SseOpcode::Addsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmRUnaligned {
@@ -1069,7 +1140,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Subss,
OperandSize::S64 => SseOpcode::Subsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmRUnaligned {
@@ -1085,7 +1156,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Mulss,
OperandSize::S64 => SseOpcode::Mulsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmRUnaligned {
@@ -1101,7 +1172,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Divss,
OperandSize::S64 => SseOpcode::Divsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmRUnaligned {
@@ -1140,7 +1211,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Roundss,
OperandSize::S64 => SseOpcode::Roundsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
let imm: u8 = match mode {
@@ -1162,7 +1233,7 @@ impl Assembler {
let op = match size {
OperandSize::S32 => SseOpcode::Sqrtss,
OperandSize::S64 => SseOpcode::Sqrtsd,
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.emit(Inst::XmmRmR {
@@ -1257,7 +1328,7 @@ impl Assembler {
}
/// Conditional trap.
- pub fn trapif(&mut self, cc: IntCmpKind, trap_code: TrapCode) {
+ pub fn trapif(&mut self, cc: impl Into, trap_code: TrapCode) {
self.emit(Inst::TrapIf {
cc: cc.into(),
trap_code,
@@ -1266,8 +1337,13 @@ impl Assembler {
/// Load effective address.
pub fn lea(&mut self, addr: &Address, dst: Reg, size: OperandSize) {
- let addr =
- Self::to_synthetic_amode(addr, &mut self.pool, &mut self.constants, &mut self.buffer);
+ let addr = Self::to_synthetic_amode(
+ addr,
+ &mut self.pool,
+ &mut self.constants,
+ &mut self.buffer,
+ MemFlags::trusted(),
+ );
self.emit(Inst::LoadEffectiveAddress {
addr,
dst: dst.into(),
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index bc8f240f28c8..673c622151b1 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -16,7 +16,7 @@ use crate::{
};
use crate::{
abi::{self, align_to, calculate_frame_adjustment, LocalSlot},
- codegen::{ptr_type_from_ptr_size, Callee, CodeGenContext, FnCall, TableData},
+ codegen::{ptr_type_from_ptr_size, Callee, CodeGenContext, FnCall, HeapData, TableData},
stack::Val,
};
use crate::{
@@ -24,8 +24,13 @@ use crate::{
masm::CalleeKind,
};
use cranelift_codegen::{
- isa::unwind::UnwindInst, isa::x64::settings as x64_settings, settings, Final,
- MachBufferFinalized, MachLabel,
+ ir::{Endianness, MemFlags},
+ isa::unwind::UnwindInst,
+ isa::x64::{
+ args::{ExtMode, CC},
+ settings as x64_settings,
+ },
+ settings, Final, MachBufferFinalized, MachLabel,
};
use wasmtime_environ::{PtrSize, WasmValType, WASM_PAGE_SIZE};
@@ -42,6 +47,10 @@ pub(crate) struct MacroAssembler {
shared_flags: settings::Flags,
/// The target pointer size.
ptr_size: OperandSize,
+ /// Flags for native loads/stores.
+ trusted_flags: MemFlags,
+ /// Flags for Wasm loads/stores.
+ untrusted_flags: MemFlags,
}
impl Masm for MacroAssembler {
@@ -96,7 +105,7 @@ impl Masm for MacroAssembler {
fn push(&mut self, reg: Reg, size: OperandSize) -> StackSlot {
let bytes = match (reg.class(), size) {
(RegClass::Int, OperandSize::S64) => {
- let word_bytes = ::word_bytes();
+ let word_bytes = ::word_bytes() as u32;
self.asm.push_r(reg);
self.increment_sp(word_bytes);
word_bytes
@@ -105,15 +114,24 @@ impl Masm for MacroAssembler {
let bytes = size.bytes();
self.reserve_stack(bytes);
let sp_offset = SPOffset::from_u32(self.sp_offset);
- self.asm.mov_rm(reg, &self.address_from_sp(sp_offset), size);
+ self.asm.mov_rm(
+ reg,
+ &self.address_from_sp(sp_offset),
+ size,
+ self.trusted_flags,
+ );
bytes
}
(RegClass::Float, _) => {
let bytes = size.bytes();
self.reserve_stack(bytes);
let sp_offset = SPOffset::from_u32(self.sp_offset);
- self.asm
- .xmm_mov_rm(reg, &self.address_from_sp(sp_offset), size);
+ self.asm.xmm_mov_rm(
+ reg,
+ &self.address_from_sp(sp_offset),
+ size,
+ self.trusted_flags,
+ );
bytes
}
_ => unreachable!(),
@@ -190,8 +208,12 @@ impl Masm for MacroAssembler {
// If the table data declares a particular offset base,
// load the address into a register to further use it as
// the table address.
- self.asm
- .mov_mr(&self.address_at_vmctx(offset), ptr_base, self.ptr_size);
+ self.asm.movzx_mr(
+ &self.address_at_vmctx(offset),
+ ptr_base,
+ self.ptr_size.into(),
+ self.trusted_flags,
+ );
} else {
// Else, simply move the vmctx register into the addr register as
// the base to calculate the table address.
@@ -201,7 +223,8 @@ impl Masm for MacroAssembler {
// OOB check.
let bound_addr = self.address_at_reg(ptr_base, table_data.current_elems_offset);
let bound_size = table_data.current_elements_size;
- self.asm.mov_mr(&bound_addr, bound, bound_size);
+ self.asm
+ .movzx_mr(&bound_addr, bound, bound_size.into(), self.trusted_flags);
self.asm.cmp_rr(bound, index, bound_size);
self.asm.trapif(IntCmpKind::GeU, TrapCode::TableOutOfBounds);
@@ -215,10 +238,11 @@ impl Masm for MacroAssembler {
scratch,
table_data.element_size,
);
- self.asm.mov_mr(
+ self.asm.movzx_mr(
&self.address_at_reg(ptr_base, table_data.offset),
ptr_base,
- self.ptr_size,
+ self.ptr_size.into(),
+ self.trusted_flags,
);
// Copy the value of the table base into a temporary register
// so that we can use it later in case of a misspeculation.
@@ -242,34 +266,51 @@ impl Masm for MacroAssembler {
let size = context.any_gpr(self);
if let Some(offset) = table_data.import_from {
- self.asm
- .mov_mr(&self.address_at_vmctx(offset), scratch, self.ptr_size);
+ self.asm.movzx_mr(
+ &self.address_at_vmctx(offset),
+ scratch,
+ self.ptr_size.into(),
+ self.trusted_flags,
+ );
} else {
self.asm.mov_rr(vmctx, scratch, self.ptr_size);
};
let size_addr = Address::offset(scratch, table_data.current_elems_offset);
- self.asm
- .mov_mr(&size_addr, size, table_data.current_elements_size);
+ self.asm.movzx_mr(
+ &size_addr,
+ size,
+ table_data.current_elements_size.into(),
+ self.trusted_flags,
+ );
context.stack.push(TypedReg::i32(size).into());
}
- fn memory_size(&mut self, heap_data: &crate::codegen::HeapData, context: &mut CodeGenContext) {
+ fn memory_size(&mut self, heap_data: &HeapData, context: &mut CodeGenContext) {
let size_reg = context.any_gpr(self);
let scratch = regs::scratch();
let vmctx = ::vmctx_reg();
let base = if let Some(offset) = heap_data.import_from {
- self.asm
- .mov_mr(&self.address_at_vmctx(offset), scratch, self.ptr_size);
+ self.asm.movzx_mr(
+ &self.address_at_vmctx(offset),
+ scratch,
+ self.ptr_size.into(),
+ self.trusted_flags,
+ );
scratch
} else {
vmctx
};
let size_addr = Address::offset(base, heap_data.current_length_offset);
- self.asm.mov_mr(&size_addr, size_reg, self.ptr_size);
+ self.asm.movzx_mr(
+ &size_addr,
+ size_reg,
+ self.ptr_size.into(),
+ self.trusted_flags,
+ );
// Prepare the stack to emit a shift to get the size in pages rather
// than in bytes.
context
@@ -310,39 +351,11 @@ impl Masm for MacroAssembler {
}
fn store(&mut self, src: RegImm, dst: Address, size: OperandSize) {
- let scratch = ::scratch_reg();
- let float_scratch = ::float_scratch_reg();
- match src {
- RegImm::Imm(imm) => match imm {
- I::I32(v) => self.asm.mov_im(v as i32, &dst, size),
- I::I64(v) => match v.try_into() {
- Ok(v) => self.asm.mov_im(v, &dst, size),
- Err(_) => {
- // If the immediate doesn't sign extend, use a scratch
- // register.
- self.asm.mov_ir(v, scratch, size);
- self.asm.mov_rm(scratch, &dst, size);
- }
- },
- I::F32(v) => {
- let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
- self.asm.xmm_mov_mr(&addr, float_scratch, size);
- self.asm.xmm_mov_rm(float_scratch, &dst, size);
- }
- I::F64(v) => {
- let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
- self.asm.xmm_mov_mr(&addr, float_scratch, size);
- self.asm.xmm_mov_rm(float_scratch, &dst, size);
- }
- },
- RegImm::Reg(reg) => {
- if reg.is_int() {
- self.asm.mov_rm(reg, &dst, size);
- } else {
- self.asm.xmm_mov_rm(reg, &dst, size);
- }
- }
- }
+ self.store_impl(src, dst, size, self.trusted_flags);
+ }
+
+ fn wasm_store(&mut self, src: Reg, dst: Self::Address, size: OperandSize) {
+ self.store_impl(src.into(), dst, size, self.untrusted_flags);
}
fn pop(&mut self, dst: Reg, size: OperandSize) {
@@ -350,16 +363,17 @@ impl Masm for MacroAssembler {
match (dst.class(), size) {
(RegClass::Int, OperandSize::S32) => {
let addr = self.address_from_sp(current_sp);
- self.asm.mov_mr(&addr, dst, size);
+ self.asm
+ .movzx_mr(&addr, dst, size.into(), self.trusted_flags);
self.free_stack(size.bytes());
}
(RegClass::Int, OperandSize::S64) => {
self.asm.pop_r(dst);
- self.decrement_sp(::word_bytes());
+ self.decrement_sp(::word_bytes() as u32);
}
(RegClass::Float, _) => {
let addr = self.address_from_sp(current_sp);
- self.asm.xmm_mov_mr(&addr, dst, size);
+ self.asm.xmm_mov_mr(&addr, dst, size, self.trusted_flags);
self.free_stack(size.bytes());
}
_ => unreachable!(),
@@ -395,10 +409,20 @@ impl Masm for MacroAssembler {
}
fn load(&mut self, src: Address, dst: Reg, size: OperandSize) {
- if dst.is_int() {
- self.asm.mov_mr(&src, dst, size);
+ self.load_impl::(src, dst, size, self.trusted_flags);
+ }
+
+ fn wasm_load(
+ &mut self,
+ src: Self::Address,
+ dst: Reg,
+ size: OperandSize,
+ kind: Option,
+ ) {
+ if let Some(ext) = kind {
+ self.asm.movsx_mr(&src, dst, ext, self.untrusted_flags);
} else {
- self.asm.xmm_mov_mr(&src, dst, size);
+ self.load_impl::(src, dst, size, self.untrusted_flags)
}
}
@@ -422,11 +446,11 @@ impl Masm for MacroAssembler {
I::I64(v) => self.asm.mov_ir(v, dst, size),
I::F32(v) => {
let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
- self.asm.xmm_mov_mr(&addr, dst, size);
+ self.asm.xmm_mov_mr(&addr, dst, size, self.trusted_flags);
}
I::F64(v) => {
let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
- self.asm.xmm_mov_mr(&addr, dst, size);
+ self.asm.xmm_mov_mr(&addr, dst, size, self.trusted_flags);
}
},
}
@@ -459,6 +483,11 @@ impl Masm for MacroAssembler {
}
}
+ fn checked_uadd(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize, trap: TrapCode) {
+ self.add(dst, lhs, rhs, size);
+ self.asm.trapif(CC::B, trap);
+ }
+
fn sub(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize) {
Self::ensure_two_argument_form(&dst, &lhs);
match (rhs, dst) {
@@ -534,7 +563,7 @@ impl Masm for MacroAssembler {
let sign_mask = match size {
OperandSize::S32 => I::I32(0x80000000),
OperandSize::S64 => I::I64(0x8000000000000000),
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
self.load_constant(&sign_mask, scratch_gpr, size);
self.asm.gpr_to_xmm(scratch_gpr, scratch_xmm, size);
@@ -556,7 +585,7 @@ impl Masm for MacroAssembler {
let mask = match size {
OperandSize::S32 => I::I32(0x80000000),
OperandSize::S64 => I::I64(0x8000000000000000),
- OperandSize::S128 => unreachable!(),
+ OperandSize::S8 | OperandSize::S16 | OperandSize::S128 => unreachable!(),
};
let scratch_gpr = regs::scratch();
self.load_constant(&mask, scratch_gpr, size);
@@ -570,7 +599,7 @@ impl Masm for MacroAssembler {
let mask = match size {
OperandSize::S32 => I::I32(0x7fffffff),
OperandSize::S64 => I::I64(0x7fffffffffffffff),
- OperandSize::S128 => unreachable!(),
+ OperandSize::S128 | OperandSize::S16 | OperandSize::S8 => unreachable!(),
};
let scratch_gpr = regs::scratch();
self.load_constant(&mask, scratch_gpr, size);
@@ -860,7 +889,7 @@ impl Masm for MacroAssembler {
self.asm.bsr(src.into(), dst.into(), size);
self.asm.setcc(IntCmpKind::Ne, scratch.into());
self.asm.neg(dst, dst, size);
- self.asm.add_ir(size.num_bits(), dst, size);
+ self.asm.add_ir(size.num_bits() as i32, dst, size);
self.asm.sub_rr(scratch, dst, size);
}
}
@@ -1082,6 +1111,10 @@ impl Masm for MacroAssembler {
self.asm.trap(TrapCode::UnreachableCodeReached)
}
+ fn trap(&mut self, code: TrapCode) {
+ self.asm.trap(code);
+ }
+
fn trapif(&mut self, cc: IntCmpKind, code: TrapCode) {
self.asm.trapif(cc, code);
}
@@ -1118,12 +1151,21 @@ impl MacroAssembler {
shared_flags: settings::Flags,
isa_flags: x64_settings::Flags,
) -> Self {
+ let ptr_type: WasmValType = ptr_type_from_ptr_size(ptr_size.size()).into();
+ // Flags used for WebAssembly loads / stores.
+ // Untrusted by default so we don't set `no_trap`.
+ // We also ensure that the endianess is the right one for WebAssembly.
+ let mut untrusted_flags = MemFlags::new();
+ untrusted_flags.set_endianness(Endianness::Little);
+
Self {
sp_offset: 0,
asm: Assembler::new(shared_flags.clone(), isa_flags.clone()),
flags: isa_flags,
shared_flags,
- ptr_size: ptr_type_from_ptr_size(ptr_size.size()).into(),
+ ptr_size: ptr_type.into(),
+ trusted_flags: MemFlags::trusted(),
+ untrusted_flags,
}
}
@@ -1149,6 +1191,70 @@ impl MacroAssembler {
}
}
+ /// A common implemenation for zero-extend stack loads.
+ fn load_impl(&mut self, src: Address, dst: Reg, size: OperandSize, flags: MemFlags)
+ where
+ M: Masm,
+ {
+ if dst.is_int() {
+ let access_bits = size.num_bits() as u16;
+
+ let ext_mode = match access_bits {
+ 8 => Some(ExtMode::BQ),
+ 16 => Some(ExtMode::WQ),
+ 32 => Some(ExtMode::LQ),
+ _ => None,
+ };
+
+ self.asm.movzx_mr(&src, dst, ext_mode, flags);
+ } else {
+ self.asm.xmm_mov_mr(&src, dst, size, flags);
+ }
+ }
+
+ /// A common implemenation for stack stores.
+ fn store_impl(&mut self, src: RegImm, dst: Address, size: OperandSize, flags: MemFlags) {
+ let scratch = ::ABI::scratch_reg();
+ let float_scratch = ::ABI::float_scratch_reg();
+ match src {
+ RegImm::Imm(imm) => match imm {
+ I::I32(v) => self.asm.mov_im(v as i32, &dst, size, flags),
+ I::I64(v) => match v.try_into() {
+ Ok(v) => self.asm.mov_im(v, &dst, size, flags),
+ Err(_) => {
+ // If the immediate doesn't sign extend, use a scratch
+ // register.
+ self.asm.mov_ir(v, scratch, size);
+ self.asm.mov_rm(scratch, &dst, size, flags);
+ }
+ },
+ I::F32(v) => {
+ let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
+ // Always trusted, since we are loading the constant from
+ // the constant pool.
+ self.asm
+ .xmm_mov_mr(&addr, float_scratch, size, MemFlags::trusted());
+ self.asm.xmm_mov_rm(float_scratch, &dst, size, flags);
+ }
+ I::F64(v) => {
+ let addr = self.asm.add_constant(v.to_le_bytes().as_slice());
+ // Similar to above, always trusted since we are loading the
+ // constant from the constant pool.
+ self.asm
+ .xmm_mov_mr(&addr, float_scratch, size, MemFlags::trusted());
+ self.asm.xmm_mov_rm(float_scratch, &dst, size, flags);
+ }
+ },
+ RegImm::Reg(reg) => {
+ if reg.is_int() {
+ self.asm.mov_rm(reg, &dst, size, flags);
+ } else {
+ self.asm.xmm_mov_rm(reg, &dst, size, flags);
+ }
+ }
+ }
+ }
+
fn handle_invalid_operand_combination(src: impl Into, dst: impl Into) -> T {
panic!(
"Invalid operand combination; src={:?}, dst={:?}",
diff --git a/winch/codegen/src/isa/x64/mod.rs b/winch/codegen/src/isa/x64/mod.rs
index e14baae00c6d..59ecc31e2314 100644
--- a/winch/codegen/src/isa/x64/mod.rs
+++ b/winch/codegen/src/isa/x64/mod.rs
@@ -107,7 +107,7 @@ impl TargetIsa for X64 {
let stack = Stack::new();
let abi_sig = abi::X64ABI::sig(sig, &CallingConvention::Default);
- let env = FuncEnv::new(&vmoffsets, translation, types);
+ let env = FuncEnv::new(&vmoffsets, translation, types, self);
let defined_locals = DefinedLocals::new::(&env, &mut body, validator)?;
let frame = Frame::new::(&abi_sig, &defined_locals)?;
let gpr = RegBitSet::int(
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 3ea75a40e941..0e7339fe3fb7 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1,5 +1,5 @@
use crate::abi::{self, align_to, LocalSlot};
-use crate::codegen::{ptr_type_from_ptr_size, CodeGenContext, HeapData, TableData};
+use crate::codegen::{CodeGenContext, HeapData, TableData};
use crate::isa::reg::Reg;
use cranelift_codegen::{ir::LibCall, Final, MachBufferFinalized, MachLabel};
use std::{fmt::Debug, ops::Range};
@@ -127,7 +127,7 @@ pub(crate) enum ShiftKind {
Rotr,
}
-/// Kinds of extends in WebAssembly. The [`masm`] implementation for each ISA
+/// Kinds of extends in WebAssembly. Each MacroAssembler implementation
/// is responsible for emitting the correct sequence of instructions when
/// lowering to machine code.
pub(crate) enum ExtendKind {
@@ -150,6 +150,10 @@ pub(crate) enum ExtendKind {
/// Operand size, in bits.
#[derive(Copy, Debug, Clone, Eq, PartialEq)]
pub(crate) enum OperandSize {
+ /// 8 bits.
+ S8,
+ /// 16 bits.
+ S16,
/// 32 bits.
S32,
/// 64 bits.
@@ -160,8 +164,10 @@ pub(crate) enum OperandSize {
impl OperandSize {
/// The number of bits in the operand.
- pub fn num_bits(&self) -> i32 {
+ pub fn num_bits(&self) -> u8 {
match self {
+ OperandSize::S8 => 8,
+ OperandSize::S16 => 16,
OperandSize::S32 => 32,
OperandSize::S64 => 64,
OperandSize::S128 => 128,
@@ -171,6 +177,8 @@ impl OperandSize {
/// The number of bytes in the operand.
pub fn bytes(&self) -> u32 {
match self {
+ Self::S8 => 1,
+ Self::S16 => 2,
Self::S32 => 4,
Self::S64 => 8,
Self::S128 => 16,
@@ -180,6 +188,8 @@ impl OperandSize {
/// The binary logarithm of the number of bits in the operand.
pub fn log2(&self) -> u8 {
match self {
+ OperandSize::S8 => 3,
+ OperandSize::S16 => 4,
OperandSize::S32 => 5,
OperandSize::S64 => 6,
OperandSize::S128 => 7,
@@ -421,9 +431,39 @@ pub(crate) trait MacroAssembler {
/// Perform a stack store.
fn store(&mut self, src: RegImm, dst: Self::Address, size: OperandSize);
- /// Perform a stack load.
+ /// Alias for `MacroAssembler::store` with the operand size corresponding
+ /// to the pointer size of the target.
+ fn store_ptr(&mut self, src: Reg, dst: Self::Address);
+
+ /// Perform a WebAssembly store.
+ /// A WebAssebly store introduces several additional invariants compared to
+ /// [Self::store], more precisely, it can implicitly trap, in certain
+ /// circumstances, even if explicit bounds checks are elided, in that sense,
+ /// we consider this type of load as untrusted. It can also differ with
+ /// regards to the endianess depending on the target ISA. For this reason,
+ /// [Self::wasm_store], should be explicitly used when emitting WebAssembly
+ /// stores.
+ fn wasm_store(&mut self, src: Reg, dst: Self::Address, size: OperandSize);
+
+ /// Perform a zero-extended stack load.
fn load(&mut self, src: Self::Address, dst: Reg, size: OperandSize);
+ /// Perform a WebAssembly load.
+ /// A WebAssebly load introduces several additional invariants compared to
+ /// [Self::load], more precisely, it can implicitly trap, in certain
+ /// circumstances, even if explicit bounds checks are elided, in that sense,
+ /// we consider this type of load as untrusted. It can also differ with
+ /// regards to the endianess depending on the target ISA. For this reason,
+ /// [Self::wasm_load], should be explicitly used when emitting WebAssembly
+ /// loads.
+ fn wasm_load(
+ &mut self,
+ src: Self::Address,
+ dst: Reg,
+ size: OperandSize,
+ kind: Option,
+ );
+
/// Alias for `MacroAssembler::load` with the operand size corresponding
/// to the pointer size of the target.
fn load_ptr(&mut self, src: Self::Address, dst: Reg);
@@ -431,10 +471,6 @@ pub(crate) trait MacroAssembler {
/// Loads the effective address into destination.
fn load_addr(&mut self, _src: Self::Address, _dst: Reg, _size: OperandSize);
- /// Alias for `MacroAssembler::store` with the operand size corresponding
- /// to the pointer size of the target.
- fn store_ptr(&mut self, src: Reg, dst: Self::Address);
-
/// Pop a value from the machine stack into the given register.
fn pop(&mut self, dst: Reg, size: OperandSize);
@@ -456,25 +492,20 @@ pub(crate) trait MacroAssembler {
let mut remaining = bytes;
let word_bytes = ::word_bytes();
let scratch = ::scratch_reg();
- let ptr_size: OperandSize = ptr_type_from_ptr_size(word_bytes as u8).into();
let mut dst_offs = dst.as_u32() - bytes;
let mut src_offs = src.as_u32() - bytes;
+ let word_bytes = word_bytes as u32;
while remaining >= word_bytes {
remaining -= word_bytes;
dst_offs += word_bytes;
src_offs += word_bytes;
- self.load(
- self.address_from_sp(SPOffset::from_u32(src_offs)),
- scratch,
- ptr_size,
- );
- self.store(
+ self.load_ptr(self.address_from_sp(SPOffset::from_u32(src_offs)), scratch);
+ self.store_ptr(
scratch.into(),
self.address_from_sp(SPOffset::from_u32(dst_offs)),
- ptr_size,
);
}
@@ -501,6 +532,10 @@ pub(crate) trait MacroAssembler {
/// Perform add operation.
fn add(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize);
+ /// Perform a checked unsigned integer addition, emitting the provided trap
+ /// if the addition overflows.
+ fn checked_uadd(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize, trap: TrapCode);
+
/// Perform subtraction operation.
fn sub(&mut self, dst: Reg, lhs: Reg, rhs: RegImm, size: OperandSize);
@@ -676,7 +711,7 @@ pub(crate) trait MacroAssembler {
/// into word-sized slots. Then it unrolls a series of store
/// instructions, effectively assigning zero to each slot.
fn zero_mem_range(&mut self, mem: &Range) {
- let word_size = ::word_bytes();
+ let word_size = ::word_bytes() as u32;
if mem.is_empty() {
return;
}
@@ -748,6 +783,9 @@ pub(crate) trait MacroAssembler {
/// Emit an unreachable code trap.
fn unreachable(&mut self);
+ /// Emit an unconditional trap.
+ fn trap(&mut self, code: TrapCode);
+
/// Traps if the condition code is met.
fn trapif(&mut self, cc: IntCmpKind, code: TrapCode);
diff --git a/winch/codegen/src/trampoline.rs b/winch/codegen/src/trampoline.rs
index f02f1cedacb2..515373415bad 100644
--- a/winch/codegen/src/trampoline.rs
+++ b/winch/codegen/src/trampoline.rs
@@ -145,11 +145,8 @@ where
// Move the val ptr back into the scratch register so we can
// load the return values.
let val_ptr_offset = offsets[2];
- self.masm.load(
- self.masm.address_from_sp(val_ptr_offset),
- self.scratch_reg,
- OperandSize::S64,
- );
+ self.masm
+ .load_ptr(self.masm.address_from_sp(val_ptr_offset), self.scratch_reg);
self.store_results_to_array(&wasm_sig, ret_area.as_ref());
@@ -180,7 +177,8 @@ where
}
_ => unreachable!(),
};
- self.masm.load(addr, self.alloc_scratch_reg, (*ty).into());
+ let size: OperandSize = (*ty).into();
+ self.masm.load(addr, self.alloc_scratch_reg, size);
self.masm.store(
self.alloc_scratch_reg.into(),
self.masm.address_at_reg(self.scratch_reg, value_offset),
@@ -294,12 +292,14 @@ where
match caller_sig.params.unwrap_results_area_operand() {
ABIOperand::Reg { ty, .. } => {
let addr = self.masm.address_from_sp(*caller_retptr_offset.unwrap());
- self.masm.load(addr, self.scratch_reg, (*ty).into());
+ let size: OperandSize = (*ty).into();
+ self.masm.load(addr, self.scratch_reg, size);
self.scratch_reg
}
ABIOperand::Stack { ty, offset, .. } => {
+ let size: OperandSize = (*ty).into();
let addr = self.masm.address_at_reg(fp, arg_base + offset);
- self.masm.load(addr, self.scratch_reg, (*ty).into());
+ self.masm.load(addr, self.scratch_reg, size);
self.scratch_reg
}
}
@@ -311,10 +311,11 @@ where
match (callee_operand, caller_operand) {
(ABIOperand::Reg { ty, .. }, ABIOperand::Stack { offset, .. }) => {
let reg_offset = spill_offsets_iter.next().unwrap();
+ let size: OperandSize = (*ty).into();
self.masm.load(
self.masm.address_from_sp(*reg_offset),
self.alloc_scratch_reg,
- (*ty).into(),
+ size,
);
self.masm.store(
self.alloc_scratch_reg.into(),
@@ -334,8 +335,9 @@ where
let slot_offset = base.as_u32() - *offset;
self.masm.address_from_sp(SPOffset::from_u32(slot_offset))
};
+ let size: OperandSize = (*ty).into();
- self.masm.load(addr, self.alloc_scratch_reg, (*ty).into());
+ self.masm.load(addr, self.alloc_scratch_reg, size);
self.masm.store(
self.alloc_scratch_reg.into(),
self.masm
@@ -354,11 +356,8 @@ where
}
(ABIOperand::Reg { ty, .. }, ABIOperand::Reg { reg: dst, .. }) => {
let spill_offset = spill_offsets_iter.next().unwrap();
- self.masm.load(
- self.masm.address_from_sp(*spill_offset),
- (*dst).into(),
- (*ty).into(),
- );
+ self.masm
+ .load(self.masm.address_from_sp(*spill_offset), *dst, (*ty).into());
}
}
}
@@ -415,7 +414,7 @@ where
let body_offset = self.pointer_size.vmnative_call_host_func_context_func_ref()
+ self.pointer_size.vm_func_ref_native_call();
let callee_addr = masm.address_at_reg(self.alloc_scratch_reg, body_offset.into());
- masm.load(callee_addr, self.scratch_reg, OperandSize::S64);
+ masm.load_ptr(callee_addr, self.scratch_reg);
CalleeKind::Indirect(self.scratch_reg)
});
@@ -461,7 +460,8 @@ where
(ABIOperand::Stack { ty, offset, .. }, ABIOperand::Reg { .. }) => {
let spill_offset = caller_stack_offsets[offset_index];
let addr = masm.address_from_sp(spill_offset);
- masm.load(addr, scratch, (*ty).into());
+ let size: OperandSize = (*ty).into();
+ masm.load(addr, scratch, size);
let arg_addr = masm.address_at_sp(SPOffset::from_u32(*offset));
masm.store(scratch.into(), arg_addr, (*ty).into());
@@ -660,7 +660,7 @@ where
ptr: &impl PtrSize,
) {
let sp = ::sp_reg();
- masm.load(vm_runtime_limits_addr, scratch, OperandSize::S64);
+ masm.load_ptr(vm_runtime_limits_addr, scratch);
let addr = masm.address_at_reg(scratch, ptr.vmruntime_limits_last_wasm_entry_sp().into());
masm.store(sp.into(), addr, OperandSize::S64);
}
@@ -672,7 +672,7 @@ where
alloc_scratch: Reg,
ptr: &impl PtrSize,
) {
- masm.load(vm_runtime_limits_addr, alloc_scratch, OperandSize::S64);
+ masm.load_ptr(vm_runtime_limits_addr, alloc_scratch);
let last_wasm_exit_fp_addr = masm.address_at_reg(
alloc_scratch,
ptr.vmruntime_limits_last_wasm_exit_fp().into(),
@@ -685,13 +685,13 @@ where
// Handle the frame pointer.
let fp = ::fp_reg();
let fp_addr = masm.address_at_reg(fp, 0);
- masm.load(fp_addr, scratch, OperandSize::S64);
+ masm.load_ptr(fp_addr, scratch);
masm.store(scratch.into(), last_wasm_exit_fp_addr, OperandSize::S64);
// Handle the return address.
let ret_addr_offset = ::ret_addr_offset();
let ret_addr = masm.address_at_reg(fp, ret_addr_offset.into());
- masm.load(ret_addr, scratch, OperandSize::S64);
+ masm.load_ptr(ret_addr, scratch);
masm.store(scratch.into(), last_wasm_exit_pc_addr, OperandSize::S64);
}
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index e55e1eb074bd..c2bbf9cd0d61 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -14,8 +14,7 @@ use crate::stack::{TypedReg, Val};
use cranelift_codegen::ir::TrapCode;
use regalloc2::RegClass;
use smallvec::SmallVec;
-use wasmparser::BrTable;
-use wasmparser::{BlockType, Ieee32, Ieee64, VisitOperator};
+use wasmparser::{BlockType, BrTable, Ieee32, Ieee64, MemArg, VisitOperator};
use wasmtime_environ::{
FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TableStyle, TypeIndex, WasmHeapType,
WasmValType, FUNCREF_INIT_BIT,
@@ -208,6 +207,30 @@ macro_rules! def_unsupported {
(emit MemoryFill $($rest:tt)*) => {};
(emit MemorySize $($rest:tt)*) => {};
(emit MemoryGrow $($rest:tt)*) => {};
+ (emit I32Load $($rest:tt)*) => {};
+ (emit I32Load8S $($rest:tt)*) => {};
+ (emit I32Load8U $($rest:tt)*) => {};
+ (emit I32Load16S $($rest:tt)*) => {};
+ (emit I32Load16U $($rest:tt)*) => {};
+ (emit I64Load8S $($rest:tt)*) => {};
+ (emit I64Load8U $($rest:tt)*) => {};
+ (emit I64Load16S $($rest:tt)*) => {};
+ (emit I64Load16U $($rest:tt)*) => {};
+ (emit I64Load32S $($rest:tt)*) => {};
+ (emit I64Load32U $($rest:tt)*) => {};
+ (emit I64Load $($rest:tt)*) => {};
+ (emit I32Store $($rest:tt)*) => {};
+ (emit I32Store $($rest:tt)*) => {};
+ (emit I32Store8 $($rest:tt)*) => {};
+ (emit I32Store16 $($rest:tt)*) => {};
+ (emit I64Store $($rest:tt)*) => {};
+ (emit I64Store8 $($rest:tt)*) => {};
+ (emit I64Store16 $($rest:tt)*) => {};
+ (emit I64Store32 $($rest:tt)*) => {};
+ (emit F32Load $($rest:tt)*) => {};
+ (emit F32Store $($rest:tt)*) => {};
+ (emit F64Load $($rest:tt)*) => {};
+ (emit F64Store $($rest:tt)*) => {};
(emit $unsupported:tt $($rest:tt)*) => {$($rest)*};
}
@@ -1804,6 +1827,123 @@ where
self.context.free_reg(cond);
}
+ fn visit_i32_load(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S32, None);
+ }
+
+ fn visit_i32_load8_s(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(
+ &memarg,
+ WasmValType::I32,
+ OperandSize::S8,
+ Some(ExtendKind::I32Extend8S),
+ );
+ }
+
+ fn visit_i32_load8_u(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S8, None);
+ }
+
+ fn visit_i32_load16_s(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(
+ &memarg,
+ WasmValType::I32,
+ OperandSize::S16,
+ Some(ExtendKind::I32Extend16S),
+ )
+ }
+
+ fn visit_i32_load16_u(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I32, OperandSize::S16, None)
+ }
+
+ fn visit_i32_store(&mut self, memarg: MemArg) {
+ self.emit_wasm_store(&memarg, OperandSize::S32);
+ }
+
+ fn visit_i32_store8(&mut self, memarg: MemArg) {
+ self.emit_wasm_store(&memarg, OperandSize::S8)
+ }
+
+ fn visit_i32_store16(&mut self, memarg: MemArg) {
+ self.emit_wasm_store(&memarg, OperandSize::S16)
+ }
+
+ fn visit_i64_load8_s(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(
+ &memarg,
+ WasmValType::I64,
+ OperandSize::S8,
+ Some(ExtendKind::I64Extend8S),
+ )
+ }
+
+ fn visit_i64_load8_u(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S8, None)
+ }
+
+ fn visit_i64_load16_u(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S16, None)
+ }
+
+ fn visit_i64_load16_s(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(
+ &memarg,
+ WasmValType::I64,
+ OperandSize::S16,
+ Some(ExtendKind::I64Extend16S),
+ )
+ }
+
+ fn visit_i64_load32_u(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S32, None)
+ }
+
+ fn visit_i64_load32_s(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(
+ &memarg,
+ WasmValType::I64,
+ OperandSize::S32,
+ Some(ExtendKind::I64Extend32S),
+ )
+ }
+
+ fn visit_i64_load(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::I64, OperandSize::S64, None)
+ }
+
+ fn visit_i64_store(&mut self, memarg: MemArg) -> Self::Output {
+ self.emit_wasm_store(&memarg, OperandSize::S64)
+ }
+
+ fn visit_i64_store8(&mut self, memarg: MemArg) -> Self::Output {
+ self.emit_wasm_store(&memarg, OperandSize::S8)
+ }
+
+ fn visit_i64_store16(&mut self, memarg: MemArg) -> Self::Output {
+ self.emit_wasm_store(&memarg, OperandSize::S16)
+ }
+
+ fn visit_i64_store32(&mut self, memarg: MemArg) -> Self::Output {
+ self.emit_wasm_store(&memarg, OperandSize::S32)
+ }
+
+ fn visit_f32_load(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::F32, OperandSize::S32, None)
+ }
+
+ fn visit_f32_store(&mut self, memarg: MemArg) {
+ self.emit_wasm_store(&memarg, OperandSize::S32)
+ }
+
+ fn visit_f64_load(&mut self, memarg: MemArg) {
+ self.emit_wasm_load(&memarg, WasmValType::F64, OperandSize::S64, None)
+ }
+
+ fn visit_f64_store(&mut self, memarg: MemArg) {
+ self.emit_wasm_store(&memarg, OperandSize::S64)
+ }
+
wasmparser::for_each_operator!(def_unsupported);
}
@@ -1842,7 +1982,7 @@ impl From for OperandSize {
t => unimplemented!("Support for WasmHeapType: {t}"),
}
}
- ty => unimplemented!("Support for WasmType {ty}"),
+ ty => unimplemented!("Support for WasmValType {ty}"),
}
}
}
diff --git a/winch/filetests/filetests/x64/load/f32.wat b/winch/filetests/filetests/x64/load/f32.wat
new file mode 100644
index 000000000000..62b5815f60f8
--- /dev/null
+++ b/winch/filetests/filetests/x64/load/f32.wat
@@ -0,0 +1,23 @@
+;;! target = "x86_64"
+
+(module
+ (memory (data "\00\00\a0\7f"))
+
+ (func (export "f32.load") (result f32) (f32.load (i32.const 0)))
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f871a000000 ja 0x32
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; b800000000 mov eax, 0
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; f30f1001 movss xmm0, dword ptr [rcx]
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 32: 0f0b ud2
diff --git a/winch/filetests/filetests/x64/load/f64.wat b/winch/filetests/filetests/x64/load/f64.wat
new file mode 100644
index 000000000000..f536325b208c
--- /dev/null
+++ b/winch/filetests/filetests/x64/load/f64.wat
@@ -0,0 +1,22 @@
+;;! target = "x86_64"
+(module
+ (memory (data "\00\00\00\00\00\00\f4\7f"))
+
+ (func (export "f64.load") (result f64) (f64.load (i32.const 0)))
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f871a000000 ja 0x32
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; b800000000 mov eax, 0
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; f20f1001 movsd xmm0, qword ptr [rcx]
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 32: 0f0b ud2
diff --git a/winch/filetests/filetests/x64/load/i32.wat b/winch/filetests/filetests/x64/load/i32.wat
new file mode 100644
index 000000000000..ec2d6d40d1e7
--- /dev/null
+++ b/winch/filetests/filetests/x64/load/i32.wat
@@ -0,0 +1,23 @@
+;;! target = "x86_64"
+(module
+ (memory 1)
+ (func (export "as-br-value") (result i32)
+ (block (result i32) (br 0 (i32.load (i32.const 0))))
+ )
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f8718000000 ja 0x30
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; b800000000 mov eax, 0
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; 8b01 mov eax, dword ptr [rcx]
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 30: 0f0b ud2
diff --git a/winch/filetests/filetests/x64/load/i64.wat b/winch/filetests/filetests/x64/load/i64.wat
new file mode 100644
index 000000000000..2fa7d56b6427
--- /dev/null
+++ b/winch/filetests/filetests/x64/load/i64.wat
@@ -0,0 +1,30 @@
+;;! target = "x86_64"
+(module
+ (memory 1)
+ (func (export "i64_load8_s") (param $i i64) (result i64)
+ (i64.store8 (i32.const 8) (local.get $i))
+ (i64.load8_s (i32.const 8))
+ )
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec10 sub rsp, 0x10
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f8732000000 ja 0x4a
+;; 18: 48897c2408 mov qword ptr [rsp + 8], rdi
+;; 4c893424 mov qword ptr [rsp], r14
+;; 488b442408 mov rax, qword ptr [rsp + 8]
+;; b908000000 mov ecx, 8
+;; 498b5650 mov rdx, qword ptr [r14 + 0x50]
+;; 4801ca add rdx, rcx
+;; 8802 mov byte ptr [rdx], al
+;; b808000000 mov eax, 8
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; 480fbe01 movsx rax, byte ptr [rcx]
+;; 4883c410 add rsp, 0x10
+;; 5d pop rbp
+;; c3 ret
+;; 4a: 0f0b ud2
diff --git a/winch/filetests/filetests/x64/store/f32.wat b/winch/filetests/filetests/x64/store/f32.wat
new file mode 100644
index 000000000000..71bf887e1237
--- /dev/null
+++ b/winch/filetests/filetests/x64/store/f32.wat
@@ -0,0 +1,26 @@
+;;! target = "x86_64"
+
+(module
+ (memory (data "\00\00\a0\7f"))
+ (func (export "f32.store") (f32.store (i32.const 0) (f32.const nan:0x200000)))
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f8722000000 ja 0x3a
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; f30f10051c000000 movss xmm0, dword ptr [rip + 0x1c]
+;; b800000000 mov eax, 0
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; f30f1101 movss dword ptr [rcx], xmm0
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 3a: 0f0b ud2
+;; 3c: 0000 add byte ptr [rax], al
+;; 3e: 0000 add byte ptr [rax], al
+;; 40: 0000 add byte ptr [rax], al
diff --git a/winch/filetests/filetests/x64/store/f64.wat b/winch/filetests/filetests/x64/store/f64.wat
new file mode 100644
index 000000000000..4f591a6c1c66
--- /dev/null
+++ b/winch/filetests/filetests/x64/store/f64.wat
@@ -0,0 +1,30 @@
+;;! target = "x86_64"
+
+(module
+ (memory (data "\00\00\00\00\00\00\f4\7f"))
+ (func (export "f64.store") (f64.store (i32.const 0) (f64.const nan:0x4000000000000)))
+)
+
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f8722000000 ja 0x3a
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; f20f10051c000000 movsd xmm0, qword ptr [rip + 0x1c]
+;; b800000000 mov eax, 0
+;; 498b4e50 mov rcx, qword ptr [r14 + 0x50]
+;; 4801c1 add rcx, rax
+;; f20f1101 movsd qword ptr [rcx], xmm0
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 3a: 0f0b ud2
+;; 3c: 0000 add byte ptr [rax], al
+;; 3e: 0000 add byte ptr [rax], al
+;; 40: 0000 add byte ptr [rax], al
+;; 42: 0000 add byte ptr [rax], al
+;; 44: 0000 add byte ptr [rax], al
+;; 46: f4 hlt
diff --git a/winch/filetests/filetests/x64/store/i32.wat b/winch/filetests/filetests/x64/store/i32.wat
new file mode 100644
index 000000000000..cb120a12478d
--- /dev/null
+++ b/winch/filetests/filetests/x64/store/i32.wat
@@ -0,0 +1,25 @@
+;;! target = "x86_64"
+(module
+ (memory 1)
+
+ (func (export "as-block-value")
+ (block (i32.store (i32.const 0) (i32.const 1)))
+ )
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f871d000000 ja 0x35
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; b801000000 mov eax, 1
+;; b900000000 mov ecx, 0
+;; 498b5650 mov rdx, qword ptr [r14 + 0x50]
+;; 4801ca add rdx, rcx
+;; 8902 mov dword ptr [rdx], eax
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 35: 0f0b ud2
diff --git a/winch/filetests/filetests/x64/store/i64.wat b/winch/filetests/filetests/x64/store/i64.wat
new file mode 100644
index 000000000000..f71efc886a59
--- /dev/null
+++ b/winch/filetests/filetests/x64/store/i64.wat
@@ -0,0 +1,22 @@
+;;! target = "x86_64"
+(module
+ (memory 1)
+ (func (export "as-store-both") (result i32)
+ (block (result i32)
+ (i64.store (br 0 (i32.const 32))) (i32.const -1)
+ )
+ )
+)
+;; 55 push rbp
+;; 4889e5 mov rbp, rsp
+;; 4883ec08 sub rsp, 8
+;; 4d8b5e08 mov r11, qword ptr [r14 + 8]
+;; 4d8b1b mov r11, qword ptr [r11]
+;; 4939e3 cmp r11, rsp
+;; 0f870f000000 ja 0x27
+;; 18: 4c893424 mov qword ptr [rsp], r14
+;; b820000000 mov eax, 0x20
+;; 4883c408 add rsp, 8
+;; 5d pop rbp
+;; c3 ret
+;; 27: 0f0b ud2