diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs index 211b62177dd2..77ae456ac978 100644 --- a/cranelift/codegen/build.rs +++ b/cranelift/codegen/build.rs @@ -15,6 +15,7 @@ // current directory is used to find the sources. use cranelift_codegen_meta as meta; +use cranelift_codegen_meta::isa::Isa; use cranelift_isle::error::Errors; use std::env; @@ -28,7 +29,7 @@ fn main() { let out_dir = env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set"); let target_triple = env::var("TARGET").expect("The TARGET environment variable must be set"); - let mut isas = meta::isa::Isa::all() + let mut isas = Isa::all() .iter() .cloned() .filter(|isa| { @@ -82,7 +83,7 @@ fn main() { } if &std::env::var("SKIP_ISLE").unwrap_or("0".to_string()) != "1" { - if let Err(err) = build_isle(crate_dir, isle_dir) { + if let Err(err) = build_isle(crate_dir, isle_dir, &isas) { eprintln!("Error: {}", err); process::exit(1); } @@ -175,18 +176,24 @@ struct IsleCompilation { fn get_isle_compilations( crate_dir: &std::path::Path, out_dir: &std::path::Path, + isas: &[Isa], ) -> Result { let cur_dir = std::env::current_dir()?; // Preludes. let clif_lower_isle = out_dir.join("clif_lower.isle"); let clif_opt_isle = out_dir.join("clif_opt.isle"); + let clif_legalize_isle = out_dir.join("clif_legalize.isle"); let prelude_isle = make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude.isle")); let prelude_opt_isle = make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude_opt.isle")); let prelude_lower_isle = make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude_lower.isle")); + let legalize_shared_isle = make_isle_source_path_relative( + &cur_dir, + crate_dir.join("src/legalizer/legalize_shared.isle"), + ); // Directory for mid-end optimizations. let src_opts = make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("opts")); @@ -213,82 +220,135 @@ fn get_isle_compilations( // N.B.: add any new compilation outputs to // `scripts/force-rebuild-isle.sh` if they do not fit the pattern // `cranelift/codegen/src/isa/*/lower/isle/generated_code.rs`! - Ok(IsleCompilations { - items: vec![ - // The mid-end optimization rules. - IsleCompilation { - output: out_dir.join("isle_opt.rs"), - inputs: vec![ - prelude_isle.clone(), - prelude_opt_isle, - src_opts.join("arithmetic.isle"), - src_opts.join("bitops.isle"), - src_opts.join("cprop.isle"), - src_opts.join("extends.isle"), - src_opts.join("icmp.isle"), - src_opts.join("remat.isle"), - src_opts.join("selects.isle"), - src_opts.join("shifts.isle"), - src_opts.join("vector.isle"), - ], - untracked_inputs: vec![clif_opt_isle], - }, - // The x86-64 instruction selector. - IsleCompilation { - output: out_dir.join("isle_x64.rs"), - inputs: vec![ - prelude_isle.clone(), - prelude_lower_isle.clone(), - src_isa_x64.join("inst.isle"), - src_isa_x64.join("lower.isle"), - ], - untracked_inputs: vec![clif_lower_isle.clone()], - }, - // The aarch64 instruction selector. - IsleCompilation { - output: out_dir.join("isle_aarch64.rs"), - inputs: vec![ - prelude_isle.clone(), - prelude_lower_isle.clone(), - src_isa_aarch64.join("inst.isle"), - src_isa_aarch64.join("inst_neon.isle"), - src_isa_aarch64.join("lower.isle"), - src_isa_aarch64.join("lower_dynamic_neon.isle"), - ], - untracked_inputs: vec![clif_lower_isle.clone()], - }, - // The s390x instruction selector. - IsleCompilation { - output: out_dir.join("isle_s390x.rs"), - inputs: vec![ - prelude_isle.clone(), - prelude_lower_isle.clone(), - src_isa_s390x.join("inst.isle"), - src_isa_s390x.join("lower.isle"), - ], - untracked_inputs: vec![clif_lower_isle.clone()], - }, - // The risc-v instruction selector. - IsleCompilation { - output: out_dir.join("isle_riscv64.rs"), - inputs: vec![ - prelude_isle.clone(), - prelude_lower_isle.clone(), - src_isa_risc_v.join("inst.isle"), - src_isa_risc_v.join("inst_vector.isle"), - src_isa_risc_v.join("lower.isle"), - ], - untracked_inputs: vec![clif_lower_isle.clone()], - }, - ], - }) + let mut items = vec![ + // The mid-end optimization rules. + IsleCompilation { + output: out_dir.join("isle_opt.rs"), + inputs: vec![ + prelude_isle.clone(), + prelude_opt_isle, + src_opts.join("arithmetic.isle"), + src_opts.join("bitops.isle"), + src_opts.join("cprop.isle"), + src_opts.join("extends.isle"), + src_opts.join("icmp.isle"), + src_opts.join("remat.isle"), + src_opts.join("selects.isle"), + src_opts.join("shifts.isle"), + src_opts.join("vector.isle"), + ], + untracked_inputs: vec![clif_opt_isle], + }, + ]; + + if isas.contains(&Isa::X86) { + // The x86-64 instruction selector. + items.push(IsleCompilation { + output: out_dir.join("isle_x64.rs"), + inputs: vec![ + prelude_isle.clone(), + prelude_lower_isle.clone(), + src_isa_x64.join("inst.isle"), + src_isa_x64.join("lower.isle"), + ], + untracked_inputs: vec![clif_lower_isle.clone()], + }); + // The x86-64 legalizer. + items.push(IsleCompilation { + output: out_dir.join("legalize_x64.rs"), + inputs: vec![ + prelude_isle.clone(), + legalize_shared_isle.clone(), + src_isa_x64.join("legalize.isle"), + ], + untracked_inputs: vec![clif_legalize_isle.clone()], + }); + } + + if isas.contains(&Isa::Arm64) { + // The aarch64 instruction selector. + items.push(IsleCompilation { + output: out_dir.join("isle_aarch64.rs"), + inputs: vec![ + prelude_isle.clone(), + prelude_lower_isle.clone(), + src_isa_aarch64.join("inst.isle"), + src_isa_aarch64.join("inst_neon.isle"), + src_isa_aarch64.join("lower.isle"), + src_isa_aarch64.join("lower_dynamic_neon.isle"), + ], + untracked_inputs: vec![clif_lower_isle.clone()], + }); + // The aarch64 legalizer. + items.push(IsleCompilation { + output: out_dir.join("legalize_aarch64.rs"), + inputs: vec![ + prelude_isle.clone(), + legalize_shared_isle.clone(), + // NB: aarch64-specific legalization here when it exists + ], + untracked_inputs: vec![clif_legalize_isle.clone()], + }); + } + + if isas.contains(&Isa::S390x) { + // The s390x instruction selector. + items.push(IsleCompilation { + output: out_dir.join("isle_s390x.rs"), + inputs: vec![ + prelude_isle.clone(), + prelude_lower_isle.clone(), + src_isa_s390x.join("inst.isle"), + src_isa_s390x.join("lower.isle"), + ], + untracked_inputs: vec![clif_lower_isle.clone()], + }); + // The s390x legalizer. + items.push(IsleCompilation { + output: out_dir.join("legalize_s390x.rs"), + inputs: vec![ + prelude_isle.clone(), + legalize_shared_isle.clone(), + // NB: s390x-specific legalization here when it exists + ], + untracked_inputs: vec![clif_legalize_isle.clone()], + }); + } + + if isas.contains(&Isa::Riscv64) { + // The risc-v instruction selector. + items.push(IsleCompilation { + output: out_dir.join("isle_riscv64.rs"), + inputs: vec![ + prelude_isle.clone(), + prelude_lower_isle.clone(), + src_isa_risc_v.join("inst.isle"), + src_isa_risc_v.join("inst_vector.isle"), + src_isa_risc_v.join("lower.isle"), + ], + untracked_inputs: vec![clif_lower_isle.clone()], + }); + // The risc-v legalizer. + items.push(IsleCompilation { + output: out_dir.join("legalize_riscv64.rs"), + inputs: vec![ + prelude_isle.clone(), + legalize_shared_isle.clone(), + src_isa_risc_v.join("legalize.isle"), + ], + untracked_inputs: vec![clif_legalize_isle.clone()], + }); + } + + Ok(IsleCompilations { items }) } fn build_isle( crate_dir: &std::path::Path, isle_dir: &std::path::Path, + isas: &[Isa], ) -> Result<(), Box> { - let isle_compilations = get_isle_compilations(crate_dir, isle_dir)?; + let isle_compilations = get_isle_compilations(crate_dir, isle_dir, isas)?; let mut had_error = false; for compilation in &isle_compilations.items { diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs index e43ebf79f1fe..e7f7019dc07e 100644 --- a/cranelift/codegen/meta/src/gen_inst.rs +++ b/cranelift/codegen/meta/src/gen_inst.rs @@ -1,18 +1,16 @@ //! Generate instruction data (including opcodes, formats, builders, etc.). -use std::fmt; -use std::rc::Rc; - -use cranelift_codegen_shared::constant_hash; use crate::cdsl::camel_case; use crate::cdsl::formats::InstructionFormat; use crate::cdsl::instructions::{AllInstructions, Instruction}; use crate::cdsl::operands::Operand; use crate::cdsl::typevar::{TypeSet, TypeVar}; - use crate::error; use crate::srcgen::{Formatter, Match}; use crate::unique_table::{UniqueSeqTable, UniqueTable}; +use cranelift_codegen_shared::constant_hash; +use std::fmt::{self, Write}; +use std::rc::Rc; // TypeSet indexes are encoded in 8 bits, with `0xff` reserved. const TYPESET_LIMIT: usize = 0xff; @@ -1218,6 +1216,24 @@ enum IsleTarget { Lower, /// Generating code for CLIF to CLIF optimizations. Opt, + /// Code for CLIF-to-CLIF legalization. + Legalize, +} + +impl IsleTarget { + fn ret_ty(&self) -> &'static str { + match self { + IsleTarget::Lower => "Inst", + IsleTarget::Opt => "Value", + IsleTarget::Legalize => "Inst", + } + } + fn ignore_value_lists(&self) -> bool { + matches!(self, IsleTarget::Opt) + } + fn match_type(&self) -> bool { + matches!(self, IsleTarget::Opt | IsleTarget::Legalize) + } } fn gen_common_isle( @@ -1227,7 +1243,6 @@ fn gen_common_isle( isle_target: IsleTarget, ) { use std::collections::{BTreeMap, BTreeSet}; - use std::fmt::Write; use crate::cdsl::formats::FormatField; @@ -1415,12 +1430,9 @@ fn gen_common_isle( ";;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;;", ); fmt.empty_line(); - let ret_ty = match isle_target { - IsleTarget::Lower => "Inst", - IsleTarget::Opt => "Value", - }; + let ret_ty = isle_target.ret_ty(); for inst in instructions { - if isle_target == IsleTarget::Opt + if isle_target.ignore_value_lists() && (inst.format.has_value_list || inst.value_results.len() != 1) { continue; @@ -1430,9 +1442,10 @@ fn gen_common_isle( fmt, "(decl {} ({}{}) {})", inst.name, - match isle_target { - IsleTarget::Lower => "", - IsleTarget::Opt => "Type ", + if isle_target.match_type() { + "Type " + } else { + "" }, inst.operands_in .iter() @@ -1454,10 +1467,7 @@ fn gen_common_isle( fmt, "({} {}{})", inst.name, - match isle_target { - IsleTarget::Lower => "", - IsleTarget::Opt => "ty ", - }, + if isle_target.match_type() { "ty " } else { "" }, inst.operands_in .iter() .map(|o| { o.name }) @@ -1467,10 +1477,7 @@ fn gen_common_isle( let mut s = format!( "(inst_data{} (InstructionData.{} (Opcode.{})", - match isle_target { - IsleTarget::Lower => "", - IsleTarget::Opt => " ty", - }, + if isle_target.match_type() { " ty" } else { "" }, inst.format.name, inst.camel_name ); @@ -1570,120 +1577,28 @@ fn gen_common_isle( }); fmt.line(")"); - // Generate a constructor if this is the mid-end prelude. - if isle_target == IsleTarget::Opt { - fmtln!( - fmt, - "(rule ({} ty {})", - inst.name, - inst.operands_in - .iter() - .map(|o| o.name) - .collect::>() - .join(" ") - ); - fmt.indent(|fmt| { - let mut s = format!( - "(make_inst ty (InstructionData.{} (Opcode.{})", - inst.format.name, inst.camel_name - ); - - // Handle values. Note that we skip generating - // constructors for any instructions with variadic - // value lists. This is fine for the mid-end because - // in practice only calls and branches (for branch - // args) use this functionality, and neither can - // really be optimized or rewritten in the mid-end - // (currently). - // - // As a consequence, we only have to handle the - // one-`Value` case, in which the `Value` is directly - // in the `InstructionData`, and the multiple-`Value` - // case, in which the `Value`s are in a - // statically-sized array (e.g. `[Value; 2]` for a - // binary op). - assert!(!inst.format.has_value_list); - if inst.format.num_value_operands == 1 { - write!( - &mut s, - " {}", - inst.operands_in.iter().find(|o| o.is_value()).unwrap().name - ) - .unwrap(); - } else if inst.format.num_value_operands > 1 { - // As above, get all bindings together, and pass - // to a sub-term; here we use a constructor to - // build the value array. - let values = inst - .operands_in - .iter() - .filter(|o| o.is_value()) - .map(|o| o.name) - .collect::>(); - assert_eq!(values.len(), inst.format.num_value_operands); - let values = values.join(" "); - write!( - &mut s, - " (value_array_{}_ctor {})", - inst.format.num_value_operands, values - ) - .unwrap(); - } - - if inst.format.num_block_operands > 0 { - let blocks: Vec<_> = inst - .operands_in - .iter() - .filter(|o| o.kind.is_block()) - .map(|o| o.name) - .collect(); - if inst.format.num_block_operands == 1 { - write!(&mut s, " {}", blocks.first().unwrap(),).unwrap(); - } else { - write!( - &mut s, - " (block_array_{} {})", - inst.format.num_block_operands, - blocks.join(" ") - ) - .unwrap(); - } - } - - // Immediates (non-value args). - for o in inst - .operands_in - .iter() - .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block()) - { - write!(&mut s, " {}", o.name).unwrap(); - } - s.push_str("))"); - fmt.line(&s); - }); - fmt.line(")"); + match isle_target { + // Lowering doens't generate new CLIF nodes. + IsleTarget::Lower => {} + + // Optimizations generate new CLIF nodes so make a new constructor + // which delegates to a `make_inst` helper. Note that this + // constructor has the same signature as the extractor above, so + // don't give it a prefix. + IsleTarget::Opt => gen_insn_constructor(fmt, inst, "make_inst"), + + // During legalization new instructions are either inserted just + // before the current instruction with `ins_*` or they're replacing + // the current instruction with `replace_*`. + IsleTarget::Legalize => { + gen_insn_constructor(fmt, inst, "ins"); + } } fmt.empty_line(); } } -fn gen_opt_isle( - formats: &[Rc], - instructions: &AllInstructions, - fmt: &mut Formatter, -) { - gen_common_isle(formats, instructions, fmt, IsleTarget::Opt); -} - -fn gen_lower_isle( - formats: &[Rc], - instructions: &AllInstructions, - fmt: &mut Formatter, -) { - gen_common_isle(formats, instructions, fmt, IsleTarget::Lower); -} - /// Generate an `enum` immediate in ISLE. fn gen_isle_enum(name: &str, mut variants: Vec<&str>, fmt: &mut Formatter) { variants.sort(); @@ -1704,6 +1619,99 @@ fn gen_isle_enum(name: &str, mut variants: Vec<&str>, fmt: &mut Formatter) { fmt.empty_line(); } +fn gen_insn_constructor(fmt: &mut Formatter, inst: &Instruction, common_ctor: &str) { + if inst.format.has_value_list { + return; + } + let input_args = inst + .operands_in + .iter() + .map(|o| o.name) + .collect::>() + .join(" "); + fmtln!(fmt, "(rule ({} ty {input_args})", inst.name); + fmt.indent(|fmt| { + let mut s = format!( + "({common_ctor} ty (InstructionData.{} (Opcode.{})", + inst.format.name, inst.camel_name + ); + + // Handle values. Note that we skip generating + // constructors for any instructions with variadic + // value lists. This is fine for the mid-end because + // in practice only calls and branches (for branch + // args) use this functionality, and neither can + // really be optimized or rewritten in the mid-end + // (currently). + // + // As a consequence, we only have to handle the + // one-`Value` case, in which the `Value` is directly + // in the `InstructionData`, and the multiple-`Value` + // case, in which the `Value`s are in a + // statically-sized array (e.g. `[Value; 2]` for a + // binary op). + assert!(!inst.format.has_value_list); + if inst.format.num_value_operands == 1 { + write!( + &mut s, + " {}", + inst.operands_in.iter().find(|o| o.is_value()).unwrap().name + ) + .unwrap(); + } else if inst.format.num_value_operands > 1 { + // As above, get all bindings together, and pass + // to a sub-term; here we use a constructor to + // build the value array. + let values = inst + .operands_in + .iter() + .filter(|o| o.is_value()) + .map(|o| o.name) + .collect::>(); + assert_eq!(values.len(), inst.format.num_value_operands); + let values = values.join(" "); + write!( + &mut s, + " (value_array_{}_ctor {})", + inst.format.num_value_operands, values + ) + .unwrap(); + } + + if inst.format.num_block_operands > 0 { + let blocks: Vec<_> = inst + .operands_in + .iter() + .filter(|o| o.kind.is_block()) + .map(|o| o.name) + .collect(); + if inst.format.num_block_operands == 1 { + write!(&mut s, " {}", blocks.first().unwrap(),).unwrap(); + } else { + write!( + &mut s, + " (block_array_{} {})", + inst.format.num_block_operands, + blocks.join(" ") + ) + .unwrap(); + } + } + + // Immediates (non-value args). + for o in inst + .operands_in + .iter() + .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block()) + { + write!(&mut s, " {}", o.name).unwrap(); + } + s.push_str("))"); + fmt.line(&s); + }); + fmt.line(")"); +} + /// Generate a Builder trait with methods for all instructions. fn gen_builder( instructions: &AllInstructions, @@ -1746,10 +1754,6 @@ fn gen_builder( pub(crate) fn generate( formats: &[Rc], all_inst: &AllInstructions, - opcode_filename: &str, - inst_builder_filename: &str, - isle_opt_filename: &str, - isle_lower_filename: &str, out_dir: &str, isle_dir: &str, ) -> Result<(), error::Error> { @@ -1763,22 +1767,27 @@ pub(crate) fn generate( gen_opcodes(all_inst, &mut fmt); fmt.empty_line(); gen_type_constraints(all_inst, &mut fmt); - fmt.update_file(opcode_filename, out_dir)?; + fmt.update_file("opcodes.rs", out_dir)?; // ISLE DSL: mid-end ("opt") generated bindings. let mut fmt = Formatter::new(); - gen_opt_isle(&formats, all_inst, &mut fmt); - fmt.update_file(isle_opt_filename, isle_dir)?; + gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Opt); + fmt.update_file("clif_opt.isle", isle_dir)?; // ISLE DSL: lowering generated bindings. let mut fmt = Formatter::new(); - gen_lower_isle(&formats, all_inst, &mut fmt); - fmt.update_file(isle_lower_filename, isle_dir)?; + gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Lower); + fmt.update_file("clif_lower.isle", isle_dir)?; + + // ISLE DSL: legalizing CLIF to something backends can handle. + let mut fmt = Formatter::new(); + gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Legalize); + fmt.update_file("clif_legalize.isle", isle_dir)?; // Instruction builder. let mut fmt = Formatter::new(); gen_builder(all_inst, &formats, &mut fmt); - fmt.update_file(inst_builder_filename, out_dir)?; + fmt.update_file("inst_builder.rs", out_dir)?; Ok(()) } diff --git a/cranelift/codegen/meta/src/lib.rs b/cranelift/codegen/meta/src/lib.rs index 689d3508b06c..84ff52f94074 100644 --- a/cranelift/codegen/meta/src/lib.rs +++ b/cranelift/codegen/meta/src/lib.rs @@ -37,10 +37,6 @@ pub fn generate(isas: &[isa::Isa], out_dir: &str, isle_dir: &str) -> Result<(), gen_inst::generate( &shared_defs.all_formats, &shared_defs.all_instructions, - "opcodes.rs", - "inst_builder.rs", - "clif_opt.isle", - "clif_lower.isle", out_dir, isle_dir, )?; diff --git a/cranelift/codegen/meta/src/shared/immediates.rs b/cranelift/codegen/meta/src/shared/immediates.rs index 5584b5564bb9..1ca7ea96bf5f 100644 --- a/cranelift/codegen/meta/src/shared/immediates.rs +++ b/cranelift/codegen/meta/src/shared/immediates.rs @@ -182,6 +182,7 @@ impl Immediates { trapcode_values.insert("int_ovf", "IntegerOverflow"); trapcode_values.insert("int_divz", "IntegerDivisionByZero"); trapcode_values.insert("bad_toint", "BadConversionToInteger"); + trapcode_values.insert("table_oob", "TableOutOfBounds"); new_enum( "code", "ir::TrapCode", diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index ca3d5fb73628..7128984bb599 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -16,10 +16,8 @@ use crate::egraph::EgraphPass; use crate::flowgraph::ControlFlowGraph; use crate::ir::Function; use crate::isa::TargetIsa; -use crate::legalizer::simple_legalize; use crate::loop_analysis::LoopAnalysis; use crate::machinst::{CompiledCode, CompiledCodeStencil}; -use crate::nan_canonicalization::do_nan_canonicalization; use crate::remove_constant_phis::do_remove_constant_phis; use crate::result::{CodegenResult, CompileResult}; use crate::settings::{FlagsOrIsa, OptLevel}; @@ -169,9 +167,6 @@ impl Context { ); self.compute_cfg(); - if isa.flags().enable_nan_canonicalization() { - self.canonicalize_nans(isa)?; - } self.legalize(isa)?; @@ -276,12 +271,6 @@ impl Context { Ok(()) } - /// Perform NaN canonicalizing rewrites on the function. - pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> { - do_nan_canonicalization(&mut self.func); - self.verify_if(isa) - } - /// Run the legalizer for `isa` on the function. pub fn legalize(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> { // Legalization invalidates the domtree and loop_analysis by mutating the CFG. @@ -290,7 +279,7 @@ impl Context { self.loop_analysis.clear(); // Run some specific legalizations only. - simple_legalize(&mut self.func, &mut self.cfg, isa); + isa.legalize_function(&mut self.func, &mut self.cfg); self.verify_if(isa) } diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs index feb7466c3f37..f806fea55c04 100644 --- a/cranelift/codegen/src/ir/dfg.rs +++ b/cranelift/codegen/src/ir/dfg.rs @@ -459,6 +459,13 @@ impl DataFlowGraph { debug_assert_ne!(ty, types::INVALID); self.values[dest] = ValueData::Alias { ty, original }.into(); + + if let Some(fact) = &self.facts[dest] { + if self.facts[original].is_none() { + let fact = fact.clone(); + self.facts[original] = Some(fact); + } + } } self.clear_results(dest_inst); diff --git a/cranelift/codegen/src/isa/aarch64/legalize.rs b/cranelift/codegen/src/isa/aarch64/legalize.rs new file mode 100644 index 000000000000..203c9eebd942 --- /dev/null +++ b/cranelift/codegen/src/isa/aarch64/legalize.rs @@ -0,0 +1,27 @@ +use crate::flowgraph; +use crate::ir; +use crate::isa::aarch64::AArch64Backend; +use crate::legalizer::isle; + +// Used by ISLE +use crate::cursor::CursorPosition; +use crate::ir::condcodes::*; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::*; +use crate::machinst::isle::*; + +#[allow(dead_code, unused_variables, unreachable_patterns)] +mod generated { + include!(concat!(env!("ISLE_DIR"), "/legalize_aarch64.rs")); +} + +pub fn run(isa: &AArch64Backend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) { + crate::legalizer::isle::run(isa, func, cfg, |cx, i| { + generated::constructor_legalize(cx, i) + }) +} + +impl generated::Context for isle::LegalizeContext<'_, AArch64Backend> { + crate::isle_common_legalizer_methods!(); +} diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs index 59541d9d1517..e40ac7ad667a 100644 --- a/cranelift/codegen/src/isa/aarch64/mod.rs +++ b/cranelift/codegen/src/isa/aarch64/mod.rs @@ -1,6 +1,7 @@ //! ARM 64-bit Instruction Set Architecture. use crate::dominator_tree::DominatorTree; +use crate::flowgraph; use crate::ir::{Function, Type}; use crate::isa::aarch64::settings as aarch64_settings; #[cfg(feature = "unwind")] @@ -20,6 +21,7 @@ use target_lexicon::{Aarch64Architecture, Architecture, OperatingSystem, Triple} // New backend: mod abi; pub mod inst; +mod legalize; mod lower; mod pcc; pub mod settings; @@ -119,6 +121,10 @@ impl TargetIsa for AArch64Backend { 16 } + fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) { + legalize::run(self, func, cfg); + } + #[cfg(feature = "unwind")] fn emit_unwind_info( &self, diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 8e68e7aaba27..1c625264ca2a 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -264,6 +264,12 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// Get the ISA-dependent maximum vector register size, in bytes. fn dynamic_vector_bytes(&self, dynamic_ty: ir::Type) -> u32; + /// Edit the IR of `func` to remove any CLIF operations which aren't + /// supported by this backend. + /// + /// Note that this is done before optimizations are executed. + fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph); + /// Compile the given function. fn compile_function( &self, diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle index 18ddcfc540d5..18a823002217 100644 --- a/cranelift/codegen/src/isa/riscv64/inst.isle +++ b/cranelift/codegen/src/isa/riscv64/inst.isle @@ -1876,7 +1876,7 @@ (decl pure partial imm12_add (Imm12 i32) Imm12) (extern constructor imm12_add imm12_add) -(decl imm12_and (Imm12 u64) Imm12) +(decl pure imm12_and (Imm12 u64) Imm12) (extern constructor imm12_and imm12_and) ;; Imm12 Extractors @@ -1905,6 +1905,9 @@ (decl pure imm12_is_zero () Imm12) (extern extractor imm12_is_zero imm12_is_zero) +(decl u64_from_imm12 (u64) Imm12) +(extern extractor u64_from_imm12 u64_from_imm12) + ;; Imm20 ;; Extractor that matches if a Imm20 is zero diff --git a/cranelift/codegen/src/isa/riscv64/legalize.isle b/cranelift/codegen/src/isa/riscv64/legalize.isle new file mode 100644 index 000000000000..df9a92e8c074 --- /dev/null +++ b/cranelift/codegen/src/isa/riscv64/legalize.isle @@ -0,0 +1,7 @@ +;; ========= `{u,s}extend` ==================================================== + +(rule (legalize (uextend $I128 x)) + (uextend_i128 x)) + +(rule (legalize (sextend $I128 x)) + (sextend_i128 x)) diff --git a/cranelift/codegen/src/isa/riscv64/legalize.rs b/cranelift/codegen/src/isa/riscv64/legalize.rs new file mode 100644 index 000000000000..d7b2df23d2d4 --- /dev/null +++ b/cranelift/codegen/src/isa/riscv64/legalize.rs @@ -0,0 +1,27 @@ +use crate::flowgraph; +use crate::ir; +use crate::isa::riscv64::Riscv64Backend; +use crate::legalizer::isle; + +// Used by ISLE +use crate::cursor::CursorPosition; +use crate::ir::condcodes::*; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::*; +use crate::machinst::isle::*; + +#[allow(dead_code, unused_variables, unreachable_patterns)] +mod generated { + include!(concat!(env!("ISLE_DIR"), "/legalize_riscv64.rs")); +} + +pub fn run(isa: &Riscv64Backend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) { + crate::legalizer::isle::run(isa, func, cfg, |cx, i| { + generated::constructor_legalize(cx, i) + }) +} + +impl generated::Context for isle::LegalizeContext<'_, Riscv64Backend> { + crate::isle_common_legalizer_methods!(); +} diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle index 21dec2bb628d..1fcdde79d14f 100644 --- a/cranelift/codegen/src/isa/riscv64/lower.isle +++ b/cranelift/codegen/src/isa/riscv64/lower.isle @@ -1109,17 +1109,14 @@ ;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (rule 0 (lower (has_type (fits_in_64 _) (uextend val))) (zext val)) -(rule 1 (lower (has_type $I128 (uextend val))) - (value_regs (zext val) (imm $I64 0))) ;;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (rule 0 (lower (has_type (fits_in_64 _) (sextend val @ (value_type in_ty)))) (sext val)) -(rule 1 (lower (has_type $I128 (sextend val @ (value_type in_ty)))) - (let ((lo XReg (sext val))) - (value_regs lo (rv_srai lo (imm12_const 63))))) ;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1210,18 +1207,31 @@ (rv_sll x (value_regs_get y 0))) ;; If the shift amount is known. We can mask it and encode it in the instruction. -(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y))))) - (rv_slliw x (imm12_and y (ty_shift_mask ty)))) +(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x y))) + (if-let shift (const_shift ty y)) + (rv_slliw x shift)) -;; We technically don't need to mask the shift amount here. The instruction -;; does the right thing. But it's neater when pretty printing it. -(rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y))))) - (rv_slli x (imm12_and y (ty_shift_mask ty)))) +(rule 3 (lower (has_type ty @ $I64 (ishl x y))) + (if-let shift (const_shift $I64 y)) + (rv_slli x shift)) ;; With `Zba` we have a shift that zero extends the LHS argument. -(rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y))))) +(rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) y))) (if-let $true (has_zba)) - (rv_slliuw x y)) + (if-let shift (const_shift $I64 y)) + (rv_slliuw x shift)) + +;; Helper for extracting a constant shift value from the `Value` input. +;; +;; This will return an appropriate `Imm12` for a shift instruction which is +;; masked to the width of the input `Type`. +(decl pure partial const_shift (Type Value) Imm12) +(rule 0 (const_shift ty (imm12_from_value n)) + (imm12_and n (ty_shift_mask ty))) +(rule 1 (const_shift ty (iconcat lower _)) + (const_shift ty lower)) +(rule 1 (const_shift ty (uextend lower)) + (const_shift ty lower)) ;; I128 cases (rule 4 (lower (has_type $I128 (ishl x y))) @@ -1250,8 +1260,15 @@ (rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ishl x y))) (rv_vsll_vx x (value_regs_get y 0) (unmasked) ty)) -(rule 6 (lower (has_type (ty_vec_fits_in_register ty) (ishl x (maybe_uextend (uimm5_from_value y))))) - (rv_vsll_vi x y (unmasked) ty)) +(rule 6 (lower (has_type (ty_vec_fits_in_register ty) (ishl x y))) + (if-let uimm5 (const_vector_shift ty y)) + (rv_vsll_vi x uimm5 (unmasked) ty)) + +(decl pure partial const_vector_shift (Type Value) UImm5) +(rule (const_vector_shift ty val) + (if-let (u64_from_imm12 imm12) (const_shift (lane_type ty) val)) + (if-let (uimm5_from_u64 uimm5) imm12) + uimm5) ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1270,14 +1287,17 @@ (rv_srl x (value_regs_get y 0))) ;; When the RHS is known we can just encode it in the instruction. -(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y))))) - (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty)))) +(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y))) + (if-let shift (const_shift ty y)) + (rv_srliw (zext x) shift)) -(rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y))))) - (rv_srliw x y)) +(rule 3 (lower (has_type $I32 (ushr x y))) + (if-let shift (const_shift $I32 y)) + (rv_srliw x shift)) -(rule 3 (lower (has_type $I64 (ushr x (maybe_uextend (imm12_from_value y))))) - (rv_srli x y)) +(rule 3 (lower (has_type $I64 (ushr x y))) + (if-let shift (const_shift $I64 y)) + (rv_srli x shift)) (rule 3 (lower (has_type $I128 (ushr x y))) (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) @@ -1305,8 +1325,9 @@ (rule 4 (lower (has_type (ty_vec_fits_in_register ty) (ushr x y))) (rv_vsrl_vx x (value_regs_get y 0) (unmasked) ty)) -(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ushr x (maybe_uextend (uimm5_from_value y))))) - (rv_vsrl_vi x y (unmasked) ty)) +(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ushr x y))) + (if-let uimm5 (const_vector_shift ty y)) + (rv_vsrl_vi x uimm5 (unmasked) ty)) ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1325,14 +1346,17 @@ (rv_sra x (value_regs_get y 0))) ;; When the RHS is known we can just encode it in the instruction. -(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y))))) - (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty)))) +(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y))) + (if-let shift (const_shift ty y)) + (rv_sraiw (sext x) shift)) -(rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y))))) - (rv_sraiw x y)) +(rule 3 (lower (has_type $I32 (sshr x y))) + (if-let shift (const_shift $I32 y)) + (rv_sraiw x shift)) -(rule 3 (lower (has_type $I64 (sshr x (maybe_uextend (imm12_from_value y))))) - (rv_srai x y)) +(rule 3 (lower (has_type $I64 (sshr x y))) + (if-let shift (const_shift $I64 y)) + (rv_srai x shift)) (rule 3 (lower (has_type $I128 (sshr x y))) (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0))) @@ -1365,9 +1389,9 @@ (rule 4 (lower (has_type (ty_vec_fits_in_register ty) (sshr x y))) (rv_vsra_vx x (value_regs_get y 0) (unmasked) ty)) -(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (sshr x (maybe_uextend (uimm5_from_value y))))) - (rv_vsra_vi x y (unmasked) ty)) - +(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (sshr x y))) + (if-let uimm5 (const_vector_shift ty y)) + (rv_vsra_vi x uimm5 (unmasked) ty)) ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs index ab5c6ddf1299..892a3d21f867 100644 --- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs +++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs @@ -190,6 +190,10 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend> Imm12::from_i16(imm.as_i16() & (x as i16)) } + fn u64_from_imm12(&mut self, imm: Imm12) -> Option { + u64::try_from(imm.as_i16()).ok() + } + fn i64_generate_imm(&mut self, imm: i64) -> Option<(Imm20, Imm12)> { MInst::generate_imm(imm as u64) } diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs index 6d727bbc6267..13b3b125762a 100644 --- a/cranelift/codegen/src/isa/riscv64/mod.rs +++ b/cranelift/codegen/src/isa/riscv64/mod.rs @@ -1,6 +1,7 @@ //! risc-v 64-bit Instruction Set Architecture. use crate::dominator_tree::DominatorTree; +use crate::flowgraph; use crate::ir::{Function, Type}; use crate::isa::riscv64::settings as riscv_settings; use crate::isa::{Builder as IsaBuilder, FunctionAlignment, OwnedTargetIsa, TargetIsa}; @@ -17,6 +18,7 @@ use cranelift_control::ControlPlane; use target_lexicon::{Architecture, Triple}; mod abi; pub(crate) mod inst; +mod legalize; mod lower; mod settings; #[cfg(feature = "unwind")] @@ -101,6 +103,10 @@ impl TargetIsa for Riscv64Backend { 16 } + fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) { + legalize::run(self, func, cfg); + } + fn triple(&self) -> &Triple { &self.triple } diff --git a/cranelift/codegen/src/isa/s390x/legalize.rs b/cranelift/codegen/src/isa/s390x/legalize.rs new file mode 100644 index 000000000000..36c541ebd97a --- /dev/null +++ b/cranelift/codegen/src/isa/s390x/legalize.rs @@ -0,0 +1,27 @@ +use crate::flowgraph; +use crate::ir; +use crate::isa::s390x::S390xBackend; +use crate::legalizer::isle; + +// Used by ISLE +use crate::cursor::CursorPosition; +use crate::ir::condcodes::*; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::*; +use crate::machinst::isle::*; + +#[allow(dead_code, unused_variables, unreachable_patterns)] +mod generated { + include!(concat!(env!("ISLE_DIR"), "/legalize_s390x.rs")); +} + +pub fn run(isa: &S390xBackend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) { + crate::legalizer::isle::run(isa, func, cfg, |cx, i| { + generated::constructor_legalize(cx, i) + }) +} + +impl generated::Context for isle::LegalizeContext<'_, S390xBackend> { + crate::isle_common_legalizer_methods!(); +} diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs index 5840450dd7ed..b18efe64b7c2 100644 --- a/cranelift/codegen/src/isa/s390x/mod.rs +++ b/cranelift/codegen/src/isa/s390x/mod.rs @@ -1,6 +1,7 @@ //! IBM Z 64-bit Instruction Set Architecture. use crate::dominator_tree::DominatorTree; +use crate::flowgraph; use crate::ir::{Function, Type}; use crate::isa::s390x::settings as s390x_settings; #[cfg(feature = "unwind")] @@ -20,6 +21,7 @@ use target_lexicon::{Architecture, Triple}; // New backend: mod abi; pub(crate) mod inst; +mod legalize; mod lower; mod settings; @@ -115,6 +117,10 @@ impl TargetIsa for S390xBackend { 16 } + fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) { + legalize::run(self, func, cfg); + } + #[cfg(feature = "unwind")] fn emit_unwind_info( &self, diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index e55867ab45ab..784ec11623b7 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -454,13 +454,6 @@ (dst WritableXmm) (src2_size OperandSize)) - ;; Converts an unsigned int64 to a float32/float64. - (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8 - (src Gpr) - (dst WritableXmm) - (tmp_gpr1 WritableGpr) - (tmp_gpr2 WritableGpr)) - ;; Converts a scalar xmm to a signed int32/int64. (CvtFloatToSintSeq (dst_size OperandSize) (src_size OperandSize) @@ -2095,15 +2088,6 @@ (_ Unit (emit (MInst.CvtIntToFloatVex op src1 src2 dst size)))) dst)) -(decl cvt_u64_to_float_seq (Type Gpr) Xmm) -(rule (cvt_u64_to_float_seq ty src) - (let ((size OperandSize (raw_operand_size_of_type ty)) - (dst WritableXmm (temp_writable_xmm)) - (tmp_gpr1 WritableGpr (temp_writable_gpr)) - (tmp_gpr2 WritableGpr (temp_writable_gpr)) - (_ Unit (emit (MInst.CvtUint64ToFloatSeq size src dst tmp_gpr1 tmp_gpr2)))) - dst)) - (decl cvt_float_to_uint_seq (Type Value bool) Gpr) (rule (cvt_float_to_uint_seq out_ty src @ (value_type src_ty) is_saturating) (let ((out_size OperandSize (raw_operand_size_of_type out_ty)) diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 6bd65154eb23..7a53f6eb4b07 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -15,33 +15,6 @@ use crate::isa::x64::inst::*; use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel, Reg, Writable}; use core::convert::TryInto; -/// A small helper to generate a signed conversion instruction. -fn emit_signed_cvt( - sink: &mut MachBuffer, - info: &EmitInfo, - state: &mut EmitState, - // Required to be RealRegs. - src: Reg, - dst: Writable, - to_f64: bool, -) { - // Handle an unsigned int, which is the "easy" case: a signed conversion will do the - // right thing. - let op = if to_f64 { - SseOpcode::Cvtsi2sd - } else { - SseOpcode::Cvtsi2ss - }; - Inst::CvtIntToFloat { - op, - dst: Writable::from_reg(Xmm::new(dst.to_reg()).unwrap()), - src1: Xmm::new(dst.to_reg()).unwrap(), - src2: GprMem::new(RegMem::reg(src)).unwrap(), - src2_size: OperandSize::Size64, - } - .emit(&[], sink, info, state); -} - /// Emits a one way conditional jump if CC is set (true). fn one_way_jmp(sink: &mut MachBuffer, cc: CC, label: MachLabel) { let cond_start = sink.cur_offset(); @@ -3300,127 +3273,6 @@ pub(crate) fn emit( .encode(sink); } - Inst::CvtUint64ToFloatSeq { - dst_size, - src, - dst, - tmp_gpr1, - tmp_gpr2, - } => { - let src = allocs.next(src.to_reg()); - let dst = allocs.next(dst.to_reg().to_reg()); - let tmp_gpr1 = allocs.next(tmp_gpr1.to_reg().to_reg()); - let tmp_gpr2 = allocs.next(tmp_gpr2.to_reg().to_reg()); - - // Note: this sequence is specific to 64-bit mode; a 32-bit mode would require a - // different sequence. - // - // Emit the following sequence: - // - // cmp 0, %src - // jl handle_negative - // - // ;; handle positive, which can't overflow - // cvtsi2sd/cvtsi2ss %src, %dst - // j done - // - // ;; handle negative: see below for an explanation of what it's doing. - // handle_negative: - // mov %src, %tmp_gpr1 - // shr $1, %tmp_gpr1 - // mov %src, %tmp_gpr2 - // and $1, %tmp_gpr2 - // or %tmp_gpr1, %tmp_gpr2 - // cvtsi2sd/cvtsi2ss %tmp_gpr2, %dst - // addsd/addss %dst, %dst - // - // done: - - assert_ne!(src, tmp_gpr1); - assert_ne!(src, tmp_gpr2); - assert_ne!(tmp_gpr1, tmp_gpr2); - - let handle_negative = sink.get_label(); - let done = sink.get_label(); - - // If x seen as a signed int64 is not negative, a signed-conversion will do the right - // thing. - // TODO use tst src, src here. - let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src); - inst.emit(&[], sink, info, state); - - one_way_jmp(sink, CC::L, handle_negative); - - // Handle a positive int64, which is the "easy" case: a signed conversion will do the - // right thing. - emit_signed_cvt( - sink, - info, - state, - src, - Writable::from_reg(dst), - *dst_size == OperandSize::Size64, - ); - - let inst = Inst::jmp_known(done); - inst.emit(&[], sink, info, state); - - sink.bind_label(handle_negative, state.ctrl_plane_mut()); - - // Divide x by two to get it in range for the signed conversion, keep the LSB, and - // scale it back up on the FP side. - let inst = Inst::gen_move(Writable::from_reg(tmp_gpr1), src, types::I64); - inst.emit(&[], sink, info, state); - - // tmp_gpr1 := src >> 1 - let inst = Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Imm8Gpr::new(Imm8Reg::Imm8 { imm: 1 }).unwrap(), - tmp_gpr1, - Writable::from_reg(tmp_gpr1), - ); - inst.emit(&[], sink, info, state); - - let inst = Inst::gen_move(Writable::from_reg(tmp_gpr2), src, types::I64); - inst.emit(&[], sink, info, state); - - let inst = Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::imm(1), - Writable::from_reg(tmp_gpr2), - ); - inst.emit(&[], sink, info, state); - - let inst = Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp_gpr1), - Writable::from_reg(tmp_gpr2), - ); - inst.emit(&[], sink, info, state); - - emit_signed_cvt( - sink, - info, - state, - tmp_gpr2, - Writable::from_reg(dst), - *dst_size == OperandSize::Size64, - ); - - let add_op = if *dst_size == OperandSize::Size64 { - SseOpcode::Addsd - } else { - SseOpcode::Addss - }; - let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst), Writable::from_reg(dst)); - inst.emit(&[], sink, info, state); - - sink.bind_label(done, state.ctrl_plane_mut()); - } - Inst::CvtFloatToSintSeq { src_size, dst_size, diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 19bda166c19f..dca711e3a11b 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -108,7 +108,6 @@ impl Inst { | Inst::CmpRmiR { .. } | Inst::CvtFloatToSintSeq { .. } | Inst::CvtFloatToUintSeq { .. } - | Inst::CvtUint64ToFloatSeq { .. } | Inst::Div { .. } | Inst::Div8 { .. } | Inst::Fence { .. } @@ -474,6 +473,7 @@ impl Inst { } } + #[cfg(test)] pub(crate) fn shift_r( size: OperandSize, kind: ShiftKind, @@ -1326,29 +1326,6 @@ impl PrettyPrint for Inst { format!("{op} {src1}, {src2}, {dst}") } - Inst::CvtUint64ToFloatSeq { - src, - dst, - dst_size, - tmp_gpr1, - tmp_gpr2, - .. - } => { - let src = pretty_print_reg(src.to_reg(), 8, allocs); - let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs); - let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8, allocs); - let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8, allocs); - let op = ljustify(format!( - "u64_to_{}_seq", - if *dst_size == OperandSize::Size64 { - "f64" - } else { - "f32" - } - )); - format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}") - } - Inst::CvtFloatToSintSeq { src, dst, @@ -2208,18 +2185,6 @@ fn x64_get_operands VReg>(inst: &Inst, collector: &mut OperandCol collector.reg_use(src1.to_reg()); src2.get_operands(collector); } - Inst::CvtUint64ToFloatSeq { - src, - dst, - tmp_gpr1, - tmp_gpr2, - .. - } => { - collector.reg_use(src.to_reg()); - collector.reg_early_def(dst.to_writable_reg()); - collector.reg_early_def(tmp_gpr1.to_writable_reg()); - collector.reg_early_def(tmp_gpr2.to_writable_reg()); - } Inst::CvtFloatToSintSeq { src, dst, diff --git a/cranelift/codegen/src/isa/x64/legalize.isle b/cranelift/codegen/src/isa/x64/legalize.isle new file mode 100644 index 000000000000..5091001351fa --- /dev/null +++ b/cranelift/codegen/src/isa/x64/legalize.isle @@ -0,0 +1,8 @@ +;; ========= `fcvt_from_uint` ================================================= + +(rule (legalize (fcvt_from_uint ty v @ (value_type $I64))) + (x64_replace_fcvt_from_u64 ty v)) + +(decl x64_replace_fcvt_from_u64 (Type Value) CursorPosition) +(extern constructor x64_replace_fcvt_from_u64 x64_replace_fcvt_from_u64) + diff --git a/cranelift/codegen/src/isa/x64/legalize.rs b/cranelift/codegen/src/isa/x64/legalize.rs new file mode 100644 index 000000000000..84342f3f23f2 --- /dev/null +++ b/cranelift/codegen/src/isa/x64/legalize.rs @@ -0,0 +1,111 @@ +use crate::flowgraph; +use crate::ir; +use crate::isa::x64::X64Backend; +use crate::legalizer::isle; + +// Used by ISLE +use crate::cursor::{Cursor, CursorPosition}; +use crate::ir::condcodes::*; +use crate::ir::immediates::*; +use crate::ir::types::*; +use crate::ir::*; +use crate::machinst::isle::*; + +#[allow(dead_code, unused_variables, unreachable_patterns)] +mod generated { + include!(concat!(env!("ISLE_DIR"), "/legalize_x64.rs")); +} + +pub(crate) fn run( + isa: &X64Backend, + func: &mut ir::Function, + cfg: &mut flowgraph::ControlFlowGraph, +) { + crate::legalizer::isle::run(isa, func, cfg, |cx, i| { + generated::constructor_legalize(cx, i) + }) +} + +impl generated::Context for isle::LegalizeContext<'_, X64Backend> { + crate::isle_common_legalizer_methods!(); + + /// On x64 there's no native instruction for converting an unsigned 64-bit + /// number into a float, only a signed 64-bit number. To handle this + /// `fcvt_from_uint` instructions are replaced with this sequence below. + /// + /// If the input value interpreted as a signed number is positive then + /// `fcvt_from_sint` can be used because it will produce the same result. + /// + /// If the input value interpreted as a signed number is negative then + /// the rough idea is to shift it to the right by one to halve it, use + /// `fcvt_from_sint` on the result, and then double the result with an + /// `fadd` afterwards. + /// + /// TODO: the previous incarnation of this code has a "see below for an + /// explanation" comment for what it was doing and I don't know why the + /// lower bit is preserved in the shifted-right temporary. I presume that + /// has to do with something about how a u64 is so large than f64 doesn't + /// have the precision to hold the lowest bit so this trick produces the + /// right result "because of float precision weirdness". If someone knows + /// better it'd be good to fill out this comment. + fn x64_replace_fcvt_from_u64(&mut self, ty: Type, uint: Value) -> CursorPosition { + let inst = self.replace.unwrap(); + let old_block = self + .pos + .func + .layout + .inst_block(inst) + .expect("inst not in layout"); + let block_is_nonnegative = self.pos.func.dfg.make_block(); + let block_is_negative = self.pos.func.dfg.make_block(); + let block_resume = self.pos.func.dfg.make_block(); + + // Test whether the input is negative when interpreted as a signed + // number, and proceed from here. + let zero = self.pos.ins().iconst(I64, 0); + let is_negative = self.pos.ins().icmp(IntCC::SignedLessThan, uint, zero); + self.pos.ins().brif( + is_negative, + block_is_negative, + &[], + block_is_nonnegative, + &[], + ); + + // If the input value was negative "do the trick". More-or-less use a + // signed conversion on a half-value and then double it afterwards. + self.pos.insert_block(block_is_negative); + let one = self.pos.ins().iconst(I64, 1); + let ushr = self.pos.ins().ushr(uint, one); + let band = self.pos.ins().band(uint, one); + let bor = self.pos.ins().bor(ushr, band); + let fcvt = self.pos.ins().fcvt_from_sint(ty, bor); + let fadd = self.pos.ins().fadd(fcvt, fcvt); + self.pos.ins().jump(block_resume, &[fadd]); + + // If the input value was positive then `fcvt_from_sint` can be used + // since it'd produce the same result. + self.pos.insert_block(block_is_nonnegative); + let fcvt = self.pos.ins().fcvt_from_sint(ty, uint); + self.pos.ins().jump(block_resume, &[fcvt]); + + // At the join point remove the previous `fcvt_from_uint` instruction + // and change its result to alias our new result which is a parameter to + // this block. + self.pos.insert_block(block_resume); + let result = self.pos.func.dfg.append_block_param(block_resume, ty); + let prev_result = self.pos.func.dfg.first_result(inst); + self.pos.func.dfg.clear_results(inst); + self.pos.func.dfg.change_to_alias(prev_result, result); + self.pos.func.layout.remove_inst(inst); + + // Finally update the CFG. + self.cfg.recompute_block(self.pos.func, old_block); + self.cfg + .recompute_block(self.pos.func, block_is_nonnegative); + self.cfg.recompute_block(self.pos.func, block_is_negative); + self.cfg.recompute_block(self.pos.func, block_resume); + + CursorPosition::Before(block_resume) + } +} diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index e0d17fe627a4..84ce6041a737 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -3409,8 +3409,7 @@ (rule 1 (lower (has_type $F64 (fcvt_from_uint val @ (value_type (fits_in_32 (ty_int ty)))))) (x64_cvtsi2sd $I64 (xmm_zero $F64X2) (extend_to_gpr val $I64 (ExtendKind.Zero)))) -(rule (lower (has_type ty (fcvt_from_uint val @ (value_type $I64)))) - (cvt_u64_to_float_seq ty val)) +;; NB: fcvt_from_uint for 64-bit values is handled in `legalize.isle.` ;; Algorithm uses unpcklps to help create a float that is equivalent ;; 0x1.0p52 + double(src). 0x1.0p52 is unique because at this exponent diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs index 38a9c668c9d3..d574d804c730 100644 --- a/cranelift/codegen/src/isa/x64/mod.rs +++ b/cranelift/codegen/src/isa/x64/mod.rs @@ -4,6 +4,7 @@ pub use self::inst::{args, CallInfo, EmitInfo, EmitState, Inst}; use super::{OwnedTargetIsa, TargetIsa}; use crate::dominator_tree::DominatorTree; +use crate::flowgraph; use crate::ir::{types, Function, Type}; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv; @@ -23,6 +24,7 @@ use target_lexicon::Triple; mod abi; pub mod encoding; mod inst; +mod legalize; mod lower; mod pcc; pub mod settings; @@ -191,6 +193,10 @@ impl TargetIsa for X64Backend { fn has_x86_pmaddubsw_lowering(&self) -> bool { self.x64_flags.use_ssse3() } + + fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) { + legalize::run(self, func, cfg); + } } impl fmt::Display for X64Backend { diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs index 97afc57466bd..90d9740f0f38 100644 --- a/cranelift/codegen/src/isa/x64/pcc.rs +++ b/cranelift/codegen/src/isa/x64/pcc.rs @@ -661,18 +661,6 @@ pub(crate) fn check( ensure_no_fact(vcode, dst.to_writable_reg().to_reg()) } - Inst::CvtUint64ToFloatSeq { - dst, - tmp_gpr1, - tmp_gpr2, - .. - } => { - ensure_no_fact(vcode, dst.to_writable_reg().to_reg())?; - ensure_no_fact(vcode, tmp_gpr1.to_writable_reg().to_reg())?; - ensure_no_fact(vcode, tmp_gpr2.to_writable_reg().to_reg())?; - Ok(()) - } - Inst::CvtFloatToSintSeq { dst, tmp_gpr, diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs index 88b6abdb8eff..552f6745ff0a 100644 --- a/cranelift/codegen/src/isle_prelude.rs +++ b/cranelift/codegen/src/isle_prelude.rs @@ -828,6 +828,16 @@ macro_rules! isle_common_prelude_methods { MemFlags::trusted() } + fn mem_flags_notrap(&mut self) -> MemFlags { + let mut mflags = MemFlags::new(); + mflags.set_notrap(); + mflags + } + + fn mem_flags_empty(&mut self) -> MemFlags { + MemFlags::new() + } + #[inline] fn intcc_unsigned(&mut self, x: &IntCC) -> IntCC { x.unsigned() diff --git a/cranelift/codegen/src/legalizer/globalvalue.rs b/cranelift/codegen/src/legalizer/globalvalue.rs deleted file mode 100644 index 9837dd59278e..000000000000 --- a/cranelift/codegen/src/legalizer/globalvalue.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Legalization of global values. -//! -//! This module exports the `expand_global_value` function which transforms a `global_value` -//! instruction into code that depends on the kind of global value referenced. - -use crate::cursor::{Cursor, FuncCursor}; -use crate::ir::{self, pcc::Fact, InstBuilder}; -use crate::isa::TargetIsa; - -/// Expand a `global_value` instruction according to the definition of the global value. -pub fn expand_global_value( - inst: ir::Inst, - func: &mut ir::Function, - isa: &dyn TargetIsa, - global_value: ir::GlobalValue, -) { - crate::trace!( - "expanding global value: {:?}: {}", - inst, - func.dfg.display_inst(inst) - ); - - match func.global_values[global_value] { - ir::GlobalValueData::VMContext => vmctx_addr(global_value, inst, func), - ir::GlobalValueData::IAddImm { - base, - offset, - global_type, - } => iadd_imm_addr(inst, func, base, offset.into(), global_type), - ir::GlobalValueData::Load { - base, - offset, - global_type, - flags, - } => load_addr(inst, func, base, offset, global_type, flags, isa), - ir::GlobalValueData::Symbol { tls, .. } => symbol(inst, func, global_value, isa, tls), - ir::GlobalValueData::DynScaleTargetConst { vector_type } => { - const_vector_scale(inst, func, vector_type, isa) - } - } -} - -fn const_vector_scale(inst: ir::Inst, func: &mut ir::Function, ty: ir::Type, isa: &dyn TargetIsa) { - assert!(ty.bytes() <= 16); - - // Use a minimum of 128-bits for the base type. - let base_bytes = std::cmp::max(ty.bytes(), 16); - let scale = (isa.dynamic_vector_bytes(ty) / base_bytes) as i64; - assert!(scale > 0); - let pos = FuncCursor::new(func).at_inst(inst); - pos.func.dfg.replace(inst).iconst(isa.pointer_type(), scale); -} - -/// Expand a `global_value` instruction for a vmctx global. -fn vmctx_addr(global_value: ir::GlobalValue, inst: ir::Inst, func: &mut ir::Function) { - // Get the value representing the `vmctx` argument. - let vmctx = func - .special_param(ir::ArgumentPurpose::VMContext) - .expect("Missing vmctx parameter"); - - // Replace the `global_value` instruction's value with an alias to the vmctx arg. - let result = func.dfg.first_result(inst); - func.dfg.clear_results(inst); - func.dfg.change_to_alias(result, vmctx); - func.layout.remove_inst(inst); - - // If there was a fact on the GV, then copy it to the vmctx arg - // blockparam def. - if let Some(fact) = &func.global_value_facts[global_value] { - if func.dfg.facts[vmctx].is_none() { - let fact = fact.clone(); - func.dfg.facts[vmctx] = Some(fact); - } - } -} - -/// Expand a `global_value` instruction for an iadd_imm global. -fn iadd_imm_addr( - inst: ir::Inst, - func: &mut ir::Function, - base: ir::GlobalValue, - offset: i64, - global_type: ir::Type, -) { - let mut pos = FuncCursor::new(func).at_inst(inst); - - // Get the value for the lhs. - let lhs = pos.ins().global_value(global_type, base); - if let Some(fact) = &pos.func.global_value_facts[base] { - pos.func.dfg.facts[lhs] = Some(fact.clone()); - } - - // Generate the constant and attach a fact to the constant if - // there is a fact on the base. - let constant = pos.ins().iconst(global_type, offset); - if pos.func.global_value_facts[base].is_some() { - let bits = u16::try_from(global_type.bits()).unwrap(); - let unsigned_offset = offset as u64; // Safety: reinterpret i64 bits as u64. - pos.func.dfg.facts[constant] = Some(Fact::constant(bits, unsigned_offset)); - } - - // Simply replace the `global_value` instruction with an `iadd_imm`, reusing the result value. - pos.func.dfg.replace(inst).iadd(lhs, constant); -} - -/// Expand a `global_value` instruction for a load global. -fn load_addr( - inst: ir::Inst, - func: &mut ir::Function, - base: ir::GlobalValue, - offset: ir::immediates::Offset32, - global_type: ir::Type, - flags: ir::MemFlags, - isa: &dyn TargetIsa, -) { - // We need to load a pointer from the `base` global value, so insert a new `global_value` - // instruction. This depends on the iterative legalization loop. Note that the IR verifier - // detects any cycles in the `load` globals. - let ptr_ty = isa.pointer_type(); - let mut pos = FuncCursor::new(func).at_inst(inst); - pos.use_srcloc(inst); - - // Get the value for the base. - let base_addr = pos.ins().global_value(ptr_ty, base); - if let Some(fact) = &pos.func.global_value_facts[base] { - pos.func.dfg.facts[base_addr] = Some(fact.clone()); - } - - // Perform the load. - pos.func - .dfg - .replace(inst) - .load(global_type, flags, base_addr, offset); -} - -/// Expand a `global_value` instruction for a symbolic name global. -fn symbol( - inst: ir::Inst, - func: &mut ir::Function, - gv: ir::GlobalValue, - isa: &dyn TargetIsa, - tls: bool, -) { - let ptr_ty = isa.pointer_type(); - - if tls { - func.dfg.replace(inst).tls_value(ptr_ty, gv); - } else { - func.dfg.replace(inst).symbol_value(ptr_ty, gv); - } -} diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs new file mode 100644 index 000000000000..1c32f06f3354 --- /dev/null +++ b/cranelift/codegen/src/legalizer/isle.rs @@ -0,0 +1,345 @@ +use crate::cursor::{Cursor, CursorPosition, FuncCursor}; +use crate::flowgraph::ControlFlowGraph; +use crate::ir::{self, Inst, InstBuilder}; +use crate::isa::TargetIsa; +use crate::trace; + +pub fn run( + backend: &T, + func: &mut ir::Function, + cfg: &mut ControlFlowGraph, + constructor_legalize: fn(&mut LegalizeContext<'_, T>, Inst) -> Option, +) where + T: TargetIsa, +{ + let pos = FuncCursor::new(func); + let mut cx = LegalizeContext { + backend, + prev_position: pos.position(), + pos, + cfg, + replace: None, + last_ins: None, + }; + let func_begin = cx.pos.position(); + cx.pos.set_position(func_begin); + while let Some(_block) = cx.pos.next_block() { + cx.prev_position = cx.pos.position(); + while let Some(inst) = cx.pos.next_inst() { + trace!("legalizing {}", cx.pos.func.dfg.display_inst(inst)); + cx.replace = Some(inst); + cx.last_ins = None; + match constructor_legalize(&mut cx, inst) { + Some(pos) => { + trace!("moving to {pos:?}"); + if let Some(ins) = cx.last_ins.take() { + cx.pos.func.dfg.replace_with_aliases(inst, ins); + cx.pos.remove_inst(); + } + cx.pos.set_position(pos); + } + None => { + trace!("fallthrough"); + debug_assert!(cx.last_ins.is_none()); + cx.prev_position = cx.pos.position(); + } + } + } + } +} + +pub struct LegalizeContext<'a, T> { + pub backend: &'a T, + pub pos: FuncCursor<'a>, + pub cfg: &'a mut ControlFlowGraph, + /// Instruction that is currently being legalized by this context and will + /// be replaced if an instruction is inserted. + pub replace: Option, + /// Last-inserted instruction. + /// + /// If this is set then all the values of `replace` will be turned into + /// aliases to the results of this instruction. + pub last_ins: Option, + /// The previous position before this context started legalizing, used to + /// rewind backwards and re-legalize instructions that were just added. + pub prev_position: CursorPosition, +} + +/// Generate common methods for the legalization trait on `LegalizeContext`. +#[macro_export] +macro_rules! isle_common_legalizer_methods { + () => { + crate::isle_common_prelude_methods!(); + + fn inst_data(&mut self, inst: Inst) -> (Type, InstructionData) { + let ty = self.pos.func.dfg.ctrl_typevar(inst); + (ty, self.pos.func.dfg.insts[inst]) + } + + fn gv_data(&mut self, gv: GlobalValue) -> GlobalValueData { + self.pos.func.global_values[gv].clone() + } + + fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst { + let ret = self.pos.ins().build(data.clone(), ty).0; + crate::trace!("ins {}", self.pos.func.dfg.display_inst(ret)); + self.last_ins = Some(ret); + ret + } + + fn value_type(&mut self, val: Value) -> Type { + self.pos.func.dfg.value_type(val) + } + + fn first_result(&mut self, inst: Inst) -> Value { + let results = self.pos.func.dfg.inst_results(inst); + assert_eq!(results.len(), 1,); + results[0] + } + + fn result_type(&mut self, inst: Inst) -> Option { + let results = self.pos.func.dfg.inst_results(inst); + if results.len() == 1 { + Some(self.value_type(results[0])) + } else { + None + } + } + + fn value_list_slice(&mut self, list: ValueList) -> ValueSlice { + (list, 0) + } + + fn value_slice_unwrap(&mut self, slice: ValueSlice) -> Option<(Value, ValueSlice)> { + let (list, off) = slice; + if let Some(val) = list.get(off, &self.pos.func.dfg.value_lists) { + Some((val, (list, off + 1))) + } else { + None + } + } + + fn value_array_2_ctor(&mut self, arg0: Value, arg1: Value) -> ValueArray2 { + [arg0, arg1] + } + + fn value_array_3_ctor(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3 { + [arg0, arg1, arg2] + } + + fn expand_trapz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition { + crate::legalizer::isle::expand_cond_trap( + self.replace.unwrap(), + self.pos.func, + self.cfg, + ir::Opcode::Trapz, + arg, + *cc, + ) + } + + fn expand_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition { + crate::legalizer::isle::expand_cond_trap( + self.replace.unwrap(), + self.pos.func, + self.cfg, + ir::Opcode::Trapnz, + arg, + *cc, + ) + } + + fn expand_resumable_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition { + crate::legalizer::isle::expand_cond_trap( + self.replace.unwrap(), + self.pos.func, + self.cfg, + ir::Opcode::ResumableTrapnz, + arg, + *cc, + ) + } + + fn pointer_type(&mut self) -> ir::Type { + (self.backend as &dyn crate::isa::TargetIsa).pointer_type() + } + + fn const_vector_scale(&mut self, ty: ir::Type) -> u64 { + assert!(ty.bytes() <= 16); + + // Use a minimum of 128-bits for the base type. + let base_bytes = std::cmp::max(ty.bytes(), 16); + (self.backend.dynamic_vector_bytes(ty) / base_bytes).into() + } + + fn update_inst_facts_with_gv(&mut self, gv: GlobalValue, val: Value) { + if let Some(fact) = &self.pos.func.global_value_facts[gv] { + if self.pos.func.dfg.facts[val].is_none() { + let fact = fact.clone(); + self.pos.func.dfg.facts[val] = Some(fact); + } + } + } + + fn update_const_facts(&mut self, ty: Type, imm: Imm64, gv: GlobalValue, constant: Value) { + if self.pos.func.global_value_facts[gv].is_none() { + return; + } + let bits = u16::try_from(ty.bits()).unwrap(); + let unsigned_offset = imm.bits() as u64; // Safety: reinterpret i64 bits as u64. + self.pos.func.dfg.facts[constant] = Some(Fact::constant(bits, unsigned_offset)); + } + + fn vmctx_addr(&mut self, global_value: ir::GlobalValue) -> CursorPosition { + // Get the value representing the `vmctx` argument. + let vmctx = self + .pos + .func + .special_param(ir::ArgumentPurpose::VMContext) + .expect("Missing vmctx parameter"); + let inst = self.replace.unwrap(); + + // Replace the `global_value` instruction's value with an alias to the vmctx arg. + let result = self.pos.func.dfg.first_result(inst); + self.pos.func.dfg.clear_results(inst); + self.pos.func.dfg.change_to_alias(result, vmctx); + self.pos.func.layout.remove_inst(inst); + + // If there was a fact on the GV, then copy it to the vmctx arg + // blockparam def. + if let Some(fact) = &self.pos.func.global_value_facts[global_value] { + if self.pos.func.dfg.facts[vmctx].is_none() { + let fact = fact.clone(); + self.pos.func.dfg.facts[vmctx] = Some(fact); + } + } + + self.prev_position + } + + fn cursor_position_at(&mut self, i: Inst) -> CursorPosition { + CursorPosition::At(i) + } + + fn prev_position(&mut self) -> CursorPosition { + self.prev_position + } + + fn table_bound_gv(&mut self, table: Table) -> GlobalValue { + self.pos.func.tables[table].bound_gv + } + + fn table_base_gv(&mut self, table: Table) -> GlobalValue { + self.pos.func.tables[table].base_gv + } + + fn table_element_size(&mut self, table: Table) -> u64 { + self.pos.func.tables[table].element_size.into() + } + + fn enable_table_access_spectre_mitigation(&mut self) -> bool { + self.backend + .flags() + .enable_table_access_spectre_mitigation() + } + + fn enable_nan_canonicalization(&mut self) -> bool { + self.backend.flags().enable_nan_canonicalization() + } + + fn canon_nan32(&mut self) -> Value { + static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000; + self.pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN)) + } + + fn canon_nan64(&mut self) -> Value { + static CANON_64BIT_NAN: u64 = + 0b0111111111111000000000000000000000000000000000000000000000000000; + self.pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN)) + } + }; +} + +/// Custom expansion for conditional trap instructions. +pub fn expand_cond_trap( + inst: ir::Inst, + func: &mut ir::Function, + cfg: &mut ControlFlowGraph, + opcode: ir::Opcode, + arg: ir::Value, + code: ir::TrapCode, +) -> CursorPosition { + trace!( + "expanding conditional trap: {:?}: {}", + inst, + func.dfg.display_inst(inst) + ); + + // Parse the instruction. + let trapz = match opcode { + ir::Opcode::Trapz => true, + ir::Opcode::Trapnz | ir::Opcode::ResumableTrapnz => false, + _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst)), + }; + + // Split the block after `inst`: + // + // trapnz arg + // .. + // + // Becomes: + // + // brif arg, new_block_trap, new_block_resume + // + // new_block_trap: + // trap + // + // new_block_resume: + // .. + let old_block = func + .layout + .inst_block(inst) + .expect("Instruction not in layout."); + let new_block_trap = func.dfg.make_block(); + let new_block_resume = func.dfg.make_block(); + + // Trapping is a rare event, mark the trapping block as cold. + func.layout.set_cold(new_block_trap); + + // Replace trap instruction by the inverted condition. + if trapz { + func.dfg + .replace(inst) + .brif(arg, new_block_resume, &[], new_block_trap, &[]); + } else { + func.dfg + .replace(inst) + .brif(arg, new_block_trap, &[], new_block_resume, &[]); + } + + // Insert the new label and the unconditional trap terminator. + let mut pos = FuncCursor::new(func).after_inst(inst); + pos.use_srcloc(inst); + pos.insert_block(new_block_trap); + + match opcode { + ir::Opcode::Trapz | ir::Opcode::Trapnz => { + pos.ins().trap(code); + } + ir::Opcode::ResumableTrapnz => { + pos.ins().resumable_trap(code); + pos.ins().jump(new_block_resume, &[]); + } + _ => unreachable!(), + } + + // Insert the new label and resume the execution when the trap fails. + pos.insert_block(new_block_resume); + + // Finally update the CFG. + cfg.recompute_block(pos.func, old_block); + cfg.recompute_block(pos.func, new_block_resume); + cfg.recompute_block(pos.func, new_block_trap); + + CursorPosition::Before(new_block_resume) +} diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle new file mode 100644 index 000000000000..5fdebd9e5d3f --- /dev/null +++ b/cranelift/codegen/src/legalizer/legalize_shared.isle @@ -0,0 +1,459 @@ +;; Legalizing CLIF to CLIF +;; +;; This file is one of the main entry points for performing "legalizations" that +;; transform CLIF to CLIF to simplify the IR for one phase or another. One +;; example of a legalization is replacing "macro instructions" with their +;; components, such as the `stack_store` CLIF instruction with a `stack_addr` +;; plus a `store`. This way backends don't need to implement lowering for +;; all three instructions, only the latter two. +;; +;; Backends also have their own forms of legalization as well. For example +;; instead of natively implementing a `uextend`-to-128-bits backends can choose +;; to legalize the `uextend` instruction in this case to be replaced with an +;; `iconcat`. This can help reduce the number of lowerings that a backend needs +;; while sharing legalization code with other backends. +;; +;; This file is organized as shared support for all backends. This contains +;; the definition of the `legalize` function which is the main entrypoint for +;; transforming instructions. Some required legalizations are defined in this +;; file such as legalizing the CLIF `stack_store` pseudo-instruction. Otherwise +;; platform-neutral helpers are defined in this file that each backend can +;; opt-in to as desired. + +;; ========= Type Definitions ================================================= + +(type Inst (primitive Inst)) +(type CursorPosition (primitive CursorPosition)) +(type ExternalName extern (enum)) + +;; ========= Helper extractors/constructors =================================== + +;; Helper used in generated extractors for each CLIF instruction to extract +;; information from an `Inst`. +(decl inst_data (Type InstructionData) Inst) +(extern extractor infallible inst_data inst_data) + +;; Helper used to perform an `ins()` operation which inserts just before the +;; current instruction being legalized. +;; +;; This is invoked by generated constructors for each CLIF instruction. +(decl ins (Type InstructionData) Inst) +(extern constructor ins ins) + +;; Constructors for value arrays. +(decl value_array_2_ctor (Value Value) ValueArray2) +(extern constructor value_array_2_ctor value_array_2_ctor) +(decl value_array_3_ctor (Value Value Value) ValueArray3) +(extern constructor value_array_3_ctor value_array_3_ctor) +(decl value_list_slice (ValueSlice) ValueList) +(extern extractor infallible value_list_slice value_list_slice) +(decl value_slice_unwrap (Value ValueSlice) ValueSlice) +(extern extractor value_slice_unwrap value_slice_unwrap) +(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList) +(extractor (unwrap_head_value_list_1 head tail) + (value_list_slice (value_slice_unwrap head tail))) + +(decl pure first_result (Inst) Value) +(extern constructor first_result first_result) +(convert Inst Value first_result) + +(decl has_type (Type Inst) Inst) +(extractor (has_type ty inst) (and (result_type ty) inst)) +(decl result_type (Type) Inst) +(extern extractor result_type result_type) + +(decl cursor_position_at (Inst) CursorPosition) +(extern constructor cursor_position_at cursor_position_at) +(convert Inst CursorPosition cursor_position_at) + +;; Returns the position of the cursor just before the current instruction was +;; being legalized. This can be used to re-legalize the sequence of instructions +;; used to replace another instruction. +(decl prev_position () CursorPosition) +(extern constructor prev_position prev_position) + +(decl pointer_type () Type) +(extern constructor pointer_type pointer_type) + +;; Helper to generate an `iconst` value which has type `I64` and the provided +;; value. +(decl ins_i64 (i64) Value) +(rule (ins_i64 val) (iconst $I64 (imm64 (i64_as_u64 val)))) + +;; Optionally "legalize" an instruction by replacing the current instruction or +;; inserting new instructions/blocks. +;; +;; This is used to transform CLIF into something which can be handled by each +;; individual backend. There is both common functionality of legalization for +;; all backends in addition to backend-specific legalization. +;; +;; Note that this function is defined here and while some legalizations are +;; performed in this file there are also backend-specific legalizations +;; throughout each backend as well. +;; +;; The return value of this function is where to resume legalization. By default +;; this is the current instruction which means that legalization will continue +;; after the current instruction. If the inserted code itself needs to be +;; re-legalized then the `(prev_position)` constructor can be used to rerun +;; over the sequence of replaced instructions. +(decl partial legalize (Inst) CursorPosition) + +;; ========= `{u,s}extend` ==================================================== + +;; Helper to replace a `uextend` to i128 with an `iconcat` instruction instead. +(decl uextend_i128 (Value) Inst) +(rule 0 (uextend_i128 x @ (value_type (fits_in_32 _))) + (iconcat $I64 (uextend $I64 x) (ins_i64 0))) +(rule 1 (uextend_i128 x @ (value_type $I64)) + (iconcat $I64 x (ins_i64 0))) + +;; Helper to replace a `sextend` to i128 with an `iconcat` instruction instead. +(decl sextend_i128 (Value) Inst) +(rule 0 (sextend_i128 x @ (value_type (fits_in_32 _))) + (let ((val64 Value (sextend $I64 x))) + (iconcat $I64 val64 (sshr $I64 val64 (ins_i64 63))))) +(rule 1 (sextend_i128 x @ (value_type $I64)) + (iconcat $I64 x (sshr $I64 x (ins_i64 63)))) + +;; ========= `{dynamic_,}stack_{load,store}` ================================== + +;; stack_load(slot, offset) => load(stack_addr(slot, offset), 0) +(rule (legalize (stack_load ty slot offset)) + (let ((addr Value (stack_addr $I64 slot offset))) + (load ty (mem_flags_notrap) addr (i32_to_offset32 0)))) + +;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0) +(rule (legalize (stack_store ty val slot offset)) + (let ((addr Value (stack_addr $I64 slot offset))) + (store ty (mem_flags_notrap) val addr (i32_to_offset32 0)))) + +;; dynamic_stack_load(slot) => load(dynamic_stack_addr(slot), 0) +(rule (legalize (dynamic_stack_load ty slot)) + (let ((addr Value (dynamic_stack_addr $I64 slot))) + (load ty (mem_flags_notrap) addr (i32_to_offset32 0)))) + +;; dynamic_stack_store(val, slot) => store(val, dynamic_stack_addr(slot), 0) +(rule (legalize (dynamic_stack_store ty val slot)) + (let ((addr Value (dynamic_stack_addr $I64 slot))) + (store ty (mem_flags_notrap) val addr (i32_to_offset32 0)))) + +;; ========= `trapz`, `trapnz`, `resumable_trapnz` ============================ + +;; Delegate back to Rust to perform the basic-block manipulation necessary to +;; expand these instructions + +(rule (legalize (trapz _ val code)) (expand_trapz val code)) +(decl expand_trapz (Value TrapCode) CursorPosition) +(extern constructor expand_trapz expand_trapz) + +(rule (legalize (trapnz _ val code)) (expand_trapnz val code)) +(decl expand_trapnz (Value TrapCode) CursorPosition) +(extern constructor expand_trapnz expand_trapnz) + +(rule (legalize (resumable_trapnz _ val code)) (expand_resumable_trapnz val code)) +(decl expand_resumable_trapnz (Value TrapCode) CursorPosition) +(extern constructor expand_resumable_trapnz expand_resumable_trapnz) + +;; ========= `*_imm` instrs =================================================== + +(rule (legalize (band_imm ty val imm)) + (band ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (bor_imm ty val imm)) + (bor ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (bxor_imm ty val imm)) + (bxor ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (ishl_imm ty val imm)) + (ishl ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (sshr_imm ty val imm)) + (sshr ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (ushr_imm ty val imm)) + (ushr ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (rotl_imm ty val imm)) + (rotl ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (rotr_imm ty val imm)) + (rotr ty val (unsigned_const (lane_type ty) imm))) + +(rule (legalize (iadd_imm ty val imm)) + (iadd ty val (signed_const (lane_type ty) imm))) + +;; note the reversed arguments in the `isub` +(rule (legalize (irsub_imm ty val imm)) + (isub ty (signed_const (lane_type ty) imm) val)) + +(rule (legalize (imul_imm ty val imm)) + (imul ty val (signed_const (lane_type ty) imm))) + +(rule (legalize (sdiv_imm ty val imm)) + (sdiv ty val (signed_const (lane_type ty) imm))) + +(rule (legalize (srem_imm ty val imm)) + (srem ty val (signed_const (lane_type ty) imm))) + +(rule (legalize (udiv_imm ty val imm)) + (udiv ty val (signed_const (lane_type ty) imm))) + +(rule (legalize (urem_imm ty val imm)) + (urem ty val (signed_const (lane_type ty) imm))) + +(rule (legalize (icmp_imm ty cc val imm)) + (icmp ty cc val (signed_const (lane_type ty) imm))) + +(decl unsigned_const (Type Imm64) Value) +(rule (unsigned_const $I8 (u64_from_imm64 n)) + (iconst $I8 (imm64 (u64_and n 0xff)))) +(rule (unsigned_const $I16 (u64_from_imm64 n)) + (iconst $I16 (imm64 (u64_and n 0xffff)))) +(rule (unsigned_const $I32 (u64_from_imm64 n)) + (iconst $I32 (imm64 (u64_and n 0xffffffff)))) +(rule (unsigned_const $I64 imm) + (iconst $I64 imm)) +(rule (unsigned_const $I128 imm) + (iconcat $I64 (iconst $I64 imm) (ins_i64 0))) + +(decl signed_const (Type Imm64) Value) +(rule 0 (signed_const (fits_in_64 ty) imm) + (unsigned_const ty imm)) +(rule 1 (signed_const $I128 imm) + (iconcat $I64 + (iconst $I64 imm) + (iconst $I64 (imm64_sshr $I64 imm (imm64 63))))) + +;; ========= `*_not` instrs =================================================== + +;; Legalize the fused bitwise-plus-not instructions into simpler +;; instructions to assist with optimizations. Lowering will +;; pattern match this sequence regardless when architectures +;; support the instruction natively. + +(rule (legalize (band_not ty a b)) + (band ty a (bnot ty b))) + +(rule (legalize (bor_not ty a b)) + (bor ty a (bnot ty b))) + +(rule (legalize (bxor_not ty a b)) + (bxor ty a (bnot ty b))) + +;; ========= `global_value` =================================================== + +(rule (legalize (global_value _ gv @ (gv_vmcontext))) + (vmctx_addr gv)) +(decl vmctx_addr (GlobalValue) CursorPosition) +(extern constructor vmctx_addr vmctx_addr) + +(rule (legalize (global_value _ (gv_load base offset ty flags))) + (let ((base Value (global_value_copy_facts (pointer_type) base)) + (_ Inst (load ty flags base offset))) + ;; re-legalize the previous instrs to legalize the new `global_value` node + (prev_position))) + +(rule (legalize (global_value _ (gv_iadd_imm gv_base offset ty))) + (let ((base Value (global_value_copy_facts ty gv_base)) + (offset Value (iconst_with_fact ty offset gv_base)) + (_ Inst (iadd ty base offset))) + ;; re-legalize the previous instrs to legalize the new `global_value` node + (prev_position))) + +(rule (legalize (global_value _ (gv_const_vector_scale ty))) + (iconst (pointer_type) (imm64 (const_vector_scale ty)))) +(decl const_vector_scale (Type) u64) +(extern constructor const_vector_scale const_vector_scale) + +(rule (legalize (global_value _ gv @ (gv_symbol _ _))) + (symbol_value (pointer_type) gv)) + +(rule (legalize (global_value _ gv @ (gv_tls_symbol _ _))) + (tls_value (pointer_type) gv)) + +(type GlobalValueData extern (enum + (VMContext) + (Load + (base GlobalValue) + (offset Offset32) + (global_type Type) + (flags MemFlags)) + (IAddImm + (base GlobalValue) + (offset Imm64) + (global_type Type)) + (Symbol + (name ExternalName) + (offset Imm64) + (colocated bool) + (tls bool)) + (DynScaleTargetConst + (vector_type Type)) +)) + +;; Convert from `GlobalValue` to `GlobalValueData` +(decl gv_data (GlobalValueData) GlobalValue) +(extern extractor infallible gv_data gv_data) + +;; Helper extractors for each variant of `GlobalValue` +(decl gv_vmcontext () GlobalValue) +(extractor (gv_vmcontext) + (gv_data (GlobalValueData.VMContext))) +(decl gv_load (GlobalValue Offset32 Type MemFlags) GlobalValue) +(extractor (gv_load base offset ty flags) + (gv_data (GlobalValueData.Load base offset ty flags))) +(decl gv_iadd_imm (GlobalValue Offset32 Type) GlobalValue) +(extractor (gv_iadd_imm base offset ty) + (gv_data (GlobalValueData.IAddImm base offset ty))) +(decl gv_symbol (Imm64 bool) GlobalValue) +(extractor (gv_symbol offset colocated) + (gv_data (GlobalValueData.Symbol _ offset colocated $false))) +(decl gv_tls_symbol (Imm64 bool) GlobalValue) +(extractor (gv_tls_symbol offset colocated) + (gv_data (GlobalValueData.Symbol _ offset colocated $true))) +(decl gv_const_vector_scale (Type) GlobalValue) +(extractor (gv_const_vector_scale ty) + (gv_data (GlobalValueData.DynScaleTargetConst ty))) + +;; This is the dual of the `global_value` constructor except that this one +;; additionally copies PCC facts from the `GlobalValue` onto the returned +;; `Inst`. +(decl global_value_copy_facts (Type GlobalValue) Inst) +(rule (global_value_copy_facts ty gv) + (let ((ret Inst (global_value ty gv)) + (_ Unit (update_inst_facts_with_gv gv ret))) + ret)) +(decl iconst_with_fact (Type Imm64 GlobalValue) Inst) +(rule (iconst_with_fact ty imm gv) + (let ((ret Inst (iconst ty imm)) + (_ Unit (update_const_facts ty imm gv ret))) + ret)) +(decl update_inst_facts_with_gv (GlobalValue Value) Unit) +(extern constructor update_inst_facts_with_gv update_inst_facts_with_gv) +(decl update_const_facts (Type Imm64 GlobalValue Value) Unit) +(extern constructor update_const_facts update_const_facts) + +;; ========= `table_addr` ===================================================== + +(rule (legalize (table_addr addr_ty table idx @ (value_type idx_ty) offset)) + (let ( + ;; Start with the bounds check. Trap if `index + 1 > bound`. + ;; `index > bound - 1` is the same as `index >= bound`. + (bound Inst (global_value_copy_facts idx_ty (table_bound_gv table))) + (cmp Inst (icmp $I8 (IntCC.UnsignedGreaterThanOrEqual) idx bound)) + (_ Inst (trapnz $I8 cmp (TrapCode.TableOutOfBounds))) + + ;; Calculate the offset, in bytes, by multiplying `idx` by the byte size + ;; of the table element. + (idx Value (extend_index_to_addr idx addr_ty)) + (mul_imm Value (iconst addr_ty (imm64 (table_element_size table)))) + (idx_offset Value (imul addr_ty idx mul_imm)) + + ;; Calculate the final address by adding the table base address to the + ;; element offset to the static offset in the `table_addr` instruction. + (base Value (global_value_copy_facts addr_ty (table_base_gv table))) + (offset Value (iconst addr_ty (imm64 (i64_as_u64 (offset32_to_i32 offset))))) + (elem_addr Value (iadd addr_ty (iadd addr_ty base idx_offset) offset)) + + ;; Generate the final `result` and replace the current instruction with that. + (_ Value (select_table_addr cmp base elem_addr)) + ) + ;; re-legalize the above instructions + (prev_position))) + +(decl select_table_addr (Value Value Value) Value) +(rule (select_table_addr cmp base @ (value_type ty) element_addr) + (if-let $true (enable_table_access_spectre_mitigation)) + (select_spectre_guard ty cmp base element_addr)) +(rule (select_table_addr cmp base @ (value_type ty) element_addr) + (if-let $false (enable_table_access_spectre_mitigation)) + element_addr) + +(decl extend_index_to_addr (Value Type) Value) +(rule 0 (extend_index_to_addr val ty) (uextend ty val)) +(rule 1 (extend_index_to_addr val @ (value_type ty) ty) val) + +(decl table_bound_gv (Table) GlobalValue) +(extern constructor table_bound_gv table_bound_gv) +(decl table_base_gv (Table) GlobalValue) +(extern constructor table_base_gv table_base_gv) +(decl table_element_size (Table) u64) +(extern constructor table_element_size table_element_size) +(decl pure enable_table_access_spectre_mitigation () bool) +(extern constructor enable_table_access_spectre_mitigation enable_table_access_spectre_mitigation) + +;; ========= NaN canonicalization ============================================= + +(rule (legalize (ceil ty val)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (ceil ty val))) + +(rule (legalize (floor ty val)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (floor ty val))) + +(rule (legalize (nearest ty val)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (nearest ty val))) + +(rule (legalize (trunc ty val)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (trunc ty val))) + +(rule (legalize (sqrt ty val)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (sqrt ty val))) + +(rule (legalize (fadd ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fadd ty a b))) + +(rule (legalize (fsub ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fsub ty a b))) + +(rule (legalize (fmul ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fmul ty a b))) + +(rule (legalize (fdiv ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fdiv ty a b))) + +(rule (legalize (fmin ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fmin ty a b))) + +(rule (legalize (fmax ty a b)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fmax ty a b))) + +(rule (legalize (fma ty a b c)) + (if-let $true (enable_nan_canonicalization)) + (canonicalized ty (fma ty a b c))) + +(decl canonicalized (Type Value) Inst) +(rule (canonicalized $F32 val) + (select $F32 (fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan32) val)) +(rule (canonicalized $F64 val) + (select $F64 (fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan64) val)) +(rule (canonicalized $F32X4 val) + (let ((cmp Value (fcmp $I32X4 (FloatCC.NotEqual) val val)) + (canon Value (splat $F32X4 (canon_nan32))) + (mask Value (bitcast $F32X4 (mem_flags_empty) cmp))) + (bitselect $F32X4 mask canon val))) +(rule (canonicalized $F64X2 val) + (let ((canon Value (splat $F64X2 (canon_nan64))) + (cmp Value (fcmp $I64X2 (FloatCC.NotEqual) val val)) + (mask Value (bitcast $F64X2 (mem_flags_empty) cmp))) + (bitselect $F64X2 mask canon val))) + +(decl pure enable_nan_canonicalization () bool) +(extern constructor enable_nan_canonicalization enable_nan_canonicalization) + +(decl canon_nan32 () Value) +(extern constructor canon_nan32 canon_nan32) + +(decl canon_nan64 () Value) +(extern constructor canon_nan64 canon_nan64) diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs index e186b153091d..ee56462362b9 100644 --- a/cranelift/codegen/src/legalizer/mod.rs +++ b/cranelift/codegen/src/legalizer/mod.rs @@ -1,356 +1 @@ -//! Legalize instructions. -//! -//! A legal instruction is one that can be mapped directly to a machine code instruction for the -//! target ISA. The `legalize_function()` function takes as input any function and transforms it -//! into an equivalent function using only legal instructions. -//! -//! The characteristics of legal instructions depend on the target ISA, so any given instruction -//! can be legal for one ISA and illegal for another. -//! -//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map -//! which provides a legal encoding recipe for every instruction. -//! -//! The legalizer does not deal with register allocation constraints. These constraints are derived -//! from the encoding recipes, and solved later by the register allocator. - -use crate::cursor::{Cursor, FuncCursor}; -use crate::flowgraph::ControlFlowGraph; -use crate::ir::immediates::Imm64; -use crate::ir::types::{self, I128, I64}; -use crate::ir::{self, InstBuilder, InstructionData, MemFlags, Value}; -use crate::isa::TargetIsa; -use crate::trace; - -mod globalvalue; -mod table; - -use self::globalvalue::expand_global_value; -use self::table::expand_table_addr; - -fn imm_const(pos: &mut FuncCursor, arg: Value, imm: Imm64, is_signed: bool) -> Value { - let ty = pos.func.dfg.value_type(arg); - match (ty, is_signed) { - (I128, true) => { - let imm = pos.ins().iconst(I64, imm); - pos.ins().sextend(I128, imm) - } - (I128, false) => { - let imm = pos.ins().iconst(I64, imm); - pos.ins().uextend(I128, imm) - } - _ => { - let bits = imm.bits(); - let unsigned = match ty.lane_type() { - types::I8 => bits as u8 as i64, - types::I16 => bits as u16 as i64, - types::I32 => bits as u32 as i64, - types::I64 => bits, - _ => unreachable!(), - }; - pos.ins().iconst(ty.lane_type(), unsigned) - } - } -} - -/// Perform a simple legalization by expansion of the function, without -/// platform-specific transforms. -pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) { - trace!("Pre-legalization function:\n{}", func.display()); - - let mut pos = FuncCursor::new(func); - let func_begin = pos.position(); - pos.set_position(func_begin); - while let Some(_block) = pos.next_block() { - let mut prev_pos = pos.position(); - while let Some(inst) = pos.next_inst() { - match pos.func.dfg.insts[inst] { - // control flow - InstructionData::CondTrap { - opcode: - opcode @ (ir::Opcode::Trapnz | ir::Opcode::Trapz | ir::Opcode::ResumableTrapnz), - arg, - code, - } => { - expand_cond_trap(inst, &mut pos.func, cfg, opcode, arg, code); - } - - // memory and constants - InstructionData::UnaryGlobalValue { - opcode: ir::Opcode::GlobalValue, - global_value, - } => expand_global_value(inst, &mut pos.func, isa, global_value), - InstructionData::StackLoad { - opcode: ir::Opcode::StackLoad, - stack_slot, - offset, - } => { - let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst)); - let addr_ty = isa.pointer_type(); - - let mut pos = FuncCursor::new(pos.func).at_inst(inst); - pos.use_srcloc(inst); - - let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset); - - // Stack slots are required to be accessible. - // We can't currently ensure that they are aligned. - let mut mflags = MemFlags::new(); - mflags.set_notrap(); - pos.func.dfg.replace(inst).load(ty, mflags, addr, 0); - } - InstructionData::StackStore { - opcode: ir::Opcode::StackStore, - arg, - stack_slot, - offset, - } => { - let addr_ty = isa.pointer_type(); - - let mut pos = FuncCursor::new(pos.func).at_inst(inst); - pos.use_srcloc(inst); - - let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset); - - // Stack slots are required to be accessible. - // We can't currently ensure that they are aligned. - let mut mflags = MemFlags::new(); - mflags.set_notrap(); - pos.func.dfg.replace(inst).store(mflags, arg, addr, 0); - } - InstructionData::DynamicStackLoad { - opcode: ir::Opcode::DynamicStackLoad, - dynamic_stack_slot, - } => { - let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst)); - assert!(ty.is_dynamic_vector()); - let addr_ty = isa.pointer_type(); - - let mut pos = FuncCursor::new(pos.func).at_inst(inst); - pos.use_srcloc(inst); - - let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot); - - // Stack slots are required to be accessible and aligned. - let mflags = MemFlags::trusted(); - pos.func.dfg.replace(inst).load(ty, mflags, addr, 0); - } - InstructionData::DynamicStackStore { - opcode: ir::Opcode::DynamicStackStore, - arg, - dynamic_stack_slot, - } => { - pos.use_srcloc(inst); - let addr_ty = isa.pointer_type(); - let vector_ty = pos.func.dfg.value_type(arg); - assert!(vector_ty.is_dynamic_vector()); - - let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot); - - let mut mflags = MemFlags::new(); - // Stack slots are required to be accessible and aligned. - mflags.set_notrap(); - mflags.set_aligned(); - pos.func.dfg.replace(inst).store(mflags, arg, addr, 0); - } - InstructionData::TableAddr { - opcode: ir::Opcode::TableAddr, - table, - arg, - offset, - } => expand_table_addr(isa, inst, &mut pos.func, table, arg, offset), - - InstructionData::BinaryImm64 { opcode, arg, imm } => { - let is_signed = match opcode { - ir::Opcode::IaddImm - | ir::Opcode::IrsubImm - | ir::Opcode::ImulImm - | ir::Opcode::SdivImm - | ir::Opcode::SremImm => true, - _ => false, - }; - - let imm = imm_const(&mut pos, arg, imm, is_signed); - let replace = pos.func.dfg.replace(inst); - match opcode { - // bitops - ir::Opcode::BandImm => { - replace.band(arg, imm); - } - ir::Opcode::BorImm => { - replace.bor(arg, imm); - } - ir::Opcode::BxorImm => { - replace.bxor(arg, imm); - } - // bitshifting - ir::Opcode::IshlImm => { - replace.ishl(arg, imm); - } - ir::Opcode::RotlImm => { - replace.rotl(arg, imm); - } - ir::Opcode::RotrImm => { - replace.rotr(arg, imm); - } - ir::Opcode::SshrImm => { - replace.sshr(arg, imm); - } - ir::Opcode::UshrImm => { - replace.ushr(arg, imm); - } - // math - ir::Opcode::IaddImm => { - replace.iadd(arg, imm); - } - ir::Opcode::IrsubImm => { - // note: arg order reversed - replace.isub(imm, arg); - } - ir::Opcode::ImulImm => { - replace.imul(arg, imm); - } - ir::Opcode::SdivImm => { - replace.sdiv(arg, imm); - } - ir::Opcode::SremImm => { - replace.srem(arg, imm); - } - ir::Opcode::UdivImm => { - replace.udiv(arg, imm); - } - ir::Opcode::UremImm => { - replace.urem(arg, imm); - } - _ => prev_pos = pos.position(), - }; - } - - // comparisons - InstructionData::IntCompareImm { - opcode: ir::Opcode::IcmpImm, - cond, - arg, - imm, - } => { - let imm = imm_const(&mut pos, arg, imm, true); - pos.func.dfg.replace(inst).icmp(cond, arg, imm); - } - - // Legalize the fused bitwise-plus-not instructions into simpler - // instructions to assist with optimizations. Lowering will - // pattern match this sequence regardless when architectures - // support the instruction natively. - InstructionData::Binary { opcode, args } => { - match opcode { - ir::Opcode::BandNot => { - let neg = pos.ins().bnot(args[1]); - pos.func.dfg.replace(inst).band(args[0], neg); - } - ir::Opcode::BorNot => { - let neg = pos.ins().bnot(args[1]); - pos.func.dfg.replace(inst).bor(args[0], neg); - } - ir::Opcode::BxorNot => { - let neg = pos.ins().bnot(args[1]); - pos.func.dfg.replace(inst).bxor(args[0], neg); - } - _ => prev_pos = pos.position(), - }; - } - - _ => { - prev_pos = pos.position(); - continue; - } - } - - // Legalization implementations require fixpoint loop here. - // TODO: fix this. - pos.set_position(prev_pos); - } - } - - trace!("Post-legalization function:\n{}", func.display()); -} - -/// Custom expansion for conditional trap instructions. -fn expand_cond_trap( - inst: ir::Inst, - func: &mut ir::Function, - cfg: &mut ControlFlowGraph, - opcode: ir::Opcode, - arg: ir::Value, - code: ir::TrapCode, -) { - trace!( - "expanding conditional trap: {:?}: {}", - inst, - func.dfg.display_inst(inst) - ); - - // Parse the instruction. - let trapz = match opcode { - ir::Opcode::Trapz => true, - ir::Opcode::Trapnz | ir::Opcode::ResumableTrapnz => false, - _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst)), - }; - - // Split the block after `inst`: - // - // trapnz arg - // .. - // - // Becomes: - // - // brif arg, new_block_trap, new_block_resume - // - // new_block_trap: - // trap - // - // new_block_resume: - // .. - let old_block = func - .layout - .inst_block(inst) - .expect("Instruction not in layout."); - let new_block_trap = func.dfg.make_block(); - let new_block_resume = func.dfg.make_block(); - - // Trapping is a rare event, mark the trapping block as cold. - func.layout.set_cold(new_block_trap); - - // Replace trap instruction by the inverted condition. - if trapz { - func.dfg - .replace(inst) - .brif(arg, new_block_resume, &[], new_block_trap, &[]); - } else { - func.dfg - .replace(inst) - .brif(arg, new_block_trap, &[], new_block_resume, &[]); - } - - // Insert the new label and the unconditional trap terminator. - let mut pos = FuncCursor::new(func).after_inst(inst); - pos.use_srcloc(inst); - pos.insert_block(new_block_trap); - - match opcode { - ir::Opcode::Trapz | ir::Opcode::Trapnz => { - pos.ins().trap(code); - } - ir::Opcode::ResumableTrapnz => { - pos.ins().resumable_trap(code); - pos.ins().jump(new_block_resume, &[]); - } - _ => unreachable!(), - } - - // Insert the new label and resume the execution when the trap fails. - pos.insert_block(new_block_resume); - - // Finally update the CFG. - cfg.recompute_block(pos.func, old_block); - cfg.recompute_block(pos.func, new_block_resume); - cfg.recompute_block(pos.func, new_block_trap); -} +pub mod isle; diff --git a/cranelift/codegen/src/legalizer/table.rs b/cranelift/codegen/src/legalizer/table.rs deleted file mode 100644 index db72508a2680..000000000000 --- a/cranelift/codegen/src/legalizer/table.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! Legalization of tables. -//! -//! This module exports the `expand_table_addr` function which transforms a `table_addr` -//! instruction into code that depends on the kind of table referenced. - -use crate::cursor::{Cursor, FuncCursor}; -use crate::ir::condcodes::IntCC; -use crate::ir::immediates::Offset32; -use crate::ir::{self, InstBuilder}; -use crate::isa::TargetIsa; - -/// Expand a `table_addr` instruction according to the definition of the table. -pub fn expand_table_addr( - isa: &dyn TargetIsa, - inst: ir::Inst, - func: &mut ir::Function, - table: ir::Table, - index: ir::Value, - element_offset: Offset32, -) { - let bound_gv = func.tables[table].bound_gv; - let index_ty = func.dfg.value_type(index); - let addr_ty = func.dfg.value_type(func.dfg.first_result(inst)); - let mut pos = FuncCursor::new(func).at_inst(inst); - pos.use_srcloc(inst); - - // Start with the bounds check. Trap if `index + 1 > bound`. - let bound = pos.ins().global_value(index_ty, bound_gv); - - // `index > bound - 1` is the same as `index >= bound`. - let oob = pos - .ins() - .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound); - pos.ins().trapnz(oob, ir::TrapCode::TableOutOfBounds); - - // If Spectre mitigations are enabled, we will use a comparison to - // short-circuit the computed table element address to the start - // of the table on the misspeculation path when out-of-bounds. - let spectre_oob_cmp = if isa.flags().enable_table_access_spectre_mitigation() { - Some((index, bound)) - } else { - None - }; - - compute_addr( - inst, - table, - addr_ty, - index, - index_ty, - element_offset, - pos.func, - spectre_oob_cmp, - ); -} - -/// Emit code for the base address computation of a `table_addr` instruction. -fn compute_addr( - inst: ir::Inst, - table: ir::Table, - addr_ty: ir::Type, - mut index: ir::Value, - index_ty: ir::Type, - element_offset: Offset32, - func: &mut ir::Function, - spectre_oob_cmp: Option<(ir::Value, ir::Value)>, -) { - let mut pos = FuncCursor::new(func).at_inst(inst); - pos.use_srcloc(inst); - - // Convert `index` to `addr_ty`. - if index_ty != addr_ty { - index = pos.ins().uextend(addr_ty, index); - } - - // Add the table base address base - let base_gv = pos.func.tables[table].base_gv; - let base = pos.ins().global_value(addr_ty, base_gv); - - let element_size = pos.func.tables[table].element_size; - let mut offset; - let element_size: u64 = element_size.into(); - if element_size == 1 { - offset = index; - } else if element_size.is_power_of_two() { - offset = pos - .ins() - .ishl_imm(index, i64::from(element_size.trailing_zeros())); - } else { - offset = pos.ins().imul_imm(index, element_size as i64); - } - - let element_addr = if element_offset == Offset32::new(0) { - pos.ins().iadd(base, offset) - } else { - let imm: i64 = element_offset.into(); - offset = pos.ins().iadd(base, offset); - pos.ins().iadd_imm(offset, imm) - }; - - let element_addr = if let Some((index, bound)) = spectre_oob_cmp { - let cond = pos - .ins() - .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound); - // If out-of-bounds, choose the table base on the misspeculation path. - pos.ins().select_spectre_guard(cond, base, element_addr) - } else { - element_addr - }; - let new_inst = pos.func.dfg.value_def(element_addr).inst().unwrap(); - - pos.func.dfg.replace_with_aliases(inst, new_inst); - pos.remove_inst(); -} diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index c856e1a1b645..9d2e9200c159 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -77,7 +77,6 @@ mod inst_predicates; mod isle_prelude; mod iterators; mod legalizer; -mod nan_canonicalization; mod opts; mod remove_constant_phis; mod result; diff --git a/cranelift/codegen/src/nan_canonicalization.rs b/cranelift/codegen/src/nan_canonicalization.rs deleted file mode 100644 index 49415a86b0d6..000000000000 --- a/cranelift/codegen/src/nan_canonicalization.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! A NaN-canonicalizing rewriting pass. Patch floating point arithmetic -//! instructions that may return a NaN result with a sequence of operations -//! that will replace nondeterministic NaN's with a single canonical NaN value. - -use crate::cursor::{Cursor, FuncCursor}; -use crate::ir::condcodes::FloatCC; -use crate::ir::immediates::{Ieee32, Ieee64}; -use crate::ir::types; -use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value}; -use crate::opts::MemFlags; -use crate::timing; - -// Canonical 32-bit and 64-bit NaN values. -static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000; -static CANON_64BIT_NAN: u64 = 0b0111111111111000000000000000000000000000000000000000000000000000; - -/// Perform the NaN canonicalization pass. -pub fn do_nan_canonicalization(func: &mut Function) { - let _tt = timing::canonicalize_nans(); - let mut pos = FuncCursor::new(func); - while let Some(_block) = pos.next_block() { - while let Some(inst) = pos.next_inst() { - if is_fp_arith(&mut pos, inst) { - add_nan_canon_seq(&mut pos, inst); - } - } - } -} - -/// Returns true/false based on whether the instruction is a floating-point -/// arithmetic operation. This ignores operations like `fneg`, `fabs`, or -/// `fcopysign` that only operate on the sign bit of a floating point value. -fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool { - match pos.func.dfg.insts[inst] { - InstructionData::Unary { opcode, .. } => { - opcode == Opcode::Ceil - || opcode == Opcode::Floor - || opcode == Opcode::Nearest - || opcode == Opcode::Sqrt - || opcode == Opcode::Trunc - } - InstructionData::Binary { opcode, .. } => { - opcode == Opcode::Fadd - || opcode == Opcode::Fdiv - || opcode == Opcode::Fmax - || opcode == Opcode::Fmin - || opcode == Opcode::Fmul - || opcode == Opcode::Fsub - } - InstructionData::Ternary { opcode, .. } => opcode == Opcode::Fma, - _ => false, - } -} - -/// Append a sequence of canonicalizing instructions after the given instruction. -fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) { - // Select the instruction result, result type. Replace the instruction - // result and step forward before inserting the canonicalization sequence. - let val = pos.func.dfg.first_result(inst); - let val_type = pos.func.dfg.value_type(val); - let new_res = pos.func.dfg.replace_result(val, val_type); - let _next_inst = pos.next_inst().expect("block missing terminator!"); - - // Insert a comparison instruction, to check if `inst_res` is NaN. Select - // the canonical NaN value if `val` is NaN, assign the result to `inst`. - let is_nan = pos.ins().fcmp(FloatCC::NotEqual, new_res, new_res); - - let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| { - pos.ins() - .with_result(val) - .select(is_nan, canon_nan, new_res); - }; - let vector_select = |pos: &mut FuncCursor, canon_nan: Value| { - let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan); - pos.ins() - .with_result(val) - .bitselect(is_nan, canon_nan, new_res); - }; - - match val_type { - types::F32 => { - let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN)); - scalar_select(pos, canon_nan); - } - types::F64 => { - let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN)); - scalar_select(pos, canon_nan); - } - types::F32X4 => { - let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN)); - let canon_nan = pos.ins().splat(types::F32X4, canon_nan); - vector_select(pos, canon_nan); - } - types::F64X2 => { - let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN)); - let canon_nan = pos.ins().splat(types::F64X2, canon_nan); - vector_select(pos, canon_nan); - } - _ => { - // Panic if the type given was not an IEEE floating point type. - panic!("Could not canonicalize NaN: Unexpected result type found."); - } - } - - pos.prev_inst(); // Step backwards so the pass does not skip instructions. -} diff --git a/cranelift/codegen/src/opts/extends.isle b/cranelift/codegen/src/opts/extends.isle index a1a4dfda0596..83ded0d7479f 100644 --- a/cranelift/codegen/src/opts/extends.isle +++ b/cranelift/codegen/src/opts/extends.isle @@ -32,3 +32,9 @@ ;; actually doing the extend in the first place. (rule (simplify (ireduce ty (sextend _ x @ (value_type ty)))) x) (rule (simplify (ireduce ty (uextend _ x @ (value_type ty)))) x) + +;; Ignore extens in `fcvt_from_{uint,sint}` since extension won't change the +;; result and backends can sometimes have more optimal lowerings if it's known +;; that the type is smaller. +(rule (simplify (fcvt_from_uint ty (uextend _ x))) (fcvt_from_uint ty x)) +(rule (simplify (fcvt_from_uint ty (sextend _ x))) (fcvt_from_uint ty x)) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index c61876b8d481..a2cf1be788b4 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -337,6 +337,14 @@ (decl pure mem_flags_trusted () MemFlags) (extern constructor mem_flags_trusted mem_flags_trusted) +;; `MemFlags::notrap` +(decl pure mem_flags_notrap () MemFlags) +(extern constructor mem_flags_notrap mem_flags_notrap) + +;; `MemFlags::new` +(decl pure mem_flags_empty () MemFlags) +(extern constructor mem_flags_empty mem_flags_empty) + ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Swap args of an IntCC flag. diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif index f2b992771bde..20733f0bd121 100644 --- a/cranelift/filetests/filetests/egraph/shifts.clif +++ b/cranelift/filetests/filetests/egraph/shifts.clif @@ -221,9 +221,9 @@ block0(v0: i32): v1 = ishl_imm v0, 24 v2 = sshr_imm v1, 24 return v2 - ; check: v5 = ireduce.i8 v0 - ; check: v6 = sextend.i32 v5 - ; check: return v6 + ; check: v7 = ireduce.i8 v0 + ; check: v8 = sextend.i32 v7 + ; check: return v8 } function %i32_shl_sshr_16_to_ireduce(i32) -> i32 { @@ -231,9 +231,9 @@ block0(v0: i32): v1 = ishl_imm v0, 16 v2 = sshr_imm v1, 16 return v2 - ; check: v5 = ireduce.i16 v0 - ; check: v6 = sextend.i32 v5 - ; check: return v6 + ; check: v7 = ireduce.i16 v0 + ; check: v8 = sextend.i32 v7 + ; check: return v8 } function %i64_shl_sshr_8_to_ireduce(i64) -> i64 { @@ -241,9 +241,9 @@ block0(v0: i64): v1 = ishl_imm v0, 56 v2 = sshr_imm v1, 56 return v2 - ; check: v5 = ireduce.i8 v0 - ; check: v6 = sextend.i64 v5 - ; check: return v6 + ; check: v7 = ireduce.i8 v0 + ; check: v8 = sextend.i64 v7 + ; check: return v8 } function %i64_shl_sshr_16_to_ireduce(i64) -> i64 { @@ -251,9 +251,9 @@ block0(v0: i64): v1 = ishl_imm v0, 48 v2 = sshr_imm v1, 48 return v2 - ; check: v5 = ireduce.i16 v0 - ; check: v6 = sextend.i64 v5 - ; check: return v6 + ; check: v7 = ireduce.i16 v0 + ; check: v8 = sextend.i64 v7 + ; check: return v8 } function %i64_shl_sshr_32_to_ireduce(i64) -> i64 { @@ -261,9 +261,9 @@ block0(v0: i64): v1 = ishl_imm v0, 32 v2 = sshr_imm v1, 32 return v2 - ; check: v5 = ireduce.i32 v0 - ; check: v6 = sextend.i64 v5 - ; check: return v6 + ; check: v7 = ireduce.i32 v0 + ; check: v8 = sextend.i64 v7 + ; check: return v8 } function %i32_shl_ushr_8_to_ireduce(i32) -> i32 { @@ -271,9 +271,9 @@ block0(v0: i32): v1 = ishl_imm v0, 24 v2 = ushr_imm v1, 24 return v2 - ; check: v7 = ireduce.i8 v0 - ; check: v8 = uextend.i32 v7 - ; check: return v8 + ; check: v9 = ireduce.i8 v0 + ; check: v10 = uextend.i32 v9 + ; check: return v10 } function %i32_shl_ushr_16_to_ireduce(i32) -> i32 { @@ -281,9 +281,9 @@ block0(v0: i32): v1 = ishl_imm v0, 16 v2 = ushr_imm v1, 16 return v2 - ; check: v7 = ireduce.i16 v0 - ; check: v8 = uextend.i32 v7 - ; check: return v8 + ; check: v9 = ireduce.i16 v0 + ; check: v10 = uextend.i32 v9 + ; check: return v10 } function %i64_shl_ushr_8_to_ireduce(i64) -> i64 { @@ -291,9 +291,9 @@ block0(v0: i64): v1 = ishl_imm v0, 56 v2 = ushr_imm v1, 56 return v2 - ; check: v7 = ireduce.i8 v0 - ; check: v8 = uextend.i64 v7 - ; check: return v8 + ; check: v9 = ireduce.i8 v0 + ; check: v10 = uextend.i64 v9 + ; check: return v10 } function %i64_shl_ushr_16_to_ireduce(i64) -> i64 { @@ -301,9 +301,9 @@ block0(v0: i64): v1 = ishl_imm v0, 48 v2 = ushr_imm v1, 48 return v2 - ; check: v7 = ireduce.i16 v0 - ; check: v8 = uextend.i64 v7 - ; check: return v8 + ; check: v9 = ireduce.i16 v0 + ; check: v10 = uextend.i64 v9 + ; check: return v10 } function %i64_shl_ushr_32_to_ireduce(i64) -> i64 { @@ -311,9 +311,9 @@ block0(v0: i64): v1 = ishl_imm v0, 32 v2 = ushr_imm v1, 32 return v2 - ; check: v7 = ireduce.i32 v0 - ; check: v8 = uextend.i64 v7 - ; check: return v8 + ; check: v9 = ireduce.i32 v0 + ; check: v10 = uextend.i64 v9 + ; check: return v10 } function %ishl_amt_type_ireduce(i8, i16) -> i8 { diff --git a/cranelift/filetests/filetests/egraph/vector.clif b/cranelift/filetests/filetests/egraph/vector.clif index 4ecbaa83b318..a9cb2cbe7746 100644 --- a/cranelift/filetests/filetests/egraph/vector.clif +++ b/cranelift/filetests/filetests/egraph/vector.clif @@ -18,10 +18,9 @@ block0(v0: i32): v2 = splat.i64x2 v1 v3 = fcvt_from_uint.f64x2 v2 return v3 - ; check: v1 = uextend.i64 v0 - ; check: v4 = fcvt_from_uint.f64 v1 - ; check: v5 = splat.f64x2 v4 - ; check: return v5 + ; check: v5 = fcvt_from_uint.f64 v0 + ; check: v7 = splat.f64x2 v5 + ; check: return v7 } function %f3(i32) -> f64x2 { diff --git a/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif b/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif index 96ca30750cce..f1832e6258dd 100644 --- a/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif +++ b/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif @@ -26,15 +26,15 @@ block0(v0: i32): ; VCode: ; block0: -; slli a2,a0,32 -; srli a0,a2,32 +; slli a3,a0,32 +; srli a0,a3,32 ; li a1,0 ; ret ; ; Disassembled: ; block0: ; offset 0x0 -; slli a2, a0, 0x20 -; srli a0, a2, 0x20 +; slli a3, a0, 0x20 +; srli a0, a3, 0x20 ; mv a1, zero ; ret @@ -46,15 +46,15 @@ block0(v0: i16): ; VCode: ; block0: -; slli a2,a0,48 -; srli a0,a2,48 +; slli a3,a0,48 +; srli a0,a3,48 ; li a1,0 ; ret ; ; Disassembled: ; block0: ; offset 0x0 -; slli a2, a0, 0x30 -; srli a0, a2, 0x30 +; slli a3, a0, 0x30 +; srli a0, a3, 0x30 ; mv a1, zero ; ret @@ -118,15 +118,15 @@ block0(v0: i16): ; VCode: ; block0: -; slli a2,a0,48 -; srai a0,a2,48 +; slli a3,a0,48 +; srai a0,a3,48 ; srai a1,a0,63 ; ret ; ; Disassembled: ; block0: ; offset 0x0 -; slli a2, a0, 0x30 -; srai a0, a2, 0x30 +; slli a3, a0, 0x30 +; srai a0, a3, 0x30 ; srai a1, a0, 0x3f ; ret @@ -138,15 +138,15 @@ block0(v0: i8): ; VCode: ; block0: -; slli a2,a0,56 -; srai a0,a2,56 +; slli a3,a0,56 +; srai a0,a3,56 ; srai a1,a0,63 ; ret ; ; Disassembled: ; block0: ; offset 0x0 -; slli a2, a0, 0x38 -; srai a0, a2, 0x38 +; slli a3, a0, 0x38 +; srai a0, a3, 0x38 ; srai a1, a0, 0x3f ; ret diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat index 04cfec907e23..aebf4b89f1de 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat @@ -50,9 +50,10 @@ ;; clgr %r4, %r2 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; lghi %r5, 4096 -;; strv %r3, 0(%r5,%r4) +;; lg %r5, 0(%r5) +;; agr %r5, %r4 +;; lghi %r2, 4096 +;; strv %r3, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 @@ -65,14 +66,16 @@ ;; block0: ;; lgr %r4, %r3 ;; llgfr %r3, %r2 -;; lg %r2, 8(%r4) -;; aghi %r2, -4100 -;; clgr %r3, %r2 +;; lgr %r5, %r4 +;; lg %r4, 8(%r5) +;; aghi %r4, -4100 +;; clgr %r3, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; lghi %r4, 4096 -;; lrv %r2, 0(%r4,%r3) +;; lg %r5, 0(%r5) +;; agr %r5, %r3 +;; lghi %r2, 4096 +;; lrv %r2, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat index 92de57a3f35a..9129071d021e 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -53,23 +52,22 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r7, %r4 -;; llgfr %r4, %r2 -;; lgr %r5, %r2 +;; llgfr %r5, %r2 +;; lgr %r10, %r2 ;; llilf %r2, 4294901764 -;; algfr %r2, %r5 +;; algfr %r2, %r10 ;; jgnle .+2 # trap=heap_oob -;; lgr %r5, %r7 -;; lg %r7, 8(%r5) -;; clgr %r2, %r7 +;; lg %r8, 8(%r4) +;; clgr %r2, %r8 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; llilh %r2, 65535 -;; strv %r3, 0(%r2,%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; llilh %r5, 65535 +;; strv %r3, 0(%r5,%r4) ;; jg label2 ;; block2: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; block3: ;; .word 0x0000 # trap=heap_oob @@ -78,20 +76,18 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r4, %r2 -;; llilf %r2, 4294901764 -;; algfr %r2, %r4 +;; llgfr %r4, %r2 +;; llilf %r5, 4294901764 +;; algfr %r5, %r2 ;; jgnle .+2 # trap=heap_oob -;; lgr %r4, %r5 -;; lg %r5, 8(%r4) -;; clgr %r2, %r5 +;; lg %r2, 8(%r3) +;; clgr %r5, %r2 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; llilh %r2, 65535 -;; lrv %r2, 0(%r2,%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r4 +;; llilh %r4, 65535 +;; lrv %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat index fd04d8dfce5f..dfd611fb7c60 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat @@ -50,9 +50,10 @@ ;; clgr %r4, %r2 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; lghi %r5, 4096 -;; stc %r3, 0(%r5,%r4) +;; lg %r5, 0(%r5) +;; agr %r5, %r4 +;; lghi %r2, 4096 +;; stc %r3, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 @@ -65,14 +66,16 @@ ;; block0: ;; lgr %r4, %r3 ;; llgfr %r3, %r2 -;; lg %r2, 8(%r4) -;; aghi %r2, -4097 -;; clgr %r3, %r2 +;; lgr %r5, %r4 +;; lg %r4, 8(%r5) +;; aghi %r4, -4097 +;; clgr %r3, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; lghi %r4, 4096 -;; llc %r2, 0(%r4,%r3) +;; lg %r5, 0(%r5) +;; agr %r5, %r3 +;; lghi %r2, 4096 +;; llc %r2, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat index 559b03c3e981..7a4c2ef807d8 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -53,23 +52,22 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r7, %r4 -;; llgfr %r4, %r2 -;; lgr %r5, %r2 +;; llgfr %r5, %r2 +;; lgr %r10, %r2 ;; llilf %r2, 4294901761 -;; algfr %r2, %r5 +;; algfr %r2, %r10 ;; jgnle .+2 # trap=heap_oob -;; lgr %r5, %r7 -;; lg %r7, 8(%r5) -;; clgr %r2, %r7 +;; lg %r8, 8(%r4) +;; clgr %r2, %r8 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; llilh %r2, 65535 -;; stc %r3, 0(%r2,%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; llilh %r5, 65535 +;; stc %r3, 0(%r5,%r4) ;; jg label2 ;; block2: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; block3: ;; .word 0x0000 # trap=heap_oob @@ -78,20 +76,18 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r4, %r2 -;; llilf %r2, 4294901761 -;; algfr %r2, %r4 +;; llgfr %r4, %r2 +;; llilf %r5, 4294901761 +;; algfr %r5, %r2 ;; jgnle .+2 # trap=heap_oob -;; lgr %r4, %r5 -;; lg %r5, 8(%r4) -;; clgr %r2, %r5 +;; lg %r2, 8(%r3) +;; clgr %r5, %r2 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; llilh %r2, 65535 -;; llc %r2, 0(%r2,%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r4 +;; llilh %r4, 65535 +;; llc %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat index d52c68aed627..ae1529e1467a 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -54,37 +53,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lgr %r2, %r5 -;; lg %r5, 8(%r2) -;; aghik %r6, %r5, -4 -;; lgr %r5, %r4 -;; ag %r5, 0(%r2) -;; lghi %r2, 0 -;; clgr %r4, %r6 -;; locgrh %r5, %r2 -;; strv %r3, 0(%r5) +;; llgfr %r7, %r2 +;; lg %r5, 8(%r4) +;; aghi %r5, -4 +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r7 +;; lghi %r4, 0 +;; clgr %r7, %r5 +;; locgrh %r2, %r4 +;; strv %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r3 -;; llgfr %r3, %r2 -;; lgr %r2, %r4 -;; lg %r4, 8(%r2) -;; aghi %r4, -4 -;; lgr %r5, %r3 -;; ag %r5, 0(%r2) -;; lghi %r2, 0 -;; clgr %r3, %r4 -;; locgrh %r5, %r2 -;; lrv %r2, 0(%r5) +;; llgfr %r4, %r2 +;; lg %r5, 8(%r3) +;; aghi %r5, -4 +;; lg %r2, 0(%r3) +;; agr %r2, %r4 +;; lghi %r3, 0 +;; clgr %r4, %r5 +;; locgrh %r2, %r3 +;; lrv %r2, 0(%r2) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat index dc74058fbcd6..441d61a0a6ba 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -56,32 +55,32 @@ ;; llgfr %r5, %r2 ;; lg %r2, 8(%r4) ;; aghi %r2, -4100 -;; lgr %r7, %r5 -;; ag %r7, 0(%r4) -;; aghik %r4, %r7, 4096 -;; lghi %r7, 0 +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; aghi %r4, 4096 +;; lghi %r8, 0 ;; clgr %r5, %r2 -;; locgrh %r4, %r7 +;; locgrh %r4, %r8 ;; strv %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r4, %r2 -;; lg %r5, 8(%r3) -;; aghi %r5, -4100 -;; lgr %r2, %r4 -;; ag %r2, 0(%r3) -;; aghi %r2, 4096 -;; lghi %r3, 0 -;; clgr %r4, %r5 -;; locgrh %r2, %r3 -;; lrv %r2, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r3) +;; aghi %r2, -4100 +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; aghi %r3, 4096 +;; lghi %r4, 0 +;; clgr %r5, %r2 +;; locgrh %r3, %r4 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat index 5d1c7f1916ee..e891ab649b04 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -41,7 +41,8 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r10, %r15, 80(%r15) +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } ;; unwind SaveReg { clobber_offset: 96, reg: p12i } @@ -50,43 +51,50 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r14, %r2 -;; llilf %r5, 4294901764 -;; algfr %r5, %r2 +;; llgfr %r10, %r2 +;; llilf %r9, 4294901764 +;; algfr %r9, %r2 ;; jgnle .+2 # trap=heap_oob -;; lg %r10, 8(%r4) -;; ag %r14, 0(%r4) -;; llilh %r4, 65535 -;; agrk %r2, %r14, %r4 -;; lghi %r4, 0 -;; clgr %r5, %r10 -;; locgrh %r2, %r4 -;; strv %r3, 0(%r2) +;; lg %r5, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r10 +;; llilh %r2, 65535 +;; agr %r4, %r2 +;; lghi %r2, 0 +;; clgr %r9, %r5 +;; locgrh %r4, %r2 +;; strv %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r10, %r15, 80(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r14, %r2 -;; llilf %r5, 4294901764 -;; algfr %r5, %r2 +;; llgfr %r5, %r2 +;; llilf %r9, 4294901764 +;; algfr %r9, %r2 ;; jgnle .+2 # trap=heap_oob ;; lg %r4, 8(%r3) -;; ag %r14, 0(%r3) -;; llilh %r3, 65535 -;; agrk %r2, %r14, %r3 -;; lghi %r3, 0 -;; clgr %r5, %r4 -;; locgrh %r2, %r3 -;; lrv %r2, 0(%r2) +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; llilh %r2, 65535 +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 +;; clgr %r9, %r4 +;; locgrh %r3, %r5 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat index 334a42b8eee9..99d184cbf083 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -41,7 +41,9 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r8, %r15, 64(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -52,32 +54,43 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 +;; llgfr %r7, %r2 ;; lg %r5, 8(%r4) -;; lgr %r8, %r4 -;; lgr %r4, %r2 -;; ag %r4, 0(%r8) -;; lghi %r14, 0 -;; clgr %r2, %r5 -;; locgrhe %r4, %r14 -;; stc %r3, 0(%r4) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r7 +;; lghi %r6, 0 +;; clgr %r7, %r5 +;; locgrhe %r2, %r6 +;; stc %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r8, %r15, 64(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lg %r5, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; lghi %r3, 0 -;; clgr %r2, %r5 -;; locgrhe %r4, %r3 -;; llc %r2, 0(%r4) +;; llgfr %r6, %r2 +;; lg %r4, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r5, %r3, %r6 +;; lghi %r2, 0 +;; clgr %r6, %r4 +;; locgrhe %r5, %r2 +;; llc %r2, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat index 44be0e98eb28..7979de1f2a26 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -56,32 +55,32 @@ ;; llgfr %r5, %r2 ;; lg %r2, 8(%r4) ;; aghi %r2, -4097 -;; lgr %r7, %r5 -;; ag %r7, 0(%r4) -;; aghik %r4, %r7, 4096 -;; lghi %r7, 0 +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; aghi %r4, 4096 +;; lghi %r8, 0 ;; clgr %r5, %r2 -;; locgrh %r4, %r7 +;; locgrh %r4, %r8 ;; stc %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r4, %r2 -;; lg %r5, 8(%r3) -;; aghi %r5, -4097 -;; lgr %r2, %r4 -;; ag %r2, 0(%r3) -;; aghi %r2, 4096 -;; lghi %r3, 0 -;; clgr %r4, %r5 -;; locgrh %r2, %r3 -;; llc %r2, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r3) +;; aghi %r2, -4097 +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; aghi %r3, 4096 +;; lghi %r4, 0 +;; clgr %r5, %r2 +;; locgrh %r3, %r4 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 93550319358b..c7827f26dbe8 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -41,7 +41,8 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r10, %r15, 80(%r15) +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } ;; unwind SaveReg { clobber_offset: 96, reg: p12i } @@ -50,43 +51,50 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r14, %r2 -;; llilf %r5, 4294901761 -;; algfr %r5, %r2 +;; llgfr %r10, %r2 +;; llilf %r9, 4294901761 +;; algfr %r9, %r2 ;; jgnle .+2 # trap=heap_oob -;; lg %r10, 8(%r4) -;; ag %r14, 0(%r4) -;; llilh %r4, 65535 -;; agrk %r2, %r14, %r4 -;; lghi %r4, 0 -;; clgr %r5, %r10 -;; locgrh %r2, %r4 -;; stc %r3, 0(%r2) +;; lg %r5, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r10 +;; llilh %r2, 65535 +;; agr %r4, %r2 +;; lghi %r2, 0 +;; clgr %r9, %r5 +;; locgrh %r4, %r2 +;; stc %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r10, %r15, 80(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r14, %r2 -;; llilf %r5, 4294901761 -;; algfr %r5, %r2 +;; llgfr %r5, %r2 +;; llilf %r9, 4294901761 +;; algfr %r9, %r2 ;; jgnle .+2 # trap=heap_oob ;; lg %r4, 8(%r3) -;; ag %r14, 0(%r3) -;; llilh %r3, 65535 -;; agrk %r2, %r14, %r3 -;; lghi %r3, 0 -;; clgr %r5, %r4 -;; locgrh %r2, %r3 -;; llc %r2, 0(%r2) +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; llilh %r2, 65535 +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 +;; clgr %r9, %r4 +;; locgrh %r3, %r5 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat index 898fac6ffcc6..a8274eefe16d 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat @@ -43,13 +43,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lg %r2, 8(%r5) -;; clgr %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r5, 8(%r4) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 ;; lghi %r5, 4096 ;; strv %r3, 0(%r5,%r4) ;; jg label2 @@ -62,15 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r3 -;; llgfr %r3, %r2 -;; lg %r5, 8(%r4) -;; clgr %r3, %r5 +;; llgfr %r2, %r2 +;; lg %r4, 8(%r3) +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; lghi %r4, 4096 -;; lrv %r2, 0(%r4,%r3) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; lrv %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat index 47d8b57e247f..2b0c428e21f6 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -43,13 +43,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lg %r2, 8(%r5) -;; clgr %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r5, 8(%r4) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 ;; llilh %r5, 65535 ;; strv %r3, 0(%r5,%r4) ;; jg label2 @@ -62,15 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r3 -;; llgfr %r3, %r2 -;; lg %r5, 8(%r4) -;; clgr %r3, %r5 +;; llgfr %r2, %r2 +;; lg %r4, 8(%r3) +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; llilh %r4, 65535 -;; lrv %r2, 0(%r4,%r3) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; llilh %r5, 65535 +;; lrv %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat index d3bac7e16fac..622e48fe73cc 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat @@ -43,13 +43,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lg %r2, 8(%r5) -;; clgr %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r5, 8(%r4) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 ;; lghi %r5, 4096 ;; stc %r3, 0(%r5,%r4) ;; jg label2 @@ -62,15 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r3 -;; llgfr %r3, %r2 -;; lg %r5, 8(%r4) -;; clgr %r3, %r5 +;; llgfr %r2, %r2 +;; lg %r4, 8(%r3) +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; lghi %r4, 4096 -;; llc %r2, 0(%r4,%r3) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; llc %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat index 50469a1d1b84..56c149aee5cc 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -43,13 +43,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lg %r2, 8(%r5) -;; clgr %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r5, 8(%r4) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 ;; llilh %r5, 65535 ;; stc %r3, 0(%r5,%r4) ;; jg label2 @@ -62,15 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r3 -;; llgfr %r3, %r2 -;; lg %r5, 8(%r4) -;; clgr %r3, %r5 +;; llgfr %r2, %r2 +;; lg %r4, 8(%r3) +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r3, 0(%r4) -;; llilh %r4, 65535 -;; llc %r2, 0(%r4,%r3) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; llilh %r5, 65535 +;; llc %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index 3f0878a849f0..88e073836a22 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -41,7 +41,9 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r8, %r15, 64(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -52,32 +54,43 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 +;; llgfr %r7, %r2 ;; lg %r5, 8(%r4) -;; lgr %r8, %r4 -;; lgr %r4, %r2 -;; ag %r4, 0(%r8) -;; lghi %r14, 0 -;; clgr %r2, %r5 -;; locgrh %r4, %r14 -;; strv %r3, 0(%r4) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r7 +;; lghi %r6, 0 +;; clgr %r7, %r5 +;; locgrh %r2, %r6 +;; strv %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r8, %r15, 64(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lg %r5, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; lghi %r3, 0 -;; clgr %r2, %r5 -;; locgrh %r4, %r3 -;; lrv %r2, 0(%r4) +;; llgfr %r6, %r2 +;; lg %r4, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r5, %r3, %r6 +;; lghi %r2, 0 +;; clgr %r6, %r4 +;; locgrh %r5, %r2 +;; lrv %r2, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat index 2d0a997c86de..268b4ab4580f 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -54,33 +53,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r6, %r2 +;; llgfr %r8, %r2 ;; lg %r5, 8(%r4) -;; lgr %r2, %r6 -;; ag %r2, 0(%r4) -;; aghi %r2, 4096 -;; lghi %r4, 0 -;; clgr %r6, %r5 -;; locgrh %r2, %r4 +;; lg %r4, 0(%r4) +;; agr %r4, %r8 +;; aghik %r2, %r4, 4096 +;; lghi %r7, 0 +;; clgr %r8, %r5 +;; locgrh %r2, %r7 ;; strv %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lg %r4, 8(%r3) -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghi %r5, 4096 +;; llgfr %r4, %r2 +;; lg %r5, 8(%r3) +;; lg %r2, 0(%r3) +;; agr %r2, %r4 +;; aghi %r2, 4096 ;; lghi %r3, 0 -;; clgr %r2, %r4 -;; locgrh %r5, %r3 -;; lrv %r2, 0(%r5) +;; clgr %r4, %r5 +;; locgrh %r2, %r3 +;; lrv %r2, 0(%r2) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat index d013a5d8a469..48acc69e20cc 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -53,35 +52,35 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r7, %r2 -;; lg %r5, 8(%r4) -;; lgr %r2, %r7 -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; agr %r2, %r4 -;; lghi %r4, 0 -;; clgr %r7, %r5 -;; locgrh %r2, %r4 -;; strv %r3, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; llilh %r8, 65535 +;; agr %r4, %r8 +;; lghi %r8, 0 +;; clgr %r5, %r2 +;; locgrh %r4, %r8 +;; strv %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r4, %r2 -;; lg %r5, 8(%r3) -;; lgr %r2, %r4 -;; ag %r2, 0(%r3) -;; llilh %r3, 65535 -;; agr %r2, %r3 -;; lghi %r3, 0 -;; clgr %r4, %r5 -;; locgrh %r2, %r3 -;; lrv %r2, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; llilh %r4, 65535 +;; agr %r3, %r4 +;; lghi %r4, 0 +;; clgr %r5, %r2 +;; locgrh %r3, %r4 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 47f15780d729..32fbdfd47665 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -41,7 +41,9 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r8, %r15, 64(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -52,32 +54,43 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 +;; llgfr %r7, %r2 ;; lg %r5, 8(%r4) -;; lgr %r8, %r4 -;; lgr %r4, %r2 -;; ag %r4, 0(%r8) -;; lghi %r14, 0 -;; clgr %r2, %r5 -;; locgrhe %r4, %r14 -;; stc %r3, 0(%r4) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r7 +;; lghi %r6, 0 +;; clgr %r7, %r5 +;; locgrhe %r2, %r6 +;; stc %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r8, %r15, 64(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lg %r5, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; lghi %r3, 0 -;; clgr %r2, %r5 -;; locgrhe %r4, %r3 -;; llc %r2, 0(%r4) +;; llgfr %r6, %r2 +;; lg %r4, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r5, %r3, %r6 +;; lghi %r2, 0 +;; clgr %r6, %r4 +;; locgrhe %r5, %r2 +;; llc %r2, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat index 342e1136474d..a71acb56f3a5 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -54,33 +53,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r6, %r2 +;; llgfr %r8, %r2 ;; lg %r5, 8(%r4) -;; lgr %r2, %r6 -;; ag %r2, 0(%r4) -;; aghi %r2, 4096 -;; lghi %r4, 0 -;; clgr %r6, %r5 -;; locgrh %r2, %r4 +;; lg %r4, 0(%r4) +;; agr %r4, %r8 +;; aghik %r2, %r4, 4096 +;; lghi %r7, 0 +;; clgr %r8, %r5 +;; locgrh %r2, %r7 ;; stc %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lg %r4, 8(%r3) -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghi %r5, 4096 +;; llgfr %r4, %r2 +;; lg %r5, 8(%r3) +;; lg %r2, 0(%r3) +;; agr %r2, %r4 +;; aghi %r2, 4096 ;; lghi %r3, 0 -;; clgr %r2, %r4 -;; locgrh %r5, %r3 -;; llc %r2, 0(%r5) +;; clgr %r4, %r5 +;; locgrh %r2, %r3 +;; llc %r2, 0(%r2) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 4a9bb7442205..ca2f67e4043c 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; stmg %r8, %r15, 64(%r15) ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -53,35 +52,35 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r7, %r2 -;; lg %r5, 8(%r4) -;; lgr %r2, %r7 -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; agr %r2, %r4 -;; lghi %r4, 0 -;; clgr %r7, %r5 -;; locgrh %r2, %r4 -;; stc %r3, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; llilh %r8, 65535 +;; agr %r4, %r8 +;; lghi %r8, 0 +;; clgr %r5, %r2 +;; locgrh %r4, %r8 +;; stc %r3, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r8, %r15, 64(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r4, %r2 -;; lg %r5, 8(%r3) -;; lgr %r2, %r4 -;; ag %r2, 0(%r3) -;; llilh %r3, 65535 -;; agr %r2, %r3 -;; lghi %r3, 0 -;; clgr %r4, %r5 -;; locgrh %r2, %r3 -;; llc %r2, 0(%r2) +;; llgfr %r5, %r2 +;; lg %r2, 8(%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; llilh %r4, 65535 +;; agr %r3, %r4 +;; lghi %r4, 0 +;; clgr %r5, %r2 +;; locgrh %r3, %r4 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat index 7b3b2bcc60b0..927e4e223bf5 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat @@ -48,9 +48,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r4, 4096 -;; strv %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; strv %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -61,14 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r3) -;; aghi %r5, -4100 -;; clgr %r2, %r5 +;; lg %r4, 8(%r3) +;; aghi %r4, -4100 +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r3, 4096 -;; lrv %r2, 0(%r3,%r2) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; lrv %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat index 50f1dc5ade66..ea04669dd9f6 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -41,7 +41,15 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } @@ -49,16 +57,17 @@ ;; lgr %r5, %r2 ;; algfi %r5, 4294901764 ;; jgnle .+2 # trap=heap_oob -;; lg %r14, 8(%r4) -;; clgr %r5, %r14 +;; lg %r6, 8(%r4) +;; clgr %r5, %r6 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; strv %r3, 0(%r4,%r2) +;; lg %r5, 0(%r4) +;; agr %r5, %r2 +;; llilh %r2, 65535 +;; strv %r3, 0(%r2,%r5) ;; jg label2 ;; block2: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; block3: ;; .word 0x0000 # trap=heap_oob @@ -67,16 +76,17 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; algfi %r5, 4294901764 +;; lgr %r4, %r2 +;; algfi %r4, 4294901764 ;; jgnle .+2 # trap=heap_oob -;; lg %r4, 8(%r3) -;; clgr %r5, %r4 +;; lg %r5, 8(%r3) +;; clgr %r4, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; llilh %r4, 65535 -;; lrv %r2, 0(%r4,%r2) +;; lg %r5, 0(%r3) +;; agr %r5, %r2 +;; llilh %r2, 65535 +;; lrv %r2, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat index b24ed73ac507..078eb88aacba 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat @@ -48,9 +48,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r4, 4096 -;; stc %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; stc %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -61,14 +62,15 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r3) -;; aghi %r5, -4097 -;; clgr %r2, %r5 +;; lg %r4, 8(%r3) +;; aghi %r4, -4097 +;; clgr %r2, %r4 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r3, 4096 -;; llc %r2, 0(%r3,%r2) +;; lg %r4, 0(%r3) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; llc %r2, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat index be59a9689720..72d33a988324 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -41,7 +41,15 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } @@ -49,16 +57,17 @@ ;; lgr %r5, %r2 ;; algfi %r5, 4294901761 ;; jgnle .+2 # trap=heap_oob -;; lg %r14, 8(%r4) -;; clgr %r5, %r14 +;; lg %r6, 8(%r4) +;; clgr %r5, %r6 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; stc %r3, 0(%r4,%r2) +;; lg %r5, 0(%r4) +;; agr %r5, %r2 +;; llilh %r2, 65535 +;; stc %r3, 0(%r2,%r5) ;; jg label2 ;; block2: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; block3: ;; .word 0x0000 # trap=heap_oob @@ -67,16 +76,17 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; algfi %r5, 4294901761 +;; lgr %r4, %r2 +;; algfi %r4, 4294901761 ;; jgnle .+2 # trap=heap_oob -;; lg %r4, 8(%r3) -;; clgr %r5, %r4 +;; lg %r5, 8(%r3) +;; clgr %r4, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; llilh %r4, 65535 -;; llc %r2, 0(%r4,%r2) +;; lg %r5, 0(%r3) +;; agr %r5, %r2 +;; llilh %r2, 65535 +;; llc %r2, 0(%r2,%r5) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat index 666728fe881f..e7b3162bd066 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -41,7 +41,9 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r8, %r15, 64(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } @@ -53,31 +55,42 @@ ;; unwind StackAlloc { size: 0 } ;; block0: ;; lg %r5, 8(%r4) -;; lgr %r8, %r4 -;; aghi %r5, -4 -;; lgr %r4, %r2 -;; ag %r4, 0(%r8) -;; lghi %r14, 0 -;; clgr %r2, %r5 -;; locgrh %r4, %r14 -;; strv %r3, 0(%r4) +;; aghik %r7, %r5, -4 +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 +;; lghi %r6, 0 +;; clgr %r2, %r7 +;; locgrh %r5, %r6 +;; strv %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r8, %r15, 64(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: ;; lg %r4, 8(%r3) -;; aghik %r5, %r4, -4 -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; lghi %r3, 0 -;; clgr %r2, %r5 -;; locgrh %r4, %r3 -;; lrv %r2, 0(%r4) +;; aghik %r6, %r4, -4 +;; lg %r3, 0(%r3) +;; agrk %r5, %r3, %r2 +;; lghi %r4, 0 +;; clgr %r2, %r6 +;; locgrh %r5, %r4 +;; lrv %r2, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat index b9c4fd6a9812..55facb05809a 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -55,17 +54,17 @@ ;; unwind StackAlloc { size: 0 } ;; block0: ;; lg %r5, 8(%r4) -;; aghik %r6, %r5, -4100 -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghi %r5, 4096 -;; lghi %r4, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r4 +;; aghik %r8, %r5, -4100 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghik %r5, %r4, 4096 +;; lghi %r7, 0 +;; clgr %r2, %r8 +;; locgrh %r5, %r7 ;; strv %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: @@ -74,13 +73,13 @@ ;; block0: ;; lg %r4, 8(%r3) ;; aghi %r4, -4100 -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghi %r5, 4096 -;; lghi %r3, 0 +;; lg %r5, 0(%r3) +;; agr %r5, %r2 +;; aghik %r3, %r5, 4096 +;; lghi %r5, 0 ;; clgr %r2, %r4 -;; locgrh %r5, %r3 -;; lrv %r2, 0(%r5) +;; locgrh %r3, %r5 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat index ee6cac682505..59622818dc2e 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -41,9 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } -;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; stmg %r9, %r15, 72(%r15) ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } @@ -53,37 +51,48 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r7, %r2 -;; algfi %r7, 4294901764 +;; lgr %r5, %r2 +;; algfi %r5, 4294901764 ;; jgnle .+2 # trap=heap_oob -;; lg %r5, 8(%r4) -;; ag %r2, 0(%r4) +;; lg %r9, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 ;; llilh %r4, 65535 -;; agrk %r4, %r2, %r4 -;; lghi %r2, 0 -;; clgr %r7, %r5 -;; locgrh %r4, %r2 -;; strv %r3, 0(%r4) +;; agr %r2, %r4 +;; lghi %r4, 0 +;; clgr %r5, %r9 +;; locgrh %r2, %r4 +;; strv %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r2 -;; algfi %r4, 4294901764 +;; lgr %r5, %r2 +;; algfi %r5, 4294901764 ;; jgnle .+2 # trap=heap_oob -;; lg %r5, 8(%r3) -;; ag %r2, 0(%r3) +;; lg %r9, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 ;; llilh %r3, 65535 -;; agrk %r3, %r2, %r3 +;; agrk %r4, %r2, %r3 ;; lghi %r2, 0 -;; clgr %r4, %r5 -;; locgrh %r3, %r2 -;; lrv %r2, 0(%r3) +;; clgr %r5, %r9 +;; locgrh %r4, %r2 +;; lrv %r2, 0(%r4) ;; jg label1 ;; block1: +;; lmg %r9, %r15, 72(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat index 907d9e6b9505..2516db71460e 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -41,7 +41,12 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } ;; unwind SaveReg { clobber_offset: 96, reg: p12i } ;; unwind SaveReg { clobber_offset: 104, reg: p13i } @@ -49,36 +54,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r4) -;; lgr %r11, %r2 -;; ag %r11, 0(%r4) -;; lghi %r13, 0 -;; clgr %r2, %r5 -;; locgrhe %r11, %r13 -;; stc %r3, 0(%r11) +;; lg %r6, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 +;; lghi %r14, 0 +;; clgr %r2, %r6 +;; locgrhe %r5, %r14 +;; stc %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) -;; unwind SaveReg { clobber_offset: 88, reg: p11i } -;; unwind SaveReg { clobber_offset: 96, reg: p12i } -;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; stmg %r14, %r15, 112(%r15) ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r3) -;; lgr %r11, %r2 -;; ag %r11, 0(%r3) -;; lghi %r4, 0 -;; clgr %r2, %r5 -;; locgrhe %r11, %r4 -;; llc %r2, 0(%r11) +;; lg %r14, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r4, %r3, %r2 +;; lghi %r5, 0 +;; clgr %r2, %r14 +;; locgrhe %r4, %r5 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r14, %r15, 112(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat index 9331c9404892..20876eabc832 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -55,17 +54,17 @@ ;; unwind StackAlloc { size: 0 } ;; block0: ;; lg %r5, 8(%r4) -;; aghik %r6, %r5, -4097 -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghi %r5, 4096 -;; lghi %r4, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r4 +;; aghik %r8, %r5, -4097 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghik %r5, %r4, 4096 +;; lghi %r7, 0 +;; clgr %r2, %r8 +;; locgrh %r5, %r7 ;; stc %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: @@ -74,13 +73,13 @@ ;; block0: ;; lg %r4, 8(%r3) ;; aghi %r4, -4097 -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghi %r5, 4096 -;; lghi %r3, 0 +;; lg %r5, 0(%r3) +;; agr %r5, %r2 +;; aghik %r3, %r5, 4096 +;; lghi %r5, 0 ;; clgr %r2, %r4 -;; locgrh %r5, %r3 -;; llc %r2, 0(%r5) +;; locgrh %r3, %r5 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 8fcb0d779b38..aa582fced9a9 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -41,9 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r7, %r15, 56(%r15) -;; unwind SaveReg { clobber_offset: 56, reg: p7i } -;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; stmg %r9, %r15, 72(%r15) ;; unwind SaveReg { clobber_offset: 72, reg: p9i } ;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } @@ -53,37 +51,48 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r7, %r2 -;; algfi %r7, 4294901761 +;; lgr %r5, %r2 +;; algfi %r5, 4294901761 ;; jgnle .+2 # trap=heap_oob -;; lg %r5, 8(%r4) -;; ag %r2, 0(%r4) +;; lg %r9, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 ;; llilh %r4, 65535 -;; agrk %r4, %r2, %r4 -;; lghi %r2, 0 -;; clgr %r7, %r5 -;; locgrh %r4, %r2 -;; stc %r3, 0(%r4) +;; agr %r2, %r4 +;; lghi %r4, 0 +;; clgr %r5, %r9 +;; locgrh %r2, %r4 +;; stc %r3, 0(%r2) ;; jg label1 ;; block1: -;; lmg %r7, %r15, 56(%r15) +;; lmg %r9, %r15, 72(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r9, %r15, 72(%r15) +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r4, %r2 -;; algfi %r4, 4294901761 +;; lgr %r5, %r2 +;; algfi %r5, 4294901761 ;; jgnle .+2 # trap=heap_oob -;; lg %r5, 8(%r3) -;; ag %r2, 0(%r3) +;; lg %r9, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 ;; llilh %r3, 65535 -;; agrk %r3, %r2, %r3 +;; agrk %r4, %r2, %r3 ;; lghi %r2, 0 -;; clgr %r4, %r5 -;; locgrh %r3, %r2 -;; llc %r2, 0(%r3) +;; clgr %r5, %r9 +;; locgrh %r4, %r2 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: +;; lmg %r9, %r15, 72(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat index c034f17edad9..76ab2e750760 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat @@ -47,9 +47,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r4, 4096 -;; strv %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; strv %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -60,13 +61,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r4, 8(%r3) -;; clgr %r2, %r4 +;; lg %r5, 8(%r3) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r3, 4096 -;; lrv %r2, 0(%r3,%r2) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; lghi %r4, 4096 +;; lrv %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat index 27825b12e20d..ef080b4b88d5 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -47,9 +47,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; strv %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; llilh %r5, 65535 +;; strv %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -60,13 +61,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r4, 8(%r3) -;; clgr %r2, %r4 +;; lg %r5, 8(%r3) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; llilh %r3, 65535 -;; lrv %r2, 0(%r3,%r2) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; llilh %r4, 65535 +;; lrv %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat index 076de1172519..8e2d6e944a92 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat @@ -47,9 +47,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r4, 4096 -;; stc %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 4096 +;; stc %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -60,13 +61,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r4, 8(%r3) -;; clgr %r2, %r4 +;; lg %r5, 8(%r3) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r3, 4096 -;; llc %r2, 0(%r3,%r2) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; lghi %r4, 4096 +;; llc %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat index d4473f5877f8..cc758ab83039 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -47,9 +47,10 @@ ;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; llilh %r4, 65535 -;; stc %r3, 0(%r4,%r2) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; llilh %r5, 65535 +;; stc %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -60,13 +61,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r4, 8(%r3) -;; clgr %r2, %r4 +;; lg %r5, 8(%r3) +;; clgr %r2, %r5 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; llilh %r3, 65535 -;; llc %r2, 0(%r3,%r2) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; llilh %r4, 65535 +;; llc %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index 7ffb39d00c00..be7704012f59 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -41,7 +41,12 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } ;; unwind SaveReg { clobber_offset: 96, reg: p12i } ;; unwind SaveReg { clobber_offset: 104, reg: p13i } @@ -49,36 +54,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r4) -;; lgr %r11, %r2 -;; ag %r11, 0(%r4) -;; lghi %r13, 0 -;; clgr %r2, %r5 -;; locgrh %r11, %r13 -;; strv %r3, 0(%r11) +;; lg %r6, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 +;; lghi %r14, 0 +;; clgr %r2, %r6 +;; locgrh %r5, %r14 +;; strv %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) -;; unwind SaveReg { clobber_offset: 88, reg: p11i } -;; unwind SaveReg { clobber_offset: 96, reg: p12i } -;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; stmg %r14, %r15, 112(%r15) ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r3) -;; lgr %r11, %r2 -;; ag %r11, 0(%r3) -;; lghi %r4, 0 -;; clgr %r2, %r5 -;; locgrh %r11, %r4 -;; lrv %r2, 0(%r11) +;; lg %r14, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r4, %r3, %r2 +;; lghi %r5, 0 +;; clgr %r2, %r14 +;; locgrh %r4, %r5 +;; lrv %r2, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r14, %r15, 112(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat index 1c2070ffc42c..f11d7bf61098 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -54,13 +54,13 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r6, 8(%r4) -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghi %r5, 4096 -;; lghi %r14, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r14 +;; lg %r7, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghik %r5, %r4, 4096 +;; lghi %r6, 0 +;; clgr %r2, %r7 +;; locgrh %r5, %r6 ;; strv %r3, 0(%r5) ;; jg label1 ;; block1: @@ -69,20 +69,28 @@ ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r14, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; aghi %r4, 4096 -;; lghi %r5, 0 -;; clgr %r2, %r14 -;; locgrh %r4, %r5 -;; lrv %r2, 0(%r4) +;; lg %r6, 8(%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r5, %r3, 4096 +;; lghi %r4, 0 +;; clgr %r2, %r6 +;; locgrh %r5, %r4 +;; lrv %r2, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat index 3d8f014252ba..ebd6f401c062 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -54,45 +53,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r7, 8(%r4) -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) +;; lg %r8, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 ;; llilh %r4, 65535 ;; agr %r5, %r4 -;; lghi %r6, 0 -;; clgr %r2, %r7 -;; locgrh %r5, %r6 +;; lghi %r7, 0 +;; clgr %r2, %r8 +;; locgrh %r5, %r7 ;; strv %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } -;; unwind SaveReg { clobber_offset: 56, reg: p7i } -;; unwind SaveReg { clobber_offset: 64, reg: p8i } -;; unwind SaveReg { clobber_offset: 72, reg: p9i } -;; unwind SaveReg { clobber_offset: 80, reg: p10i } -;; unwind SaveReg { clobber_offset: 88, reg: p11i } -;; unwind SaveReg { clobber_offset: 96, reg: p12i } -;; unwind SaveReg { clobber_offset: 104, reg: p13i } -;; unwind SaveReg { clobber_offset: 112, reg: p14i } -;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r6, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) +;; lg %r4, 8(%r3) +;; lg %r5, 0(%r3) +;; agr %r5, %r2 ;; llilh %r3, 65535 -;; agrk %r5, %r4, %r3 -;; lghi %r4, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r4 -;; lrv %r2, 0(%r5) +;; agrk %r3, %r5, %r3 +;; lghi %r5, 0 +;; clgr %r2, %r4 +;; locgrh %r3, %r5 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 175a19bf50e8..a7f62a297712 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -41,7 +41,12 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } ;; unwind SaveReg { clobber_offset: 88, reg: p11i } ;; unwind SaveReg { clobber_offset: 96, reg: p12i } ;; unwind SaveReg { clobber_offset: 104, reg: p13i } @@ -49,36 +54,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r4) -;; lgr %r11, %r2 -;; ag %r11, 0(%r4) -;; lghi %r13, 0 -;; clgr %r2, %r5 -;; locgrhe %r11, %r13 -;; stc %r3, 0(%r11) +;; lg %r6, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 +;; lghi %r14, 0 +;; clgr %r2, %r6 +;; locgrhe %r5, %r14 +;; stc %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r11, %r15, 88(%r15) -;; unwind SaveReg { clobber_offset: 88, reg: p11i } -;; unwind SaveReg { clobber_offset: 96, reg: p12i } -;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; stmg %r14, %r15, 112(%r15) ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r5, 8(%r3) -;; lgr %r11, %r2 -;; ag %r11, 0(%r3) -;; lghi %r4, 0 -;; clgr %r2, %r5 -;; locgrhe %r11, %r4 -;; llc %r2, 0(%r11) +;; lg %r14, 8(%r3) +;; lg %r3, 0(%r3) +;; agrk %r4, %r3, %r2 +;; lghi %r5, 0 +;; clgr %r2, %r14 +;; locgrhe %r4, %r5 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: -;; lmg %r11, %r15, 88(%r15) +;; lmg %r14, %r15, 112(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat index 9a62871d7006..5fbf8ea239ef 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -54,13 +54,13 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r6, 8(%r4) -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghi %r5, 4096 -;; lghi %r14, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r14 +;; lg %r7, 8(%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghik %r5, %r4, 4096 +;; lghi %r6, 0 +;; clgr %r2, %r7 +;; locgrh %r5, %r6 ;; stc %r3, 0(%r5) ;; jg label1 ;; block1: @@ -69,20 +69,28 @@ ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r14, %r15, 112(%r15) +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } ;; unwind SaveReg { clobber_offset: 112, reg: p14i } ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r14, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; aghi %r4, 4096 -;; lghi %r5, 0 -;; clgr %r2, %r14 -;; locgrh %r4, %r5 -;; llc %r2, 0(%r4) +;; lg %r6, 8(%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r5, %r3, 4096 +;; lghi %r4, 0 +;; clgr %r2, %r6 +;; locgrh %r5, %r4 +;; llc %r2, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r14, %r15, 112(%r15) +;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 2f861cc1f10a..dba033d3a90a 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -41,8 +41,7 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; stmg %r7, %r15, 56(%r15) ;; unwind SaveReg { clobber_offset: 56, reg: p7i } ;; unwind SaveReg { clobber_offset: 64, reg: p8i } ;; unwind SaveReg { clobber_offset: 72, reg: p9i } @@ -54,45 +53,33 @@ ;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r7, 8(%r4) -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) +;; lg %r8, 8(%r4) +;; lg %r4, 0(%r4) +;; agrk %r5, %r4, %r2 ;; llilh %r4, 65535 ;; agr %r5, %r4 -;; lghi %r6, 0 -;; clgr %r2, %r7 -;; locgrh %r5, %r6 +;; lghi %r7, 0 +;; clgr %r2, %r8 +;; locgrh %r5, %r7 ;; stc %r3, 0(%r5) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) +;; lmg %r7, %r15, 56(%r15) ;; br %r14 ;; ;; function u0:1: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } -;; stmg %r6, %r15, 48(%r15) -;; unwind SaveReg { clobber_offset: 48, reg: p6i } -;; unwind SaveReg { clobber_offset: 56, reg: p7i } -;; unwind SaveReg { clobber_offset: 64, reg: p8i } -;; unwind SaveReg { clobber_offset: 72, reg: p9i } -;; unwind SaveReg { clobber_offset: 80, reg: p10i } -;; unwind SaveReg { clobber_offset: 88, reg: p11i } -;; unwind SaveReg { clobber_offset: 96, reg: p12i } -;; unwind SaveReg { clobber_offset: 104, reg: p13i } -;; unwind SaveReg { clobber_offset: 112, reg: p14i } -;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lg %r6, 8(%r3) -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) +;; lg %r4, 8(%r3) +;; lg %r5, 0(%r3) +;; agr %r5, %r2 ;; llilh %r3, 65535 -;; agrk %r5, %r4, %r3 -;; lghi %r4, 0 -;; clgr %r2, %r6 -;; locgrh %r5, %r4 -;; llc %r2, 0(%r5) +;; agrk %r3, %r5, %r3 +;; lghi %r5, 0 +;; clgr %r2, %r4 +;; locgrh %r3, %r5 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: -;; lmg %r6, %r15, 48(%r15) ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat index dff24197b473..dde65b188d7d 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat @@ -41,14 +41,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; clgfi %r4, 268431356 +;; llgfr %r5, %r2 +;; clgfi %r5, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; lghi %r2, 4096 -;; strv %r3, 0(%r2,%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; lghi %r5, 4096 +;; strv %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -59,15 +59,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; clgfi %r3, 268431356 +;; llgfr %r5, %r2 +;; clgfi %r5, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; lgr %r4, %r5 -;; ag %r3, 0(%r4) -;; lghi %r2, 4096 -;; lrv %r2, 0(%r2,%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; lghi %r4, 4096 +;; lrv %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat index 072dfed1e7c6..1a0a4d7f7c95 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat @@ -41,14 +41,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; clgfi %r4, 268431359 +;; llgfr %r5, %r2 +;; clgfi %r5, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r4, 0(%r5) -;; lghi %r2, 4096 -;; stc %r3, 0(%r2,%r4) +;; lg %r4, 0(%r4) +;; agr %r4, %r5 +;; lghi %r5, 4096 +;; stc %r3, 0(%r5,%r4) ;; jg label2 ;; block2: ;; br %r14 @@ -59,15 +59,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; clgfi %r3, 268431359 +;; llgfr %r5, %r2 +;; clgfi %r5, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; lgr %r4, %r5 -;; ag %r3, 0(%r4) -;; lghi %r2, 4096 -;; llc %r2, 0(%r2,%r3) +;; lg %r3, 0(%r3) +;; agr %r3, %r5 +;; lghi %r4, 4096 +;; llc %r2, 0(%r4,%r3) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat index 502fc9856a03..b2e1740f68d4 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -41,13 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r5, %r2 -;; lgr %r2, %r4 -;; lgr %r4, %r5 -;; ag %r4, 0(%r2) -;; lghi %r2, 0 -;; clgfi %r5, 268435452 -;; locgrh %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 0 +;; clgfi %r2, 268435452 +;; locgrh %r4, %r5 ;; strv %r3, 0(%r4) ;; jg label1 ;; block1: @@ -57,14 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r5, %r2 -;; lgr %r4, %r3 -;; lgr %r3, %r5 -;; ag %r3, 0(%r4) -;; lghi %r2, 0 -;; clgfi %r5, 268435452 -;; locgrh %r3, %r2 -;; lrv %r2, 0(%r3) +;; llgfr %r2, %r2 +;; lg %r3, 0(%r3) +;; agrk %r4, %r3, %r2 +;; lghi %r3, 0 +;; clgfi %r2, 268435452 +;; locgrh %r4, %r3 +;; lrv %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat index 8366b8251c3f..1000414b1f3f 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -39,18 +39,30 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 -;; lghi %r5, 0 -;; clgfi %r2, 268431356 -;; locgrh %r4, %r5 -;; strv %r3, 0(%r4) +;; llgfr %r6, %r2 +;; lg %r4, 0(%r4) +;; agr %r4, %r6 +;; aghik %r5, %r4, 4096 +;; lghi %r2, 0 +;; clgfi %r6, 268431356 +;; locgrh %r5, %r2 +;; strv %r3, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: @@ -58,13 +70,13 @@ ;; unwind StackAlloc { size: 0 } ;; block0: ;; llgfr %r2, %r2 -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; aghi %r4, 4096 -;; lghi %r3, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r5, %r3, 4096 +;; lghi %r4, 0 ;; clgfi %r2, 268431356 -;; locgrh %r4, %r3 -;; lrv %r2, 0(%r4) +;; locgrh %r5, %r4 +;; lrv %r2, 0(%r5) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat index 3594b2b9e64c..58d3daabe08f 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -41,13 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r5, %r2 -;; lgr %r2, %r4 -;; lgr %r4, %r5 -;; ag %r4, 0(%r2) -;; lghi %r2, 0 -;; clgfi %r5, 268435455 -;; locgrh %r4, %r2 +;; llgfr %r2, %r2 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; lghi %r5, 0 +;; clgfi %r2, 268435455 +;; locgrh %r4, %r5 ;; stc %r3, 0(%r4) ;; jg label1 ;; block1: @@ -57,14 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r5, %r2 -;; lgr %r4, %r3 -;; lgr %r3, %r5 -;; ag %r3, 0(%r4) -;; lghi %r2, 0 -;; clgfi %r5, 268435455 -;; locgrh %r3, %r2 -;; llc %r2, 0(%r3) +;; llgfr %r2, %r2 +;; lg %r3, 0(%r3) +;; agrk %r4, %r3, %r2 +;; lghi %r3, 0 +;; clgfi %r2, 268435455 +;; locgrh %r4, %r3 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat index c51890d0a287..05c9ddee1697 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -39,18 +39,30 @@ ;; function u0:0: ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } +;; stmg %r6, %r15, 48(%r15) +;; unwind SaveReg { clobber_offset: 48, reg: p6i } +;; unwind SaveReg { clobber_offset: 56, reg: p7i } +;; unwind SaveReg { clobber_offset: 64, reg: p8i } +;; unwind SaveReg { clobber_offset: 72, reg: p9i } +;; unwind SaveReg { clobber_offset: 80, reg: p10i } +;; unwind SaveReg { clobber_offset: 88, reg: p11i } +;; unwind SaveReg { clobber_offset: 96, reg: p12i } +;; unwind SaveReg { clobber_offset: 104, reg: p13i } +;; unwind SaveReg { clobber_offset: 112, reg: p14i } +;; unwind SaveReg { clobber_offset: 120, reg: p15i } ;; unwind StackAlloc { size: 0 } ;; block0: -;; llgfr %r2, %r2 -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 -;; lghi %r5, 0 -;; clgfi %r2, 268431359 -;; locgrh %r4, %r5 -;; stc %r3, 0(%r4) +;; llgfr %r6, %r2 +;; lg %r4, 0(%r4) +;; agr %r4, %r6 +;; aghik %r5, %r4, 4096 +;; lghi %r2, 0 +;; clgfi %r6, 268431359 +;; locgrh %r5, %r2 +;; stc %r3, 0(%r5) ;; jg label1 ;; block1: +;; lmg %r6, %r15, 48(%r15) ;; br %r14 ;; ;; function u0:1: @@ -58,13 +70,13 @@ ;; unwind StackAlloc { size: 0 } ;; block0: ;; llgfr %r2, %r2 -;; lgr %r4, %r2 -;; ag %r4, 0(%r3) -;; aghi %r4, 4096 -;; lghi %r3, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r5, %r3, 4096 +;; lghi %r4, 0 ;; clgfi %r2, 268431359 -;; locgrh %r4, %r3 -;; llc %r2, 0(%r4) +;; locgrh %r5, %r4 +;; llc %r2, 0(%r5) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat index 794a1572764f..c82925958375 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat @@ -41,12 +41,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lgr %r2, %r5 -;; ag %r4, 0(%r2) -;; lghi %r5, 4096 -;; strv %r3, 0(%r5,%r4) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r4) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; strv %r3, 0(%r2,%r5) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +54,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r2, %r5 -;; ag %r3, 0(%r2) -;; lghi %r4, 4096 -;; lrv %r2, 0(%r4,%r3) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; lrv %r2, 0(%r2,%r5) ;; jg label1 ;; block1: -;; br %r14 \ No newline at end of file +;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat index bbeb3ae418b5..291393224add 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat @@ -41,12 +41,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lgr %r2, %r5 -;; ag %r4, 0(%r2) -;; lghi %r5, 4096 -;; stc %r3, 0(%r5,%r4) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r4) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; stc %r3, 0(%r2,%r5) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +54,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r2, %r5 -;; ag %r3, 0(%r2) -;; lghi %r4, 4096 -;; llc %r2, 0(%r4,%r3) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; llc %r2, 0(%r2,%r5) ;; jg label1 ;; block1: -;; br %r14 \ No newline at end of file +;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat index e76eae59fe0b..bb896678129c 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,12 +41,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lgr %r2, %r5 -;; ag %r4, 0(%r2) -;; lghi %r5, 4096 -;; strv %r3, 0(%r5,%r4) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r4) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; strv %r3, 0(%r2,%r5) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +54,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r2, %r5 -;; ag %r3, 0(%r2) -;; lghi %r4, 4096 -;; lrv %r2, 0(%r4,%r3) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; lrv %r2, 0(%r2,%r5) ;; jg label1 ;; block1: -;; br %r14 \ No newline at end of file +;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat index 94614dc1c641..461f2e2529ae 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,12 +41,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r4 -;; llgfr %r4, %r2 -;; lgr %r2, %r5 -;; ag %r4, 0(%r2) -;; lghi %r5, 4096 -;; stc %r3, 0(%r5,%r4) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r4) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; stc %r3, 0(%r2,%r5) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +54,11 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r3 -;; llgfr %r3, %r2 -;; lgr %r2, %r5 -;; ag %r3, 0(%r2) -;; lghi %r4, 4096 -;; llc %r2, 0(%r4,%r3) +;; llgfr %r5, %r2 +;; lg %r2, 0(%r3) +;; agrk %r5, %r2, %r5 +;; lghi %r2, 4096 +;; llc %r2, 0(%r2,%r5) ;; jg label1 ;; block1: -;; br %r14 \ No newline at end of file +;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat index 56cce5cd1cef..4b627be146c2 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat @@ -44,9 +44,10 @@ ;; clgfi %r2, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r5, 4096 -;; strv %r3, 0(%r5,%r2) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 +;; lghi %r4, 4096 +;; strv %r3, 0(%r4,%r2) ;; jg label2 ;; block2: ;; br %r14 @@ -60,9 +61,10 @@ ;; clgfi %r2, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r5, 4096 -;; lrv %r2, 0(%r5,%r2) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 +;; lghi %r3, 4096 +;; lrv %r2, 0(%r3,%r2) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat index 99d2a21705f1..0b6de10334b4 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat @@ -44,9 +44,10 @@ ;; clgfi %r2, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r5, 4096 -;; stc %r3, 0(%r5,%r2) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 +;; lghi %r4, 4096 +;; stc %r3, 0(%r4,%r2) ;; jg label2 ;; block2: ;; br %r14 @@ -60,9 +61,10 @@ ;; clgfi %r2, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r5, 4096 -;; llc %r2, 0(%r5,%r2) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 +;; lghi %r3, 4096 +;; llc %r2, 0(%r3,%r2) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat index 05c6839d5969..b1073fcff8b2 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -41,12 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; lghi %r4, 0 +;; lg %r5, 0(%r4) +;; agrk %r4, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435452 -;; locgrh %r5, %r4 -;; strv %r3, 0(%r5) +;; locgrh %r4, %r5 +;; strv %r3, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +55,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; lghi %r4, 0 +;; lg %r5, 0(%r3) +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435452 -;; locgrh %r5, %r4 -;; lrv %r2, 0(%r5) +;; locgrh %r3, %r5 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat index 2851ab44629e..59ad7b0da23b 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,9 +41,9 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghi %r4, 4096 ;; lghi %r5, 0 ;; clgfi %r2, 268431356 ;; locgrh %r4, %r5 @@ -56,13 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghik %r3, %r5, 4096 -;; lghi %r5, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r4, %r3, 4096 +;; lghi %r3, 0 ;; clgfi %r2, 268431356 -;; locgrh %r3, %r5 -;; lrv %r2, 0(%r3) +;; locgrh %r4, %r3 +;; lrv %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat index 0df2d11cbba4..ffdcb39a8924 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -41,12 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; lghi %r4, 0 +;; lg %r5, 0(%r4) +;; agrk %r4, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435455 -;; locgrh %r5, %r4 -;; stc %r3, 0(%r5) +;; locgrh %r4, %r5 +;; stc %r3, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +55,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; lghi %r4, 0 +;; lg %r5, 0(%r3) +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435455 -;; locgrh %r5, %r4 -;; llc %r2, 0(%r5) +;; locgrh %r3, %r5 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat index 5b8b480191b1..9380455a2fb3 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,9 +41,9 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghi %r4, 4096 ;; lghi %r5, 0 ;; clgfi %r2, 268431359 ;; locgrh %r4, %r5 @@ -56,13 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghik %r3, %r5, 4096 -;; lghi %r5, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r4, %r3, 4096 +;; lghi %r3, 0 ;; clgfi %r2, 268431359 -;; locgrh %r3, %r5 -;; llc %r2, 0(%r3) +;; locgrh %r4, %r3 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat index b74f04d06f9b..59af6deba717 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat @@ -44,9 +44,10 @@ ;; clgfi %r2, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r5, 4096 -;; strv %r3, 0(%r5,%r2) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 +;; lghi %r4, 4096 +;; strv %r3, 0(%r4,%r2) ;; jg label2 ;; block2: ;; br %r14 @@ -60,9 +61,10 @@ ;; clgfi %r2, 268431356 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r5, 4096 -;; lrv %r2, 0(%r5,%r2) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 +;; lghi %r3, 4096 +;; lrv %r2, 0(%r3,%r2) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat index 1cc7a766fae8..2a102639d434 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat @@ -44,9 +44,10 @@ ;; clgfi %r2, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r4) -;; lghi %r5, 4096 -;; stc %r3, 0(%r5,%r2) +;; lg %r4, 0(%r4) +;; agrk %r2, %r4, %r2 +;; lghi %r4, 4096 +;; stc %r3, 0(%r4,%r2) ;; jg label2 ;; block2: ;; br %r14 @@ -60,9 +61,10 @@ ;; clgfi %r2, 268431359 ;; jgh label3 ; jg label1 ;; block1: -;; ag %r2, 0(%r3) -;; lghi %r5, 4096 -;; llc %r2, 0(%r5,%r2) +;; lg %r3, 0(%r3) +;; agrk %r2, %r3, %r2 +;; lghi %r3, 4096 +;; llc %r2, 0(%r3,%r2) ;; jg label2 ;; block2: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index 486133ed00ff..e5c103ed42a4 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -41,12 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; lghi %r4, 0 +;; lg %r5, 0(%r4) +;; agrk %r4, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435452 -;; locgrh %r5, %r4 -;; strv %r3, 0(%r5) +;; locgrh %r4, %r5 +;; strv %r3, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +55,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; lghi %r4, 0 +;; lg %r5, 0(%r3) +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435452 -;; locgrh %r5, %r4 -;; lrv %r2, 0(%r5) +;; locgrh %r3, %r5 +;; lrv %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat index a71863212d37..7ac23d4685c7 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat @@ -41,9 +41,9 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghi %r4, 4096 ;; lghi %r5, 0 ;; clgfi %r2, 268431356 ;; locgrh %r4, %r5 @@ -56,13 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghik %r3, %r5, 4096 -;; lghi %r5, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r4, %r3, 4096 +;; lghi %r3, 0 ;; clgfi %r2, 268431356 -;; locgrh %r3, %r5 -;; lrv %r2, 0(%r3) +;; locgrh %r4, %r3 +;; lrv %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 89aec920be0d..2b8508130ec4 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -41,12 +41,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; lghi %r4, 0 +;; lg %r5, 0(%r4) +;; agrk %r4, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435455 -;; locgrh %r5, %r4 -;; stc %r3, 0(%r5) +;; locgrh %r4, %r5 +;; stc %r3, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 @@ -55,12 +55,12 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; lghi %r4, 0 +;; lg %r5, 0(%r3) +;; agrk %r3, %r5, %r2 +;; lghi %r5, 0 ;; clgfi %r2, 268435455 -;; locgrh %r5, %r4 -;; llc %r2, 0(%r5) +;; locgrh %r3, %r5 +;; llc %r2, 0(%r3) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat index 548feaca8244..e6b78f78abb2 100644 --- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat +++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat @@ -41,9 +41,9 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r4) -;; aghik %r4, %r5, 4096 +;; lg %r4, 0(%r4) +;; agr %r4, %r2 +;; aghi %r4, 4096 ;; lghi %r5, 0 ;; clgfi %r2, 268431359 ;; locgrh %r4, %r5 @@ -56,13 +56,13 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 } ;; unwind StackAlloc { size: 0 } ;; block0: -;; lgr %r5, %r2 -;; ag %r5, 0(%r3) -;; aghik %r3, %r5, 4096 -;; lghi %r5, 0 +;; lg %r3, 0(%r3) +;; agr %r3, %r2 +;; aghik %r4, %r3, 4096 +;; lghi %r3, 0 ;; clgfi %r2, 268431359 -;; locgrh %r3, %r5 -;; llc %r2, 0(%r3) +;; locgrh %r4, %r3 +;; llc %r2, 0(%r4) ;; jg label1 ;; block1: ;; br %r14 diff --git a/cranelift/filetests/filetests/isa/x64/fcvt.clif b/cranelift/filetests/filetests/isa/x64/fcvt.clif index 2533ccb85fa8..75ee658ccdff 100644 --- a/cranelift/filetests/filetests/isa/x64/fcvt.clif +++ b/cranelift/filetests/filetests/isa/x64/fcvt.clif @@ -285,12 +285,29 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64): ; cvtsi2ss %xmm6, %r8, %xmm6 ; uninit %xmm7 ; xorps %xmm7, %xmm7, %xmm7 -; movl %edx, %r8d -; cvtsi2ss %xmm7, %r8, %xmm7 -; u64_to_f32_seq %rcx, %xmm4, %r8, %rdx +; movl %edx, %edx +; cvtsi2ss %xmm7, %rdx, %xmm7 +; testq %rcx, %rcx +; jl label2; j label1 +; block1: +; uninit %xmm3 +; xorps %xmm3, %xmm3, %xmm3 +; cvtsi2ss %xmm3, %rcx, %xmm3 +; jmp label3 +; block2: +; movq %rcx, %rdx +; shrq $1, %rdx, %rdx +; andq %rcx, $1, %rcx +; orq %rdx, %rcx, %rdx +; uninit %xmm3 +; xorps %xmm3, %xmm3, %xmm3 +; cvtsi2ss %xmm3, %rdx, %xmm3 +; addss %xmm3, %xmm3, %xmm3 +; jmp label3 +; block3: ; addss %xmm0, %xmm6, %xmm0 ; addss %xmm0, %xmm7, %xmm0 -; addss %xmm0, %xmm4, %xmm0 +; addss %xmm0, %xmm3, %xmm0 ; movq %rbp, %rsp ; popq %rbp ; ret @@ -307,22 +324,26 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64): ; movzwq %si, %r8 ; cvtsi2ssq %r8, %xmm6 ; xorps %xmm7, %xmm7 -; movl %edx, %r8d -; cvtsi2ssq %r8, %xmm7 -; cmpq $0, %rcx -; jl 0x3b -; cvtsi2ssq %rcx, %xmm4 -; jmp 0x55 -; movq %rcx, %r8 -; shrq $1, %r8 +; movl %edx, %edx +; cvtsi2ssq %rdx, %xmm7 +; testq %rcx, %rcx +; jl 0x3c +; block2: ; offset 0x2f +; xorps %xmm3, %xmm3 +; cvtsi2ssq %rcx, %xmm3 +; jmp 0x56 +; block3: ; offset 0x3c ; movq %rcx, %rdx -; andq $1, %rdx -; orq %r8, %rdx -; cvtsi2ssq %rdx, %xmm4 -; addss %xmm4, %xmm4 +; shrq $1, %rdx +; andq $1, %rcx +; orq %rcx, %rdx +; xorps %xmm3, %xmm3 +; cvtsi2ssq %rdx, %xmm3 +; addss %xmm3, %xmm3 +; block4: ; offset 0x56 ; addss %xmm6, %xmm0 ; addss %xmm7, %xmm0 -; addss %xmm4, %xmm0 +; addss %xmm3, %xmm0 ; movq %rbp, %rsp ; popq %rbp ; retq diff --git a/cranelift/filetests/filetests/isa/x64/table.clif b/cranelift/filetests/filetests/isa/x64/table.clif index c3c217fc9a06..1b69a3e85999 100644 --- a/cranelift/filetests/filetests/isa/x64/table.clif +++ b/cranelift/filetests/filetests/isa/x64/table.clif @@ -1,6 +1,7 @@ test compile precise-output set enable_safepoints=true set enable_table_access_spectre_mitigation=true +set opt_level=speed target x86_64 @@ -24,8 +25,8 @@ block0(v0: i32, v1: r64, v2: i64): ; cmpl %r11d, %edi ; jnb label2; j label1 ; block1: -; movl %edi, %ecx ; movq 0(%rdx), %rax +; movl %edi, %ecx ; lea 0(%rax,%rcx,1), %rcx ; cmpl %r11d, %edi ; cmovnbq %rax, %rcx, %rcx @@ -45,8 +46,8 @@ block0(v0: i32, v1: r64, v2: i64): ; cmpl %r11d, %edi ; jae 0x28 ; block2: ; offset 0x11 -; movl %edi, %ecx ; movq (%rdx), %rax +; movl %edi, %ecx ; addq %rax, %rcx ; cmpl %r11d, %edi ; cmovaeq %rax, %rcx diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat index 23825be3f765..2d28a4356113 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -45,17 +45,18 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq %r8, %r11 -;; addq %r11, const(0), %r11 +;; movl %edi, %edi +;; movq %rdi, %rax +;; addq %rax, const(0), %rax ;; jb #trap=heap_oob -;; movq 8(%rdx), %rdi -;; cmpq %rdi, %r11 +;; movq 8(%rdx), %rcx +;; cmpq %rcx, %rax ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rdx), %r8 -;; movl $-65536, %eax -;; movl %esi, 0(%r8,%rax,1) +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %rcx +;; movl $-65536, %edx +;; movl %esi, 0(%rcx,%rdx,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -70,17 +71,20 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq %r8, %r11 -;; addq %r11, const(0), %r11 +;; movq %rsi, %rax +;; movl %edi, %esi +;; movq %rsi, %rdi +;; addq %rdi, const(0), %rdi ;; jb #trap=heap_oob -;; movq 8(%rsi), %rdi -;; cmpq %rdi, %r11 +;; movq %rax, %rdx +;; movq 8(%rdx), %rax +;; cmpq %rax, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rsi), %r8 -;; movl $-65536, %eax -;; movl 0(%r8,%rax,1), %eax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rsi,1), %rcx +;; movl $-65536, %edx +;; movl 0(%rcx,%rdx,1), %eax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat index 187b535a986e..cf9bb34ce3f1 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -45,17 +45,18 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq %r8, %r11 -;; addq %r11, const(0), %r11 +;; movl %edi, %edi +;; movq %rdi, %rax +;; addq %rax, const(0), %rax ;; jb #trap=heap_oob -;; movq 8(%rdx), %rdi -;; cmpq %rdi, %r11 +;; movq 8(%rdx), %rcx +;; cmpq %rcx, %rax ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rdx), %r8 -;; movl $-65536, %eax -;; movb %sil, 0(%r8,%rax,1) +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %rcx +;; movl $-65536, %edx +;; movb %sil, 0(%rcx,%rdx,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -70,17 +71,20 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq %r8, %r11 -;; addq %r11, const(0), %r11 +;; movq %rsi, %rax +;; movl %edi, %esi +;; movq %rsi, %rdi +;; addq %rdi, const(0), %rdi ;; jb #trap=heap_oob -;; movq 8(%rsi), %rdi -;; cmpq %rdi, %r11 +;; movq %rax, %rdx +;; movq 8(%rdx), %rax +;; cmpq %rax, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rsi), %r8 -;; movl $-65536, %eax -;; movzbq 0(%r8,%rax,1), %rax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rsi,1), %rcx +;; movl $-65536, %edx +;; movzbq 0(%rcx,%rdx,1), %rax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat index c8a51cb693a4..312f4d19d11f 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -46,14 +46,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: ;; movl %edi, %edi -;; movq 8(%rdx), %r11 -;; subq %r11, $4, %r11 -;; movq %rdi, %r10 -;; addq %r10, 0(%rdx), %r10 -;; xorq %rax, %rax, %rax -;; cmpq %r11, %rdi -;; cmovnbeq %rax, %r10, %r10 -;; movl %esi, 0(%r10) +;; movq 8(%rdx), %rax +;; subq %rax, $4, %rax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %r11 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbeq %rcx, %r11, %r11 +;; movl %esi, 0(%r11) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -66,16 +66,16 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rsi, %rax -;; movl %edi, %esi -;; movq 8(%rax), %r11 -;; subq %r11, $4, %r11 -;; movq %rsi, %r10 -;; addq %r10, 0(%rax), %r10 -;; xorq %rdi, %rdi, %rdi -;; cmpq %r11, %rsi -;; cmovnbeq %rdi, %r10, %r10 -;; movl 0(%r10), %eax +;; movq %rsi, %r8 +;; movl %edi, %edi +;; movq 8(%r8), %rsi +;; subq %rsi, $4, %rsi +;; movq 0(%r8), %rax +;; lea 0(%rax,%rdi,1), %r11 +;; xorq %rax, %rax, %rax +;; cmpq %rsi, %rdi +;; cmovnbeq %rax, %r11, %r11 +;; movl 0(%r11), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat index dc35c58d198f..1604f80e005a 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -45,18 +45,19 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq %r11, %rax -;; addq %rax, const(0), %rax +;; movl %edi, %r8d +;; movq %r8, %rcx +;; addq %rcx, const(0), %rcx ;; jb #trap=heap_oob -;; movq 8(%rdx), %rcx -;; addq %r11, 0(%rdx), %r11 -;; movl $-65536, %edx -;; lea 0(%r11,%rdx,1), %rdi +;; movq 8(%rdx), %r9 +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%r8,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rax ;; xorq %rdx, %rdx, %rdx -;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rdi, %rdi -;; movl %esi, 0(%rdi) +;; cmpq %r9, %rcx +;; cmovnbeq %rdx, %rax, %rax +;; movl %esi, 0(%rax) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -69,18 +70,19 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq %r11, %rax -;; addq %rax, const(0), %rax +;; movl %edi, %r8d +;; movq %r8, %rcx +;; addq %rcx, const(0), %rcx ;; jb #trap=heap_oob -;; movq 8(%rsi), %rcx -;; addq %r11, 0(%rsi), %r11 -;; movl $-65536, %edx -;; lea 0(%r11,%rdx,1), %rdi -;; xorq %rdx, %rdx, %rdx -;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rdi, %rdi -;; movl 0(%rdi), %eax +;; movq 8(%rsi), %rdx +;; movq 0(%rsi), %r9 +;; lea 0(%r9,%r8,1), %r8 +;; movl $-65536, %r9d +;; lea 0(%r8,%r9,1), %rax +;; xorq %r8, %r8, %r8 +;; cmpq %rdx, %rcx +;; cmovnbeq %r8, %rax, %rax +;; movl 0(%rax), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat index 17a6726b0cc9..cf560a569716 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -45,14 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq 8(%rdx), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rdx), %r9 -;; xorq %rax, %rax, %rax -;; cmpq %rdi, %r11 -;; cmovnbq %rax, %r9, %r9 -;; movb %sil, 0(%r9) +;; movl %edi, %edi +;; movq 8(%rdx), %rax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %r10 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbq %rcx, %r10, %r10 +;; movb %sil, 0(%r10) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -65,14 +65,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d +;; movl %edi, %ecx ;; movq 8(%rsi), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rsi), %r9 -;; xorq %rsi, %rsi, %rsi -;; cmpq %rdi, %r11 -;; cmovnbq %rsi, %r9, %r9 -;; movzbq 0(%r9), %rax +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rcx,1), %r10 +;; xorq %rax, %rax, %rax +;; cmpq %rdi, %rcx +;; cmovnbq %rax, %r10, %r10 +;; movzbq 0(%r10), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 171202116deb..a42e7eea29a3 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -45,18 +45,19 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq %r11, %rax -;; addq %rax, const(0), %rax +;; movl %edi, %r8d +;; movq %r8, %rcx +;; addq %rcx, const(0), %rcx ;; jb #trap=heap_oob -;; movq 8(%rdx), %rcx -;; addq %r11, 0(%rdx), %r11 -;; movl $-65536, %edx -;; lea 0(%r11,%rdx,1), %rdi +;; movq 8(%rdx), %r9 +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%r8,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rax ;; xorq %rdx, %rdx, %rdx -;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rdi, %rdi -;; movb %sil, 0(%rdi) +;; cmpq %r9, %rcx +;; cmovnbeq %rdx, %rax, %rax +;; movb %sil, 0(%rax) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -69,18 +70,19 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq %r11, %rax -;; addq %rax, const(0), %rax +;; movl %edi, %r8d +;; movq %r8, %rcx +;; addq %rcx, const(0), %rcx ;; jb #trap=heap_oob -;; movq 8(%rsi), %rcx -;; addq %r11, 0(%rsi), %r11 -;; movl $-65536, %edx -;; lea 0(%r11,%rdx,1), %rdi -;; xorq %rdx, %rdx, %rdx -;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rdi, %rdi -;; movzbq 0(%rdi), %rax +;; movq 8(%rsi), %rdx +;; movq 0(%rsi), %r9 +;; lea 0(%r9,%r8,1), %r8 +;; movl $-65536, %r9d +;; lea 0(%r8,%r9,1), %rax +;; xorq %r8, %r8, %r8 +;; cmpq %rdx, %rcx +;; cmovnbeq %r8, %rax, %rax +;; movzbq 0(%rax), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat index 57e2d1680d46..f08a002320bc 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -45,14 +45,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq 8(%rdx), %r10 -;; cmpq %r10, %r8 +;; movl %edi, %r11d +;; movq 8(%rdx), %rdi +;; cmpq %rdi, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rdx), %r8 -;; movl $-65536, %edi -;; movl %esi, 0(%r8,%rdi,1) +;; movq 0(%rdx), %rdi +;; lea 0(%rdi,%r11,1), %rdi +;; movl $-65536, %eax +;; movl %esi, 0(%rdi,%rax,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -67,14 +68,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq 8(%rsi), %r10 -;; cmpq %r10, %r8 +;; movl %edi, %r11d +;; movq 8(%rsi), %rdi +;; cmpq %rdi, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rsi), %r8 -;; movl $-65536, %esi -;; movl 0(%r8,%rsi,1), %eax +;; movq 0(%rsi), %rdi +;; lea 0(%rdi,%r11,1), %rdi +;; movl $-65536, %eax +;; movl 0(%rdi,%rax,1), %eax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat index f2c873612824..04fcb42f1c55 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -45,14 +45,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq 8(%rdx), %r10 -;; cmpq %r10, %r8 +;; movl %edi, %r11d +;; movq 8(%rdx), %rdi +;; cmpq %rdi, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rdx), %r8 -;; movl $-65536, %edi -;; movb %sil, 0(%r8,%rdi,1) +;; movq 0(%rdx), %rdi +;; lea 0(%rdi,%r11,1), %rdi +;; movl $-65536, %eax +;; movb %sil, 0(%rdi,%rax,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -67,14 +68,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r8d -;; movq 8(%rsi), %r10 -;; cmpq %r10, %r8 +;; movl %edi, %r11d +;; movq 8(%rsi), %rdi +;; cmpq %rdi, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %r8, 0(%rsi), %r8 -;; movl $-65536, %esi -;; movzbq 0(%r8,%rsi,1), %rax +;; movq 0(%rsi), %rdi +;; lea 0(%rdi,%r11,1), %rdi +;; movl $-65536, %eax +;; movzbq 0(%rdi,%rax,1), %rax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index 89b24d688248..69b4d7120eee 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -45,14 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq 8(%rdx), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rdx), %r9 -;; xorq %rax, %rax, %rax -;; cmpq %rdi, %r11 -;; cmovnbeq %rax, %r9, %r9 -;; movl %esi, 0(%r9) +;; movl %edi, %edi +;; movq 8(%rdx), %rax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %r10 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbeq %rcx, %r10, %r10 +;; movl %esi, 0(%r10) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -65,14 +65,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d +;; movl %edi, %ecx ;; movq 8(%rsi), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rsi), %r9 -;; xorq %rsi, %rsi, %rsi -;; cmpq %rdi, %r11 -;; cmovnbeq %rsi, %r9, %r9 -;; movl 0(%r9), %eax +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rcx,1), %r10 +;; xorq %rax, %rax, %rax +;; cmpq %rdi, %rcx +;; cmovnbeq %rax, %r10, %r10 +;; movl 0(%r10), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat index 8b6846dfce16..6e2728345df3 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -45,16 +45,16 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %edi -;; movq 8(%rdx), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rdx), %rcx -;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r11 -;; xorq %rcx, %rcx, %rcx -;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r11, %r11 -;; movl %esi, 0(%r11) +;; movl %edi, %eax +;; movq 8(%rdx), %rcx +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%rax,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi +;; xorq %rdx, %rdx, %rdx +;; cmpq %rcx, %rax +;; cmovnbeq %rdx, %rdi, %rdi +;; movl %esi, 0(%rdi) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -67,16 +67,16 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %edi -;; movq 8(%rsi), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rsi), %rcx -;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r11 -;; xorq %rcx, %rcx, %rcx -;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r11, %r11 -;; movl 0(%r11), %eax +;; movl %edi, %eax +;; movq 8(%rsi), %rcx +;; movq 0(%rsi), %rdx +;; lea 0(%rdx,%rax,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rsi +;; xorq %rdx, %rdx, %rdx +;; cmpq %rcx, %rax +;; cmovnbeq %rdx, %rsi, %rsi +;; movl 0(%rsi), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 1ac80ebbc66a..032a19a9a8bc 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -45,14 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d -;; movq 8(%rdx), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rdx), %r9 -;; xorq %rax, %rax, %rax -;; cmpq %rdi, %r11 -;; cmovnbq %rax, %r9, %r9 -;; movb %sil, 0(%r9) +;; movl %edi, %edi +;; movq 8(%rdx), %rax +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %r10 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbq %rcx, %r10, %r10 +;; movb %sil, 0(%r10) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -65,14 +65,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r11d +;; movl %edi, %ecx ;; movq 8(%rsi), %rdi -;; movq %r11, %r9 -;; addq %r9, 0(%rsi), %r9 -;; xorq %rsi, %rsi, %rsi -;; cmpq %rdi, %r11 -;; cmovnbq %rsi, %r9, %r9 -;; movzbq 0(%r9), %rax +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rcx,1), %r10 +;; xorq %rax, %rax, %rax +;; cmpq %rdi, %rcx +;; cmovnbq %rax, %r10, %r10 +;; movzbq 0(%r10), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 5ac256e173d3..3004cd718888 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -45,16 +45,16 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %edi -;; movq 8(%rdx), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rdx), %rcx -;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r11 -;; xorq %rcx, %rcx, %rcx -;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r11, %r11 -;; movb %sil, 0(%r11) +;; movl %edi, %eax +;; movq 8(%rdx), %rcx +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%rax,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi +;; xorq %rdx, %rdx, %rdx +;; cmpq %rcx, %rax +;; cmovnbeq %rdx, %rdi, %rdi +;; movb %sil, 0(%rdi) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -67,16 +67,16 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %edi -;; movq 8(%rsi), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rsi), %rcx -;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r11 -;; xorq %rcx, %rcx, %rcx -;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r11, %r11 -;; movzbq 0(%r11), %rax +;; movl %edi, %eax +;; movq 8(%rsi), %rcx +;; movq 0(%rsi), %rdx +;; lea 0(%rdx,%rax,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rsi +;; xorq %rdx, %rdx, %rdx +;; cmpq %rcx, %rax +;; cmovnbeq %rdx, %rsi, %rsi +;; movzbq 0(%rsi), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat index 606ac690204d..308f142069b5 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -45,16 +45,17 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r10 -;; addq %r10, const(0), %r10 +;; movq %rdi, %r11 +;; addq %r11, const(0), %r11 ;; jb #trap=heap_oob -;; movq 8(%rdx), %r11 -;; cmpq %r11, %r10 +;; movq 8(%rdx), %rax +;; cmpq %rax, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %eax -;; movl %esi, 0(%rdi,%rax,1) +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %rax +;; movl $-65536, %ecx +;; movl %esi, 0(%rax,%rcx,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -69,16 +70,17 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r10 -;; addq %r10, const(0), %r10 +;; movq %rdi, %r11 +;; addq %r11, const(0), %r11 ;; jb #trap=heap_oob -;; movq 8(%rsi), %r11 -;; cmpq %r11, %r10 +;; movq 8(%rsi), %rax +;; cmpq %rax, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %eax -;; movl 0(%rdi,%rax,1), %eax +;; movq 0(%rsi), %rax +;; lea 0(%rax,%rdi,1), %rax +;; movl $-65536, %ecx +;; movl 0(%rax,%rcx,1), %eax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat index 1253391f756b..36c6ccde055f 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -45,16 +45,17 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r10 -;; addq %r10, const(0), %r10 +;; movq %rdi, %r11 +;; addq %r11, const(0), %r11 ;; jb #trap=heap_oob -;; movq 8(%rdx), %r11 -;; cmpq %r11, %r10 +;; movq 8(%rdx), %rax +;; cmpq %rax, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %eax -;; movb %sil, 0(%rdi,%rax,1) +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %rax +;; movl $-65536, %ecx +;; movb %sil, 0(%rax,%rcx,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -69,16 +70,17 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r10 -;; addq %r10, const(0), %r10 +;; movq %rdi, %r11 +;; addq %r11, const(0), %r11 ;; jb #trap=heap_oob -;; movq 8(%rsi), %r11 -;; cmpq %r11, %r10 +;; movq 8(%rsi), %rax +;; cmpq %rax, %r11 ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %eax -;; movzbq 0(%rdi,%rax,1), %rax +;; movq 0(%rsi), %rax +;; lea 0(%rax,%rdi,1), %rax +;; movl $-65536, %ecx +;; movzbq 0(%rax,%rcx,1), %rax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat index 7208a2984645..d0a2d3ab3d01 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -45,14 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r10 -;; subq %r10, $4, %r10 -;; movq %rdi, %r9 -;; addq %r9, 0(%rdx), %r9 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbeq %r11, %r9, %r9 -;; movl %esi, 0(%r9) +;; movq 8(%rdx), %r11 +;; subq %r11, $4, %r11 +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %r10 +;; xorq %rax, %rax, %rax +;; cmpq %r11, %rdi +;; cmovnbeq %rax, %r10, %r10 +;; movl %esi, 0(%r10) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -65,14 +65,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r10 -;; subq %r10, $4, %r10 -;; movq %rdi, %r9 -;; addq %r9, 0(%rsi), %r9 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbeq %r11, %r9, %r9 -;; movl 0(%r9), %eax +;; movq 8(%rsi), %r11 +;; subq %r11, $4, %r11 +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %r10 +;; xorq %rsi, %rsi, %rsi +;; cmpq %r11, %rdi +;; cmovnbeq %rsi, %r10, %r10 +;; movl 0(%r10), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat index ca713ccc731b..a70d1cb486f0 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -49,9 +49,10 @@ ;; addq %rax, const(0), %rax ;; jb #trap=heap_oob ;; movq 8(%rdx), %rcx -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %edx -;; lea 0(%rdi,%rdx,1), %rdi +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%rdi,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi ;; xorq %rdx, %rdx, %rdx ;; cmpq %rcx, %rax ;; cmovnbeq %rdx, %rdi, %rdi @@ -72,13 +73,14 @@ ;; addq %rax, const(0), %rax ;; jb #trap=heap_oob ;; movq 8(%rsi), %rcx -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %edx -;; lea 0(%rdi,%rdx,1), %rsi +;; movq 0(%rsi), %rdx +;; lea 0(%rdx,%rdi,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi ;; xorq %rdx, %rdx, %rdx ;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rsi, %rsi -;; movl 0(%rsi), %eax +;; cmovnbeq %rdx, %rdi, %rdi +;; movl 0(%rdi), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat index b2f45beb12eb..cca108fb4fd5 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -45,13 +45,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbq %r11, %r8, %r8 -;; movb %sil, 0(%r8) +;; movq 8(%rdx), %r11 +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %r9 +;; xorq %rax, %rax, %rax +;; cmpq %r11, %rdi +;; cmovnbq %rax, %r9, %r9 +;; movb %sil, 0(%r9) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -64,13 +64,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rsi), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbq %r11, %r8, %r8 -;; movzbq 0(%r8), %rax +;; movq 8(%rsi), %r11 +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %r9 +;; xorq %rsi, %rsi, %rsi +;; cmpq %r11, %rdi +;; cmovnbq %rsi, %r9, %r9 +;; movzbq 0(%r9), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat index c0ae962583e7..a20b2fca5c61 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -49,9 +49,10 @@ ;; addq %rax, const(0), %rax ;; jb #trap=heap_oob ;; movq 8(%rdx), %rcx -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %edx -;; lea 0(%rdi,%rdx,1), %rdi +;; movq 0(%rdx), %rdx +;; lea 0(%rdx,%rdi,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi ;; xorq %rdx, %rdx, %rdx ;; cmpq %rcx, %rax ;; cmovnbeq %rdx, %rdi, %rdi @@ -72,13 +73,14 @@ ;; addq %rax, const(0), %rax ;; jb #trap=heap_oob ;; movq 8(%rsi), %rcx -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %edx -;; lea 0(%rdi,%rdx,1), %rsi +;; movq 0(%rsi), %rdx +;; lea 0(%rdx,%rdi,1), %rdx +;; movl $-65536, %r8d +;; lea 0(%rdx,%r8,1), %rdi ;; xorq %rdx, %rdx, %rdx ;; cmpq %rcx, %rax -;; cmovnbeq %rdx, %rsi, %rsi -;; movzbq 0(%rsi), %rax +;; cmovnbeq %rdx, %rdi, %rdi +;; movzbq 0(%rdi), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat index e0f869ff951d..b444dc60178a 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat @@ -45,13 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r9 -;; cmpq %r9, %rdi +;; movq 8(%rdx), %r10 +;; cmpq %r10, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %r11d -;; movl %esi, 0(%rdi,%r11,1) +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %rdi +;; movl $-65536, %eax +;; movl %esi, 0(%rdi,%rax,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -66,13 +67,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r9 -;; cmpq %r9, %rdi +;; movq 8(%rsi), %r10 +;; cmpq %r10, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %r11d -;; movl 0(%rdi,%r11,1), %eax +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %rsi +;; movl $-65536, %edi +;; movl 0(%rsi,%rdi,1), %eax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat index 0a8abfc3ceb3..7dc38fadf04f 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat @@ -45,13 +45,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r9 -;; cmpq %r9, %rdi +;; movq 8(%rdx), %r10 +;; cmpq %r10, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rdx), %rdi -;; movl $-65536, %r11d -;; movb %sil, 0(%rdi,%r11,1) +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %rdi +;; movl $-65536, %eax +;; movb %sil, 0(%rdi,%rax,1) ;; jmp label2 ;; block2: ;; movq %rbp, %rsp @@ -66,13 +67,14 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r9 -;; cmpq %r9, %rdi +;; movq 8(%rsi), %r10 +;; cmpq %r10, %rdi ;; jnbe label3; j label1 ;; block1: -;; addq %rdi, 0(%rsi), %rdi -;; movl $-65536, %r11d -;; movzbq 0(%rdi,%r11,1), %rax +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %rsi +;; movl $-65536, %edi +;; movzbq 0(%rsi,%rdi,1), %rax ;; jmp label2 ;; block2: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index c7c43a0d3f9e..1e36556796db 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -45,13 +45,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbeq %r11, %r8, %r8 -;; movl %esi, 0(%r8) +;; movq 8(%rdx), %r11 +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %r9 +;; xorq %rax, %rax, %rax +;; cmpq %r11, %rdi +;; cmovnbeq %rax, %r9, %r9 +;; movl %esi, 0(%r9) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -64,13 +64,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rsi), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbeq %r11, %r8, %r8 -;; movl 0(%r8), %eax +;; movq 8(%rsi), %r11 +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %r9 +;; xorq %rsi, %rsi, %rsi +;; cmpq %r11, %rdi +;; cmovnbeq %rsi, %r9, %r9 +;; movl 0(%r9), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat index 3320b9fa3c4c..454ece1fa7f6 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat @@ -46,14 +46,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: ;; movq 8(%rdx), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rdx), %rcx +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %rcx ;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r10 +;; lea 0(%rcx,%rdx,1), %r11 ;; xorq %rcx, %rcx, %rcx ;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r10, %r10 -;; movl %esi, 0(%r10) +;; cmovnbeq %rcx, %r11, %r11 +;; movl %esi, 0(%r11) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -66,15 +66,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %rcx -;; movq %rdi, %rax -;; addq %rax, 0(%rsi), %rax -;; movl $-65536, %esi -;; lea 0(%rax,%rsi,1), %r10 -;; xorq %rax, %rax, %rax -;; cmpq %rcx, %rdi -;; cmovnbeq %rax, %r10, %r10 -;; movl 0(%r10), %eax +;; movq 8(%rsi), %rax +;; movq 0(%rsi), %rcx +;; lea 0(%rcx,%rdi,1), %rcx +;; movl $-65536, %edx +;; lea 0(%rcx,%rdx,1), %r11 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbeq %rcx, %r11, %r11 +;; movl 0(%r11), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 449f63560bf7..bb5d00619029 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -45,13 +45,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rdx), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbq %r11, %r8, %r8 -;; movb %sil, 0(%r8) +;; movq 8(%rdx), %r11 +;; movq 0(%rdx), %rax +;; lea 0(%rax,%rdi,1), %r9 +;; xorq %rax, %rax, %rax +;; cmpq %r11, %rdi +;; cmovnbq %rax, %r9, %r9 +;; movb %sil, 0(%r9) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -64,13 +64,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %r10 -;; movq %rdi, %r8 -;; addq %r8, 0(%rsi), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq %r10, %rdi -;; cmovnbq %r11, %r8, %r8 -;; movzbq 0(%r8), %rax +;; movq 8(%rsi), %r11 +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%rdi,1), %r9 +;; xorq %rsi, %rsi, %rsi +;; cmpq %r11, %rdi +;; cmovnbq %rsi, %r9, %r9 +;; movzbq 0(%r9), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat index 214b6e06f87f..16ecad9dfd11 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat @@ -46,14 +46,14 @@ ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: ;; movq 8(%rdx), %rax -;; movq %rdi, %rcx -;; addq %rcx, 0(%rdx), %rcx +;; movq 0(%rdx), %rcx +;; lea 0(%rcx,%rdi,1), %rcx ;; movl $-65536, %edx -;; lea 0(%rcx,%rdx,1), %r10 +;; lea 0(%rcx,%rdx,1), %r11 ;; xorq %rcx, %rcx, %rcx ;; cmpq %rax, %rdi -;; cmovnbeq %rcx, %r10, %r10 -;; movb %sil, 0(%r10) +;; cmovnbeq %rcx, %r11, %r11 +;; movb %sil, 0(%r11) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -66,15 +66,15 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq 8(%rsi), %rcx -;; movq %rdi, %rax -;; addq %rax, 0(%rsi), %rax -;; movl $-65536, %esi -;; lea 0(%rax,%rsi,1), %r10 -;; xorq %rax, %rax, %rax -;; cmpq %rcx, %rdi -;; cmovnbeq %rax, %r10, %r10 -;; movzbq 0(%r10), %rax +;; movq 8(%rsi), %rax +;; movq 0(%rsi), %rcx +;; lea 0(%rcx,%rdi,1), %rcx +;; movl $-65536, %edx +;; lea 0(%rcx,%rdx,1), %r11 +;; xorq %rcx, %rcx, %rcx +;; cmpq %rax, %rdi +;; cmovnbeq %rcx, %r11, %r11 +;; movzbq 0(%r11), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat index 8066a12155e7..ac8ed3a38403 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -43,13 +43,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r10d -;; movq %r10, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq $268435452, %r10 -;; cmovnbeq %r11, %r8, %r8 -;; movl %esi, 0(%r8) +;; movl %edi, %r11d +;; movq 0(%rdx), %rdi +;; lea 0(%rdi,%r11,1), %r9 +;; xorq %rdi, %rdi, %rdi +;; cmpq $268435452, %r11 +;; cmovnbeq %rdi, %r9, %r9 +;; movl %esi, 0(%r9) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -62,13 +62,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r10d -;; movq %r10, %r8 -;; addq %r8, 0(%rsi), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq $268435452, %r10 -;; cmovnbeq %r11, %r8, %r8 -;; movl 0(%r8), %eax +;; movl %edi, %r11d +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%r11,1), %r9 +;; xorq %rsi, %rsi, %rsi +;; cmpq $268435452, %r11 +;; cmovnbeq %rsi, %r9, %r9 +;; movl 0(%r9), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat index 93d48967ab97..cdfe8b68d46a 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -43,13 +43,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r10d -;; movq %r10, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq $268435455, %r10 -;; cmovnbeq %r11, %r8, %r8 -;; movb %sil, 0(%r8) +;; movl %edi, %r11d +;; movq 0(%rdx), %rdi +;; lea 0(%rdi,%r11,1), %r9 +;; xorq %rdi, %rdi, %rdi +;; cmpq $268435455, %r11 +;; cmovnbeq %rdi, %r9, %r9 +;; movb %sil, 0(%r9) ;; jmp label1 ;; block1: ;; movq %rbp, %rsp @@ -62,13 +62,13 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movl %edi, %r10d -;; movq %r10, %r8 -;; addq %r8, 0(%rsi), %r8 -;; xorq %r11, %r11, %r11 -;; cmpq $268435455, %r10 -;; cmovnbeq %r11, %r8, %r8 -;; movzbq 0(%r8), %rax +;; movl %edi, %r11d +;; movq 0(%rsi), %rsi +;; lea 0(%rsi,%r11,1), %r9 +;; xorq %rsi, %rsi, %rsi +;; cmpq $268435455, %r11 +;; cmovnbeq %rsi, %r9, %r9 +;; movzbq 0(%r9), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat index c0113a35f739..3d96a293536a 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat @@ -43,11 +43,11 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r9, %r9, %r9 +;; movq 0(%rdx), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435452, %rdi -;; cmovnbeq %r9, %r8, %r8 +;; cmovnbeq %r10, %r8, %r8 ;; movl %esi, 0(%r8) ;; jmp label1 ;; block1: @@ -61,12 +61,12 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %rdx -;; addq %rdx, 0(%rsi), %rdx -;; xorq %r9, %r9, %r9 +;; movq 0(%rsi), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435452, %rdi -;; cmovnbeq %r9, %rdx, %rdx -;; movl 0(%rdx), %eax +;; cmovnbeq %r10, %r8, %r8 +;; movl 0(%r8), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat index 6bf8e9cc8a0d..c9cc5db35a71 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat @@ -43,11 +43,11 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r9, %r9, %r9 +;; movq 0(%rdx), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435455, %rdi -;; cmovnbeq %r9, %r8, %r8 +;; cmovnbeq %r10, %r8, %r8 ;; movb %sil, 0(%r8) ;; jmp label1 ;; block1: @@ -61,12 +61,12 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %rdx -;; addq %rdx, 0(%rsi), %rdx -;; xorq %r9, %r9, %r9 +;; movq 0(%rsi), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435455, %rdi -;; cmovnbeq %r9, %rdx, %rdx -;; movzbq 0(%rdx), %rax +;; cmovnbeq %r10, %r8, %r8 +;; movzbq 0(%r8), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat index c58eeb9a84b1..4e2897db6dcd 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat @@ -43,11 +43,11 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r9, %r9, %r9 +;; movq 0(%rdx), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435452, %rdi -;; cmovnbeq %r9, %r8, %r8 +;; cmovnbeq %r10, %r8, %r8 ;; movl %esi, 0(%r8) ;; jmp label1 ;; block1: @@ -61,12 +61,12 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %rdx -;; addq %rdx, 0(%rsi), %rdx -;; xorq %r9, %r9, %r9 +;; movq 0(%rsi), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435452, %rdi -;; cmovnbeq %r9, %rdx, %rdx -;; movl 0(%rdx), %eax +;; cmovnbeq %r10, %r8, %r8 +;; movl 0(%r8), %eax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat index 421d738d86b7..ba3e20f79317 100644 --- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat +++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat @@ -43,11 +43,11 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %r8 -;; addq %r8, 0(%rdx), %r8 -;; xorq %r9, %r9, %r9 +;; movq 0(%rdx), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435455, %rdi -;; cmovnbeq %r9, %r8, %r8 +;; cmovnbeq %r10, %r8, %r8 ;; movb %sil, 0(%r8) ;; jmp label1 ;; block1: @@ -61,12 +61,12 @@ ;; movq %rsp, %rbp ;; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 } ;; block0: -;; movq %rdi, %rdx -;; addq %rdx, 0(%rsi), %rdx -;; xorq %r9, %r9, %r9 +;; movq 0(%rsi), %r10 +;; lea 0(%r10,%rdi,1), %r8 +;; xorq %r10, %r10, %r10 ;; cmpq $268435455, %rdi -;; cmovnbeq %r9, %rdx, %rdx -;; movzbq 0(%rdx), %rax +;; cmovnbeq %r10, %r8, %r8 +;; movzbq 0(%r8), %rax ;; jmp label1 ;; block1: ;; movq %rbp, %rsp diff --git a/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif new file mode 100644 index 000000000000..dace18d7054f --- /dev/null +++ b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif @@ -0,0 +1,20 @@ +test legalizer +target x86_64 + +function %foo() -> i32 { +block0: + v1 = iconst.i32 1 + v2 = iadd_imm v1, 1 + v3 = iadd_imm v2, 1 + return v3 +} + +; check: v1 = iconst.i32 1 +; check: v4 = iconst.i32 1 +; check: v5 = iadd v1, v4 +; check: v2 -> v5 +; check: v6 = iconst.i32 1 +; check: v7 = iadd v2, v6 +; check: v3 -> v7 +; check: return v3 + diff --git a/cranelift/filetests/filetests/legalizer/bxor_imm.clif b/cranelift/filetests/filetests/legalizer/bxor_imm.clif index 12a149eed9c0..12a803732c54 100644 --- a/cranelift/filetests/filetests/legalizer/bxor_imm.clif +++ b/cranelift/filetests/filetests/legalizer/bxor_imm.clif @@ -9,4 +9,6 @@ block0(v0: i64, v1: i64): } ; check: v3 = iconst.i64 0x0001_0000_0000 -; check: v2 = bxor v0, v3 +; check: v4 = bxor v0, v3 +; check: v2 -> v4 +; check: return v2 diff --git a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif new file mode 100644 index 000000000000..69eb80d2d5ef --- /dev/null +++ b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif @@ -0,0 +1,200 @@ +test optimize precise-output +set enable_nan_canonicalization=true +target x86_64 + +function %ceil(f32) -> f32 { +block0(v0: f32): + v1 = ceil v0 + return v1 +} + +; function %ceil(f32) -> f32 fast { +; block0(v0: f32): +; v2 = ceil v0 +; v3 = fcmp ne v2, v2 +; v4 = f32const +NaN +; v5 = select v3, v4, v2 ; v4 = +NaN +; v1 -> v5 +; return v1 +; } + +function %floor(f64) -> f64 { +block0(v0: f64): + v1 = floor v0 + return v1 +} + +; function %floor(f64) -> f64 fast { +; block0(v0: f64): +; v2 = floor v0 +; v3 = fcmp ne v2, v2 +; v4 = f64const +NaN +; v5 = select v3, v4, v2 ; v4 = +NaN +; v1 -> v5 +; return v1 +; } + +function %nearest(f32x4) -> f32x4 { +block0(v0: f32x4): + v1 = nearest v0 + return v1 +} + +; function %nearest(f32x4) -> f32x4 fast { +; block0(v0: f32x4): +; v2 = nearest v0 +; v3 = fcmp ne v2, v2 +; v4 = f32const +NaN +; v5 = splat.f32x4 v4 ; v4 = +NaN +; v6 = bitcast.f32x4 v3 +; v7 = bitselect v6, v5, v2 +; v1 -> v7 +; return v1 +; } + +function %sqrt(f64x2) -> f64x2 { +block0(v0: f64x2): + v1 = sqrt v0 + return v1 +} + +; function %sqrt(f64x2) -> f64x2 fast { +; block0(v0: f64x2): +; v2 = sqrt v0 +; v3 = f64const +NaN +; v4 = splat.f64x2 v3 ; v3 = +NaN +; v5 = fcmp ne v2, v2 +; v6 = bitcast.f64x2 v5 +; v7 = bitselect v6, v4, v2 +; v1 -> v7 +; return v1 +; } + +function %trunc(f32) -> f32 { +block0(v0: f32): + v1 = trunc v0 + return v1 +} + +; function %trunc(f32) -> f32 fast { +; block0(v0: f32): +; v2 = trunc v0 +; v3 = fcmp ne v2, v2 +; v4 = f32const +NaN +; v5 = select v3, v4, v2 ; v4 = +NaN +; v1 -> v5 +; return v1 +; } + +function %fadd(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fadd v0, v1 + return v2 +} + +; function %fadd(f32, f32) -> f32 fast { +; block0(v0: f32, v1: f32): +; v3 = fadd v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f32const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fsub(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = fsub v0, v1 + return v2 +} + +; function %fsub(f32, f32) -> f32 fast { +; block0(v0: f32, v1: f32): +; v3 = fsub v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f32const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fdiv(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fdiv v0, v1 + return v2 +} + +; function %fdiv(f64, f64) -> f64 fast { +; block0(v0: f64, v1: f64): +; v3 = fdiv v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f64const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fmul(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fmul v0, v1 + return v2 +} + +; function %fmul(f64, f64) -> f64 fast { +; block0(v0: f64, v1: f64): +; v3 = fmul v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f64const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fmax(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fmax v0, v1 + return v2 +} + +; function %fmax(f64, f64) -> f64 fast { +; block0(v0: f64, v1: f64): +; v3 = fmax v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f64const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fmin(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = fmin v0, v1 + return v2 +} + +; function %fmin(f64, f64) -> f64 fast { +; block0(v0: f64, v1: f64): +; v3 = fmin v0, v1 +; v4 = fcmp ne v3, v3 +; v5 = f64const +NaN +; v6 = select v4, v5, v3 ; v5 = +NaN +; v2 -> v6 +; return v2 +; } + +function %fma(f64, f64, f64) -> f64 { +block0(v0: f64, v1: f64, v2: f64): + v3 = fma v0, v1, v2 + return v3 +} + +; function %fma(f64, f64, f64) -> f64 fast { +; block0(v0: f64, v1: f64, v2: f64): +; v4 = fma v0, v1, v2 +; v5 = fcmp ne v4, v4 +; v6 = f64const +NaN +; v7 = select v5, v6, v4 ; v6 = +NaN +; v3 -> v7 +; return v3 +; } + diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat index 524f7eb83657..6efaa39f97bd 100644 --- a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat +++ b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat @@ -49,15 +49,17 @@ ;; ;; block0(v0: i32, v1: i64): ;; v20 -> v1 -;; v21 -> v1 ;; v22 -> v1 -;; v23 -> v1 -;; @0057 v5 = load.i64 notrap aligned v1+8 -;; @0057 v7 = load.i64 notrap aligned v1 +;; v24 -> v1 +;; v26 -> v1 +;; v21 = load.i64 notrap aligned v1+8 +;; v5 -> v21 +;; v23 = load.i64 notrap aligned v1 +;; v7 -> v23 ;; @0057 v4 = uextend.i64 v0 -;; @0057 v6 = icmp ugt v4, v5 +;; @0057 v6 = icmp ugt v4, v21 ;; @0057 v9 = iconst.i64 0 -;; @0057 v8 = iadd v7, v4 +;; @0057 v8 = iadd v23, v4 ;; @0057 v10 = select_spectre_guard v6, v9, v8 ; v9 = 0 ;; @0057 v11 = load.i32 little heap v10 ;; v2 -> v11 @@ -74,15 +76,17 @@ ;; ;; block0(v0: i32, v1: i64): ;; v24 -> v1 -;; v25 -> v1 ;; v26 -> v1 -;; v27 -> v1 -;; @0064 v5 = load.i64 notrap aligned v1+8 -;; @0064 v7 = load.i64 notrap aligned v1 +;; v28 -> v1 +;; v30 -> v1 +;; v25 = load.i64 notrap aligned v1+8 +;; v5 -> v25 +;; v27 = load.i64 notrap aligned v1 +;; v7 -> v27 ;; @0064 v4 = uextend.i64 v0 -;; @0064 v6 = icmp ugt v4, v5 +;; @0064 v6 = icmp ugt v4, v25 ;; @0064 v11 = iconst.i64 0 -;; @0064 v8 = iadd v7, v4 +;; @0064 v8 = iadd v27, v4 ;; @0064 v9 = iconst.i64 1234 ;; @0064 v10 = iadd v8, v9 ; v9 = 1234 ;; @0064 v12 = select_spectre_guard v6, v11, v10 ; v11 = 0 diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat index c63173922333..e103b3b33084 100644 --- a/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat +++ b/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat @@ -44,10 +44,11 @@ ;; ;; block0(v0: i32, v1: i64): ;; v12 -> v1 -;; v13 -> v1 -;; @0057 v5 = load.i64 notrap aligned readonly v1 +;; v14 -> v1 +;; v13 = load.i64 notrap aligned readonly v1 +;; v5 -> v13 ;; @0057 v4 = uextend.i64 v0 -;; @0057 v6 = iadd v5, v4 +;; @0057 v6 = iadd v13, v4 ;; @0057 v7 = load.i32 little heap v6 ;; v2 -> v7 ;; @005f jump block1 @@ -62,10 +63,11 @@ ;; ;; block0(v0: i32, v1: i64): ;; v16 -> v1 -;; v17 -> v1 -;; @0064 v5 = load.i64 notrap aligned readonly v1 +;; v18 -> v1 +;; v17 = load.i64 notrap aligned readonly v1 +;; v5 -> v17 ;; @0064 v4 = uextend.i64 v0 -;; @0064 v6 = iadd v5, v4 +;; @0064 v6 = iadd v17, v4 ;; @0064 v7 = iconst.i64 1234 ;; @0064 v8 = iadd v6, v7 ; v7 = 1234 ;; @0064 v9 = load.i32 little heap v8 diff --git a/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat b/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat index f7f5f72bce6a..a0879e2ced8e 100644 --- a/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat +++ b/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat @@ -62,22 +62,24 @@ ;; ;; block0(v0: i32, v1: i64): ;; v29 -> v1 -;; v30 -> v1 ;; v31 -> v1 -;; v32 -> v1 ;; v33 -> v1 -;; v34 -> v1 -;; @0047 v6 = load.i64 notrap aligned v1+8 +;; v35 -> v1 +;; v37 -> v1 +;; v39 -> v1 +;; v30 = load.i64 notrap aligned v1+8 +;; v6 -> v30 ;; @0047 v5 = uextend.i64 v0 -;; @0047 v7 = icmp ugt v5, v6 +;; @0047 v7 = icmp ugt v5, v30 ;; @0047 brif v7, block2, block3 ;; ;; block2 cold: ;; @0047 trap heap_oob ;; ;; block3: -;; @0047 v8 = load.i64 notrap aligned v1 -;; @0047 v9 = iadd v8, v5 +;; v32 = load.i64 notrap aligned v1 +;; v8 -> v32 +;; @0047 v9 = iadd v32, v5 ;; @0047 v10 = load.i32 little heap v9 ;; v2 -> v10 ;; @004c brif.i8 v7, block4, block5 @@ -92,15 +94,16 @@ ;; v3 -> v18 ;; @0051 v20 = iconst.i64 0x0010_0003 ;; @0051 v21 = uadd_overflow_trap.i64 v5, v20, heap_oob ; v20 = 0x0010_0003 -;; @0051 v23 = icmp ugt v21, v6 +;; @0051 v23 = icmp ugt v21, v30 ;; @0051 brif v23, block6, block7 ;; ;; block6 cold: ;; @0051 trap heap_oob ;; ;; block7: -;; @0051 v24 = load.i64 notrap aligned v1 -;; @0051 v25 = iadd v24, v5 +;; v40 = load.i64 notrap aligned v1 +;; v24 -> v40 +;; @0051 v25 = iadd v40, v5 ;; @0051 v26 = iconst.i64 0x000f_ffff ;; @0051 v27 = iadd v25, v26 ; v26 = 0x000f_ffff ;; @0051 v28 = load.i32 little heap v27 @@ -118,22 +121,24 @@ ;; ;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i64): ;; v26 -> v4 -;; v27 -> v4 ;; v28 -> v4 -;; v29 -> v4 ;; v30 -> v4 -;; v31 -> v4 -;; @005d v6 = load.i64 notrap aligned v4+8 +;; v32 -> v4 +;; v34 -> v4 +;; v36 -> v4 +;; v27 = load.i64 notrap aligned v4+8 +;; v6 -> v27 ;; @005d v5 = uextend.i64 v0 -;; @005d v7 = icmp ugt v5, v6 +;; @005d v7 = icmp ugt v5, v27 ;; @005d brif v7, block2, block3 ;; ;; block2 cold: ;; @005d trap heap_oob ;; ;; block3: -;; @005d v8 = load.i64 notrap aligned v4 -;; @005d v9 = iadd v8, v5 +;; v29 = load.i64 notrap aligned v4 +;; v8 -> v29 +;; @005d v9 = iadd v29, v5 ;; @005d store.i32 little heap v1, v9 ;; @0064 brif.i8 v7, block4, block5 ;; @@ -146,15 +151,16 @@ ;; @0064 store.i32 little heap v2, v16 ;; @006b v18 = iconst.i64 0x0010_0003 ;; @006b v19 = uadd_overflow_trap.i64 v5, v18, heap_oob ; v18 = 0x0010_0003 -;; @006b v21 = icmp ugt v19, v6 +;; @006b v21 = icmp ugt v19, v27 ;; @006b brif v21, block6, block7 ;; ;; block6 cold: ;; @006b trap heap_oob ;; ;; block7: -;; @006b v22 = load.i64 notrap aligned v4 -;; @006b v23 = iadd v22, v5 +;; v37 = load.i64 notrap aligned v4 +;; v22 -> v37 +;; @006b v23 = iadd v37, v5 ;; @006b v24 = iconst.i64 0x000f_ffff ;; @006b v25 = iadd v23, v24 ; v24 = 0x000f_ffff ;; @006b store.i32 little heap v3, v25 diff --git a/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat b/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat index 55e4b6b67f7f..226b54067ea6 100644 --- a/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat +++ b/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat @@ -62,17 +62,19 @@ ;; ;; block0(v0: i32, v1: i64): ;; v35 -> v1 -;; v36 -> v1 ;; v37 -> v1 -;; v38 -> v1 ;; v39 -> v1 -;; v40 -> v1 -;; @0047 v6 = load.i64 notrap aligned v1+8 -;; @0047 v8 = load.i64 notrap aligned v1 +;; v41 -> v1 +;; v43 -> v1 +;; v45 -> v1 +;; v36 = load.i64 notrap aligned v1+8 +;; v6 -> v36 +;; v38 = load.i64 notrap aligned v1 +;; v8 -> v38 ;; @0047 v5 = uextend.i64 v0 -;; @0047 v7 = icmp ugt v5, v6 +;; @0047 v7 = icmp ugt v5, v36 ;; @0047 v10 = iconst.i64 0 -;; @0047 v9 = iadd v8, v5 +;; @0047 v9 = iadd v38, v5 ;; @0047 v11 = select_spectre_guard v7, v10, v9 ; v10 = 0 ;; @0047 v12 = load.i32 little heap v11 ;; v2 -> v12 @@ -83,7 +85,7 @@ ;; v3 -> v22 ;; @0051 v24 = iconst.i64 0x0010_0003 ;; @0051 v25 = uadd_overflow_trap v5, v24, heap_oob ; v24 = 0x0010_0003 -;; @0051 v27 = icmp ugt v25, v6 +;; @0051 v27 = icmp ugt v25, v36 ;; @0051 v30 = iconst.i64 0x000f_ffff ;; @0051 v31 = iadd v9, v30 ; v30 = 0x000f_ffff ;; @0051 v33 = select_spectre_guard v27, v10, v31 ; v10 = 0 @@ -102,17 +104,19 @@ ;; ;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i64): ;; v32 -> v4 -;; v33 -> v4 ;; v34 -> v4 -;; v35 -> v4 ;; v36 -> v4 -;; v37 -> v4 -;; @005d v6 = load.i64 notrap aligned v4+8 -;; @005d v8 = load.i64 notrap aligned v4 +;; v38 -> v4 +;; v40 -> v4 +;; v42 -> v4 +;; v33 = load.i64 notrap aligned v4+8 +;; v6 -> v33 +;; v35 = load.i64 notrap aligned v4 +;; v8 -> v35 ;; @005d v5 = uextend.i64 v0 -;; @005d v7 = icmp ugt v5, v6 +;; @005d v7 = icmp ugt v5, v33 ;; @005d v10 = iconst.i64 0 -;; @005d v9 = iadd v8, v5 +;; @005d v9 = iadd v35, v5 ;; @005d v11 = select_spectre_guard v7, v10, v9 ; v10 = 0 ;; @005d store little heap v1, v11 ;; @0064 v17 = iconst.i64 4 @@ -121,7 +125,7 @@ ;; @0064 store little heap v2, v20 ;; @006b v22 = iconst.i64 0x0010_0003 ;; @006b v23 = uadd_overflow_trap v5, v22, heap_oob ; v22 = 0x0010_0003 -;; @006b v25 = icmp ugt v23, v6 +;; @006b v25 = icmp ugt v23, v33 ;; @006b v28 = iconst.i64 0x000f_ffff ;; @006b v29 = iadd v9, v28 ; v28 = 0x000f_ffff ;; @006b v31 = select_spectre_guard v25, v10, v29 ; v10 = 0 diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs index d71658182492..80257473fb27 100644 --- a/cranelift/fuzzgen/src/lib.rs +++ b/cranelift/fuzzgen/src/lib.rs @@ -122,6 +122,7 @@ where let flags = settings::Flags::new({ let mut builder = settings::builder(); builder.set("enable_verifier", "false").unwrap(); + builder.set("enable_nan_canonicalization", "true").unwrap(); builder }); @@ -129,9 +130,7 @@ where .expect("Unable to build a TargetIsa for the current host") .finish(flags) .expect("Failed to build TargetISA"); - - ctx.canonicalize_nans(isa.as_ref()) - .expect("Failed NaN canonicalization pass"); + isa.legalize_function(&mut ctx.func, &mut ctx.cfg); // Run the int_divz pass // diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs index d5eae5a09235..45ea723bdc49 100644 --- a/cranelift/isle/isle/src/codegen.rs +++ b/cranelift/isle/isle/src/codegen.rs @@ -135,7 +135,6 @@ impl<'a> Codegen<'a> { } writeln!(code, "\nuse super::*; // Pulls in all external types.").unwrap(); - writeln!(code, "use std::marker::PhantomData;").unwrap(); } fn generate_trait_sig(&self, code: &mut String, indent: &str, sig: &ExternalSig) {