From 2d8b8b5117d9f5a439bb48b75aacc5763b023d0c Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 13 Nov 2023 12:46:32 -0800
Subject: [PATCH 01/21] riscv64: Optimize shift-by-i128 a bit more

Pattern-match an `iconcat` with a constant lower value in addition to
`maybe_uextend`. This is in preparation for the next commit where
`uextend`-to-i128 always appears as an `iconcat`.
---
 cranelift/codegen/src/isa/riscv64/inst.isle   |  5 +-
 cranelift/codegen/src/isa/riscv64/lower.isle  | 81 ++++++++++++-------
 .../codegen/src/isa/riscv64/lower/isle.rs     |  4 +
 3 files changed, 62 insertions(+), 28 deletions(-)

diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 18ddcfc540d5..18a823002217 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -1876,7 +1876,7 @@
 (decl pure partial imm12_add (Imm12 i32) Imm12)
 (extern constructor imm12_add imm12_add)
 
-(decl imm12_and (Imm12 u64) Imm12)
+(decl pure imm12_and (Imm12 u64) Imm12)
 (extern constructor imm12_and imm12_and)
 
 ;; Imm12 Extractors
@@ -1905,6 +1905,9 @@
 (decl pure imm12_is_zero () Imm12)
 (extern extractor imm12_is_zero imm12_is_zero)
 
+(decl u64_from_imm12 (u64) Imm12)
+(extern extractor u64_from_imm12 u64_from_imm12)
+
 ;; Imm20
 
 ;; Extractor that matches if a Imm20 is zero
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 21dec2bb628d..de51dc5f020d 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1210,18 +1210,31 @@
   (rv_sll x (value_regs_get y 0)))
 
 ;; If the shift amount is known. We can mask it and encode it in the instruction.
-(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slliw x (imm12_and y (ty_shift_mask ty))))
+(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x y)))
+  (if-let shift (const_shift ty y))
+  (rv_slliw x shift))
 
-;; We technically don't need to mask the shift amount here. The instruction
-;; does the right thing. But it's neater when pretty printing it.
-(rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slli x (imm12_and y (ty_shift_mask ty))))
+(rule 3 (lower (has_type ty @ $I64 (ishl x y)))
+  (if-let shift (const_shift $I64 y))
+  (rv_slli x shift))
 
 ;; With `Zba` we have a shift that zero extends the LHS argument.
-(rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))
+(rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) y)))
   (if-let $true (has_zba))
-  (rv_slliuw x y))
+  (if-let shift (const_shift $I64 y))
+  (rv_slliuw x shift))
+
+;; Helper for extracting a constant shift value from the `Value` input.
+;;
+;; This will return an appropriate `Imm12` for a shift instruction which is
+;; masked to the width of the input `Type`.
+(decl pure partial const_shift (Type Value) Imm12)
+(rule 0 (const_shift ty (imm12_from_value n))
+  (imm12_and n (ty_shift_mask ty)))
+(rule 1 (const_shift ty (iconcat lower _))
+  (const_shift ty lower))
+(rule 1 (const_shift ty (uextend lower))
+  (const_shift ty lower))
 
 ;; I128 cases
 (rule 4 (lower (has_type $I128 (ishl x y)))
@@ -1250,8 +1263,15 @@
 (rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ishl x y)))
   (rv_vsll_vx x (value_regs_get y 0) (unmasked) ty))
 
-(rule 6 (lower (has_type (ty_vec_fits_in_register ty) (ishl x (maybe_uextend (uimm5_from_value y)))))
-  (rv_vsll_vi x y (unmasked) ty))
+(rule 6 (lower (has_type (ty_vec_fits_in_register ty) (ishl x y)))
+  (if-let uimm5 (const_vector_shift ty y))
+  (rv_vsll_vi x uimm5 (unmasked) ty))
+
+(decl pure partial const_vector_shift (Type Value) UImm5)
+(rule (const_vector_shift ty val)
+  (if-let (u64_from_imm12 imm12) (const_shift (lane_type ty) val))
+  (if-let (uimm5_from_u64 uimm5) imm12)
+  uimm5)
 
 ;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1270,14 +1290,17 @@
   (rv_srl x (value_regs_get y 0)))
 
 ;; When the RHS is known we can just encode it in the instruction.
-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))
+(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y)))
+  (if-let shift (const_shift ty y))
+  (rv_srliw (zext x) shift))
 
-(rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srliw x y))
+(rule 3 (lower (has_type $I32 (ushr x y)))
+  (if-let shift (const_shift $I32 y))
+  (rv_srliw x shift))
 
-(rule 3 (lower (has_type $I64 (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srli x y))
+(rule 3 (lower (has_type $I64 (ushr x y)))
+  (if-let shift (const_shift $I64 y))
+  (rv_srli x shift))
 
 (rule 3 (lower (has_type $I128 (ushr x y)))
   (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))
@@ -1305,8 +1328,9 @@
 (rule 4 (lower (has_type (ty_vec_fits_in_register ty) (ushr x y)))
   (rv_vsrl_vx x (value_regs_get y 0) (unmasked) ty))
 
-(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ushr x (maybe_uextend (uimm5_from_value y)))))
-  (rv_vsrl_vi x y (unmasked) ty))
+(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (ushr x y)))
+  (if-let uimm5 (const_vector_shift ty y))
+  (rv_vsrl_vi x uimm5 (unmasked) ty))
 
 ;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
@@ -1325,14 +1349,17 @@
   (rv_sra x (value_regs_get y 0)))
 
 ;; When the RHS is known we can just encode it in the instruction.
-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))
+(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y)))
+  (if-let shift (const_shift ty y))
+  (rv_sraiw (sext x) shift))
 
-(rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_sraiw x y))
+(rule 3 (lower (has_type $I32 (sshr x y)))
+  (if-let shift (const_shift $I32 y))
+  (rv_sraiw x shift))
 
-(rule 3 (lower (has_type $I64 (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srai x y))
+(rule 3 (lower (has_type $I64 (sshr x y)))
+  (if-let shift (const_shift $I64 y))
+  (rv_srai x shift))
 
 (rule 3 (lower (has_type $I128 (sshr x y)))
   (let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))
@@ -1365,9 +1392,9 @@
 (rule 4 (lower (has_type (ty_vec_fits_in_register ty) (sshr x y)))
   (rv_vsra_vx x (value_regs_get y 0) (unmasked) ty))
 
-(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (sshr x (maybe_uextend (uimm5_from_value y)))))
-  (rv_vsra_vi x y (unmasked) ty))
-
+(rule 5 (lower (has_type (ty_vec_fits_in_register ty) (sshr x y)))
+  (if-let uimm5 (const_vector_shift ty y))
+  (rv_vsra_vi x uimm5 (unmasked) ty))
 
 ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index ab5c6ddf1299..892a3d21f867 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -190,6 +190,10 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
         Imm12::from_i16(imm.as_i16() & (x as i16))
     }
 
+    fn u64_from_imm12(&mut self, imm: Imm12) -> Option<u64> {
+        u64::try_from(imm.as_i16()).ok()
+    }
+
     fn i64_generate_imm(&mut self, imm: i64) -> Option<(Imm20, Imm12)> {
         MInst::generate_imm(imm as u64)
     }

From 763b571da02f0cff91dd6d27c9922b6b92d2afa4 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 10 Oct 2023 03:22:51 -0700
Subject: [PATCH 02/21] Scaffold a way of legalizing through ISLE

This commit creates the scaffolding necessary to start writing
legalizations in ISLE. This intention here is to start out simple,
replacing sign/zero-extend instructions to `i128` with an appropriate
`iconcat` node.

The goal is to eventually rewrite all current legalization into ISLE.
Additionally all current legalization would still be shared across
backends but each backend would be able to easily define its own
legalization rules as well. The intention is to help move some of this
legalization specifically before the egraph optimization phase so
backends have an easier time of generating optimal code without having
to do so much locally.
---
 cranelift/codegen/build.rs                    |  11 +
 cranelift/codegen/meta/src/gen_inst.rs        | 314 ++++++++++--------
 cranelift/codegen/meta/src/lib.rs             |   4 -
 cranelift/codegen/src/context.rs              |   1 +
 cranelift/codegen/src/isa/mod.rs              |  10 +
 .../codegen/src/isa/riscv64/legalize.isle     |  60 ++++
 cranelift/codegen/src/isa/riscv64/legalize.rs |  87 +++++
 cranelift/codegen/src/isa/riscv64/lower.isle  |   7 +-
 cranelift/codegen/src/isa/riscv64/mod.rs      |   6 +
 .../filetests/isa/riscv64/extend-i128.clif    |  32 +-
 10 files changed, 367 insertions(+), 165 deletions(-)
 create mode 100644 cranelift/codegen/src/isa/riscv64/legalize.isle
 create mode 100644 cranelift/codegen/src/isa/riscv64/legalize.rs

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 211b62177dd2..7402ed43c50b 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -181,6 +181,7 @@ fn get_isle_compilations(
     // Preludes.
     let clif_lower_isle = out_dir.join("clif_lower.isle");
     let clif_opt_isle = out_dir.join("clif_opt.isle");
+    let clif_legalize_isle = out_dir.join("clif_legalize.isle");
     let prelude_isle =
         make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude.isle"));
     let prelude_opt_isle =
@@ -280,6 +281,16 @@ fn get_isle_compilations(
                 ],
                 untracked_inputs: vec![clif_lower_isle.clone()],
             },
+            // The risc-v legalizer.
+            IsleCompilation {
+                output: out_dir.join("legalize_riscv64.rs"),
+                inputs: vec![
+                    prelude_isle.clone(),
+                    //..
+                    src_isa_risc_v.join("legalize.isle"),
+                ],
+                untracked_inputs: vec![clif_legalize_isle.clone()],
+            },
         ],
     })
 }
diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index e43ebf79f1fe..adecfbdcbeda 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -1,18 +1,16 @@
 //! Generate instruction data (including opcodes, formats, builders, etc.).
-use std::fmt;
-use std::rc::Rc;
-
-use cranelift_codegen_shared::constant_hash;
 
 use crate::cdsl::camel_case;
 use crate::cdsl::formats::InstructionFormat;
 use crate::cdsl::instructions::{AllInstructions, Instruction};
 use crate::cdsl::operands::Operand;
 use crate::cdsl::typevar::{TypeSet, TypeVar};
-
 use crate::error;
 use crate::srcgen::{Formatter, Match};
 use crate::unique_table::{UniqueSeqTable, UniqueTable};
+use cranelift_codegen_shared::constant_hash;
+use std::fmt::{self, Write};
+use std::rc::Rc;
 
 // TypeSet indexes are encoded in 8 bits, with `0xff` reserved.
 const TYPESET_LIMIT: usize = 0xff;
@@ -1218,6 +1216,24 @@ enum IsleTarget {
     Lower,
     /// Generating code for CLIF to CLIF optimizations.
     Opt,
+    /// Code for CLIF-to-CLIF legalization.
+    Legalize,
+}
+
+impl IsleTarget {
+    fn ret_ty(&self) -> &'static str {
+        match self {
+            IsleTarget::Lower => "Inst",
+            IsleTarget::Opt => "Value",
+            IsleTarget::Legalize => "Inst",
+        }
+    }
+    fn ignore_value_lists(&self) -> bool {
+        matches!(self, IsleTarget::Opt | IsleTarget::Legalize)
+    }
+    fn match_type(&self) -> bool {
+        matches!(self, IsleTarget::Opt)
+    }
 }
 
 fn gen_common_isle(
@@ -1227,7 +1243,6 @@ fn gen_common_isle(
     isle_target: IsleTarget,
 ) {
     use std::collections::{BTreeMap, BTreeSet};
-    use std::fmt::Write;
 
     use crate::cdsl::formats::FormatField;
 
@@ -1415,12 +1430,9 @@ fn gen_common_isle(
         ";;;; Extracting Opcode, Operands, and Immediates from `InstructionData` ;;;;;;;;",
     );
     fmt.empty_line();
-    let ret_ty = match isle_target {
-        IsleTarget::Lower => "Inst",
-        IsleTarget::Opt => "Value",
-    };
+    let ret_ty = isle_target.ret_ty();
     for inst in instructions {
-        if isle_target == IsleTarget::Opt
+        if isle_target.ignore_value_lists()
             && (inst.format.has_value_list || inst.value_results.len() != 1)
         {
             continue;
@@ -1430,9 +1442,10 @@ fn gen_common_isle(
             fmt,
             "(decl {} ({}{}) {})",
             inst.name,
-            match isle_target {
-                IsleTarget::Lower => "",
-                IsleTarget::Opt => "Type ",
+            if isle_target.match_type() {
+                "Type "
+            } else {
+                ""
             },
             inst.operands_in
                 .iter()
@@ -1454,10 +1467,7 @@ fn gen_common_isle(
                 fmt,
                 "({} {}{})",
                 inst.name,
-                match isle_target {
-                    IsleTarget::Lower => "",
-                    IsleTarget::Opt => "ty ",
-                },
+                if isle_target.match_type() { "ty " } else { "" },
                 inst.operands_in
                     .iter()
                     .map(|o| { o.name })
@@ -1467,10 +1477,7 @@ fn gen_common_isle(
 
             let mut s = format!(
                 "(inst_data{} (InstructionData.{} (Opcode.{})",
-                match isle_target {
-                    IsleTarget::Lower => "",
-                    IsleTarget::Opt => " ty",
-                },
+                if isle_target.match_type() { " ty" } else { "" },
                 inst.format.name,
                 inst.camel_name
             );
@@ -1570,120 +1577,29 @@ fn gen_common_isle(
         });
         fmt.line(")");
 
-        // Generate a constructor if this is the mid-end prelude.
-        if isle_target == IsleTarget::Opt {
-            fmtln!(
-                fmt,
-                "(rule ({} ty {})",
-                inst.name,
-                inst.operands_in
-                    .iter()
-                    .map(|o| o.name)
-                    .collect::<Vec<_>>()
-                    .join(" ")
-            );
-            fmt.indent(|fmt| {
-                let mut s = format!(
-                    "(make_inst ty (InstructionData.{} (Opcode.{})",
-                    inst.format.name, inst.camel_name
-                );
-
-                // Handle values. Note that we skip generating
-                // constructors for any instructions with variadic
-                // value lists. This is fine for the mid-end because
-                // in practice only calls and branches (for branch
-                // args) use this functionality, and neither can
-                // really be optimized or rewritten in the mid-end
-                // (currently).
-                //
-                // As a consequence, we only have to handle the
-                // one-`Value` case, in which the `Value` is directly
-                // in the `InstructionData`, and the multiple-`Value`
-                // case, in which the `Value`s are in a
-                // statically-sized array (e.g. `[Value; 2]` for a
-                // binary op).
-                assert!(!inst.format.has_value_list);
-                if inst.format.num_value_operands == 1 {
-                    write!(
-                        &mut s,
-                        " {}",
-                        inst.operands_in.iter().find(|o| o.is_value()).unwrap().name
-                    )
-                    .unwrap();
-                } else if inst.format.num_value_operands > 1 {
-                    // As above, get all bindings together, and pass
-                    // to a sub-term; here we use a constructor to
-                    // build the value array.
-                    let values = inst
-                        .operands_in
-                        .iter()
-                        .filter(|o| o.is_value())
-                        .map(|o| o.name)
-                        .collect::<Vec<_>>();
-                    assert_eq!(values.len(), inst.format.num_value_operands);
-                    let values = values.join(" ");
-                    write!(
-                        &mut s,
-                        " (value_array_{}_ctor {})",
-                        inst.format.num_value_operands, values
-                    )
-                    .unwrap();
-                }
-
-                if inst.format.num_block_operands > 0 {
-                    let blocks: Vec<_> = inst
-                        .operands_in
-                        .iter()
-                        .filter(|o| o.kind.is_block())
-                        .map(|o| o.name)
-                        .collect();
-                    if inst.format.num_block_operands == 1 {
-                        write!(&mut s, " {}", blocks.first().unwrap(),).unwrap();
-                    } else {
-                        write!(
-                            &mut s,
-                            " (block_array_{} {})",
-                            inst.format.num_block_operands,
-                            blocks.join(" ")
-                        )
-                        .unwrap();
-                    }
-                }
-
-                // Immediates (non-value args).
-                for o in inst
-                    .operands_in
-                    .iter()
-                    .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block())
-                {
-                    write!(&mut s, " {}", o.name).unwrap();
-                }
-                s.push_str("))");
-                fmt.line(&s);
-            });
-            fmt.line(")");
+        match isle_target {
+            // Lowering doens't generate new CLIF nodes.
+            IsleTarget::Lower => {}
+
+            // Optimizations generate new CLIF nodes so make a new constructor
+            // which delegates to a `make_inst` helper. Note that this
+            // constructor has the same signature as the extractor above, so
+            // don't give it a prefix.
+            IsleTarget::Opt => gen_insn_constructor(fmt, inst, "", "make_inst", "Value"),
+
+            // During legalization new instructions are either inserted just
+            // before the current instruction with `ins_*` or they're replacing
+            // the current instruction with `replace_*`.
+            IsleTarget::Legalize => {
+                gen_insn_constructor(fmt, inst, "ins_", "ins", "Inst");
+                gen_insn_constructor(fmt, inst, "replace_", "replace", "Inst");
+            }
         }
 
         fmt.empty_line();
     }
 }
 
-fn gen_opt_isle(
-    formats: &[Rc<InstructionFormat>],
-    instructions: &AllInstructions,
-    fmt: &mut Formatter,
-) {
-    gen_common_isle(formats, instructions, fmt, IsleTarget::Opt);
-}
-
-fn gen_lower_isle(
-    formats: &[Rc<InstructionFormat>],
-    instructions: &AllInstructions,
-    fmt: &mut Formatter,
-) {
-    gen_common_isle(formats, instructions, fmt, IsleTarget::Lower);
-}
-
 /// Generate an `enum` immediate in ISLE.
 fn gen_isle_enum(name: &str, mut variants: Vec<&str>, fmt: &mut Formatter) {
     variants.sort();
@@ -1704,6 +1620,123 @@ fn gen_isle_enum(name: &str, mut variants: Vec<&str>, fmt: &mut Formatter) {
     fmt.empty_line();
 }
 
+fn gen_insn_constructor(
+    fmt: &mut Formatter,
+    inst: &Instruction,
+    prefix: &str,
+    common_ctor: &str,
+    ret_ty: &str,
+) {
+    // Generate a constructor if this is the mid-end prelude.
+    let input_tys = inst
+        .operands_in
+        .iter()
+        .map(|o| {
+            let ty = o.kind.rust_type;
+            if ty == "&[Value]" {
+                "ValueSlice"
+            } else {
+                ty.rsplit("::").next().unwrap()
+            }
+        })
+        .collect::<Vec<_>>()
+        .join(" ");
+    if prefix != "" {
+        fmtln!(
+            fmt,
+            "(decl {prefix}{} (Type {input_tys}) {ret_ty})",
+            inst.name
+        );
+    }
+    let input_args = inst
+        .operands_in
+        .iter()
+        .map(|o| o.name)
+        .collect::<Vec<_>>()
+        .join(" ");
+    fmtln!(fmt, "(rule ({prefix}{} ty {input_args})", inst.name);
+    fmt.indent(|fmt| {
+        let mut s = format!(
+            "({common_ctor} ty (InstructionData.{} (Opcode.{})",
+            inst.format.name, inst.camel_name
+        );
+
+        // Handle values. Note that we skip generating
+        // constructors for any instructions with variadic
+        // value lists. This is fine for the mid-end because
+        // in practice only calls and branches (for branch
+        // args) use this functionality, and neither can
+        // really be optimized or rewritten in the mid-end
+        // (currently).
+        //
+        // As a consequence, we only have to handle the
+        // one-`Value` case, in which the `Value` is directly
+        // in the `InstructionData`, and the multiple-`Value`
+        // case, in which the `Value`s are in a
+        // statically-sized array (e.g. `[Value; 2]` for a
+        // binary op).
+        assert!(!inst.format.has_value_list);
+        if inst.format.num_value_operands == 1 {
+            write!(
+                &mut s,
+                " {}",
+                inst.operands_in.iter().find(|o| o.is_value()).unwrap().name
+            )
+            .unwrap();
+        } else if inst.format.num_value_operands > 1 {
+            // As above, get all bindings together, and pass
+            // to a sub-term; here we use a constructor to
+            // build the value array.
+            let values = inst
+                .operands_in
+                .iter()
+                .filter(|o| o.is_value())
+                .map(|o| o.name)
+                .collect::<Vec<_>>();
+            assert_eq!(values.len(), inst.format.num_value_operands);
+            let values = values.join(" ");
+            write!(
+                &mut s,
+                " (value_array_{}_ctor {})",
+                inst.format.num_value_operands, values
+            )
+            .unwrap();
+        }
+
+        if inst.format.num_block_operands > 0 {
+            let blocks: Vec<_> = inst
+                .operands_in
+                .iter()
+                .filter(|o| o.kind.is_block())
+                .map(|o| o.name)
+                .collect();
+            if inst.format.num_block_operands == 1 {
+                write!(&mut s, " {}", blocks.first().unwrap(),).unwrap();
+            } else {
+                write!(
+                    &mut s,
+                    " (block_array_{} {})",
+                    inst.format.num_block_operands,
+                    blocks.join(" ")
+                )
+                .unwrap();
+            }
+        }
+
+        // Immediates (non-value args).
+        for o in inst
+            .operands_in
+            .iter()
+            .filter(|o| !o.is_value() && !o.is_varargs() && !o.kind.is_block())
+        {
+            write!(&mut s, " {}", o.name).unwrap();
+        }
+        s.push_str("))");
+        fmt.line(&s);
+    });
+    fmt.line(")");
+}
+
 /// Generate a Builder trait with methods for all instructions.
 fn gen_builder(
     instructions: &AllInstructions,
@@ -1746,10 +1779,6 @@ fn gen_builder(
 pub(crate) fn generate(
     formats: &[Rc<InstructionFormat>],
     all_inst: &AllInstructions,
-    opcode_filename: &str,
-    inst_builder_filename: &str,
-    isle_opt_filename: &str,
-    isle_lower_filename: &str,
     out_dir: &str,
     isle_dir: &str,
 ) -> Result<(), error::Error> {
@@ -1763,22 +1792,27 @@ pub(crate) fn generate(
     gen_opcodes(all_inst, &mut fmt);
     fmt.empty_line();
     gen_type_constraints(all_inst, &mut fmt);
-    fmt.update_file(opcode_filename, out_dir)?;
+    fmt.update_file("opcodes.rs", out_dir)?;
 
     // ISLE DSL: mid-end ("opt") generated bindings.
     let mut fmt = Formatter::new();
-    gen_opt_isle(&formats, all_inst, &mut fmt);
-    fmt.update_file(isle_opt_filename, isle_dir)?;
+    gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Opt);
+    fmt.update_file("clif_opt.isle", isle_dir)?;
 
     // ISLE DSL: lowering generated bindings.
     let mut fmt = Formatter::new();
-    gen_lower_isle(&formats, all_inst, &mut fmt);
-    fmt.update_file(isle_lower_filename, isle_dir)?;
+    gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Lower);
+    fmt.update_file("clif_lower.isle", isle_dir)?;
+
+    // ISLE DSL: legalizing CLIF to something backends can handle.
+    let mut fmt = Formatter::new();
+    gen_common_isle(&formats, all_inst, &mut fmt, IsleTarget::Legalize);
+    fmt.update_file("clif_legalize.isle", isle_dir)?;
 
     // Instruction builder.
     let mut fmt = Formatter::new();
     gen_builder(all_inst, &formats, &mut fmt);
-    fmt.update_file(inst_builder_filename, out_dir)?;
+    fmt.update_file("inst_builder.rs", out_dir)?;
 
     Ok(())
 }
diff --git a/cranelift/codegen/meta/src/lib.rs b/cranelift/codegen/meta/src/lib.rs
index 689d3508b06c..84ff52f94074 100644
--- a/cranelift/codegen/meta/src/lib.rs
+++ b/cranelift/codegen/meta/src/lib.rs
@@ -37,10 +37,6 @@ pub fn generate(isas: &[isa::Isa], out_dir: &str, isle_dir: &str) -> Result<(),
     gen_inst::generate(
         &shared_defs.all_formats,
         &shared_defs.all_instructions,
-        "opcodes.rs",
-        "inst_builder.rs",
-        "clif_opt.isle",
-        "clif_lower.isle",
         out_dir,
         isle_dir,
     )?;
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index ca3d5fb73628..2a8ddf8aa79b 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -291,6 +291,7 @@ impl Context {
 
         // Run some specific legalizations only.
         simple_legalize(&mut self.func, &mut self.cfg, isa);
+        isa.legalize_function(&mut self.func, &mut self.cfg);
         self.verify_if(isa)
     }
 
diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs
index 8e68e7aaba27..a9344072e613 100644
--- a/cranelift/codegen/src/isa/mod.rs
+++ b/cranelift/codegen/src/isa/mod.rs
@@ -264,6 +264,16 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
     /// Get the ISA-dependent maximum vector register size, in bytes.
     fn dynamic_vector_bytes(&self, dynamic_ty: ir::Type) -> u32;
 
+    /// Edit the IR of `func` to remove any CLIF operations which aren't
+    /// supported by this backend.
+    ///
+    /// Note that this is done before optimizations are executed.
+    fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
+        // TODO: there shouldn't be a default impl.
+        let _ = func;
+        let _ = cfg;
+    }
+
     /// Compile the given function.
     fn compile_function(
         &self,
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.isle b/cranelift/codegen/src/isa/riscv64/legalize.isle
new file mode 100644
index 000000000000..9556f4609b4f
--- /dev/null
+++ b/cranelift/codegen/src/isa/riscv64/legalize.isle
@@ -0,0 +1,60 @@
+(type Inst (primitive Inst))
+
+;; Helper used in generated extractors for each CLIF instruction to extract
+;; information from an `Inst`.
+(decl inst_data (InstructionData) Inst)
+(extern extractor infallible inst_data inst_data)
+
+;; Helper used to perform an `ins()` operation which inserts just before the
+;; current instruction being legalized.
+;;
+;; This is invoked by generated constructors for each CLIF instruction with the
+;; `ins_*` prefix.
+(decl ins (Type InstructionData) Inst)
+(extern constructor ins ins)
+
+;; Helper used to perform a `replace()` operation which replaces the current
+;; instruction being legalized.
+;;
+;; This is invoked by generated constructors for each CLIF instruction with the
+;; `replace_*` prefix.
+(decl replace (Type InstructionData) Inst)
+(extern constructor replace replace)
+
+;; Constructors for value arrays.
+(decl value_array_2_ctor (Value Value) ValueArray2)
+(extern constructor value_array_2_ctor value_array_2_ctor)
+(decl value_array_3_ctor (Value Value Value) ValueArray3)
+(extern constructor value_array_3_ctor value_array_3_ctor)
+
+(decl pure first_result (Inst) Value)
+(extern constructor first_result first_result)
+(convert Inst Value first_result)
+
+(decl has_type (Type Inst) Inst)
+(extractor (has_type ty inst) (and (result_type ty) inst))
+(decl result_type (Type) Inst)
+(extern extractor result_type result_type)
+
+;; Helper to generate an `iconst` value which has type `I64` and the provided
+;; value.
+(decl ins_i64 (i64) Value)
+(rule (ins_i64 val) (ins_iconst $I64 (imm64 (i64_as_u64 val))))
+
+;; Optionally "legalize" an instruction to a new instruction.
+;;
+;; This is used to transform CLIF into something which can be handled by each
+;; individual backend. There is both common functionality of legalization for
+;; all backends in addition to backend-specific legalization.
+(decl partial legalize (Inst) Inst)
+
+;; Transform extend-to-i128 to an `iconcat` instruction
+(rule 0 (legalize (has_type $I128 (uextend x @ (value_type (fits_in_32 _)))))
+  (replace_iconcat $I64 (ins_uextend $I64 x) (ins_i64 0)))
+(rule 1 (legalize (has_type $I128 (uextend x @ (value_type $I64))))
+  (replace_iconcat $I64 x (ins_i64 0)))
+(rule 0 (legalize (has_type $I128 (sextend x @ (value_type (fits_in_32 _)))))
+  (let ((val64 Value (ins_sextend $I64 x)))
+    (replace_iconcat $I64 val64 (ins_sshr $I64 val64 (ins_i64 63)))))
+(rule 1 (legalize (has_type $I128 (sextend x @ (value_type $I64))))
+  (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.rs b/cranelift/codegen/src/isa/riscv64/legalize.rs
new file mode 100644
index 000000000000..81f432fb2f98
--- /dev/null
+++ b/cranelift/codegen/src/isa/riscv64/legalize.rs
@@ -0,0 +1,87 @@
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph;
+use crate::ir;
+use crate::isa::riscv64::Riscv64Backend;
+
+// Used by ISLE
+use crate::ir::condcodes::*;
+use crate::ir::immediates::*;
+use crate::ir::types::*;
+use crate::ir::*;
+use crate::machinst::isle::*;
+
+#[allow(dead_code, unused_variables)]
+mod generated {
+    include!(concat!(env!("ISLE_DIR"), "/legalize_riscv64.rs"));
+}
+
+pub fn run(isa: &Riscv64Backend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
+    let mut cx = Legalize {
+        isa,
+        pos: FuncCursor::new(func),
+        cfg,
+        replace: None,
+    };
+    let func_begin = cx.pos.position();
+    cx.pos.set_position(func_begin);
+    while let Some(_block) = cx.pos.next_block() {
+        while let Some(inst) = cx.pos.next_inst() {
+            cx.replace = Some(inst);
+            match generated::constructor_legalize(&mut cx, inst) {
+                Some(_) => {}
+                None => {}
+            }
+        }
+    }
+}
+
+struct Legalize<'a> {
+    isa: &'a Riscv64Backend,
+    pos: FuncCursor<'a>,
+    cfg: &'a mut flowgraph::ControlFlowGraph,
+    replace: Option<Inst>,
+}
+
+impl generated::Context for Legalize<'_> {
+    crate::isle_common_prelude_methods!();
+
+    fn inst_data(&mut self, inst: Inst) -> InstructionData {
+        self.pos.func.dfg.insts[inst]
+    }
+
+    fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
+        self.pos.ins().build(data.clone(), ty).0
+    }
+
+    fn replace(&mut self, ty: Type, data: &InstructionData) -> Inst {
+        let ins = self.pos.func.dfg.replace(self.replace.unwrap());
+        ins.build(data.clone(), ty).0
+    }
+
+    fn value_type(&mut self, val: Value) -> Type {
+        self.pos.func.dfg.value_type(val)
+    }
+
+    fn first_result(&mut self, inst: Inst) -> Value {
+        let results = self.pos.func.dfg.inst_results(inst);
+        assert_eq!(results.len(), 1,);
+        results[0]
+    }
+
+    fn result_type(&mut self, inst: Inst) -> Option<Type> {
+        let results = self.pos.func.dfg.inst_results(inst);
+        if results.len() == 1 {
+            Some(self.value_type(results[0]))
+        } else {
+            None
+        }
+    }
+
+    fn value_array_2_ctor(&mut self, arg0: Value, arg1: Value) -> ValueArray2 {
+        [arg0, arg1]
+    }
+
+    fn value_array_3_ctor(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3 {
+        [arg0, arg1, arg2]
+    }
+}
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index de51dc5f020d..1fcdde79d14f 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1109,17 +1109,14 @@
 
 
 ;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (rule 0 (lower (has_type (fits_in_64 _) (uextend val)))
   (zext val))
-(rule 1 (lower (has_type $I128 (uextend val)))
-  (value_regs (zext val) (imm $I64 0)))
 
 ;;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
 (rule 0 (lower (has_type (fits_in_64 _) (sextend val @ (value_type in_ty))))
   (sext val))
-(rule 1 (lower (has_type $I128 (sextend val @ (value_type in_ty))))
-  (let ((lo XReg (sext val)))
-    (value_regs lo (rv_srai lo (imm12_const 63)))))
 
 ;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs
index 6d727bbc6267..13b3b125762a 100644
--- a/cranelift/codegen/src/isa/riscv64/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/mod.rs
@@ -1,6 +1,7 @@
 //! risc-v 64-bit Instruction Set Architecture.
 
 use crate::dominator_tree::DominatorTree;
+use crate::flowgraph;
 use crate::ir::{Function, Type};
 use crate::isa::riscv64::settings as riscv_settings;
 use crate::isa::{Builder as IsaBuilder, FunctionAlignment, OwnedTargetIsa, TargetIsa};
@@ -17,6 +18,7 @@ use cranelift_control::ControlPlane;
 use target_lexicon::{Architecture, Triple};
 mod abi;
 pub(crate) mod inst;
+mod legalize;
 mod lower;
 mod settings;
 #[cfg(feature = "unwind")]
@@ -101,6 +103,10 @@ impl TargetIsa for Riscv64Backend {
         16
     }
 
+    fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
+        legalize::run(self, func, cfg);
+    }
+
     fn triple(&self) -> &Triple {
         &self.triple
     }
diff --git a/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif b/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif
index 96ca30750cce..f1832e6258dd 100644
--- a/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/extend-i128.clif
@@ -26,15 +26,15 @@ block0(v0: i32):
 
 ; VCode:
 ; block0:
-;   slli a2,a0,32
-;   srli a0,a2,32
+;   slli a3,a0,32
+;   srli a0,a3,32
 ;   li a1,0
 ;   ret
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
-;   slli a2, a0, 0x20
-;   srli a0, a2, 0x20
+;   slli a3, a0, 0x20
+;   srli a0, a3, 0x20
 ;   mv a1, zero
 ;   ret
 
@@ -46,15 +46,15 @@ block0(v0: i16):
 
 ; VCode:
 ; block0:
-;   slli a2,a0,48
-;   srli a0,a2,48
+;   slli a3,a0,48
+;   srli a0,a3,48
 ;   li a1,0
 ;   ret
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
-;   slli a2, a0, 0x30
-;   srli a0, a2, 0x30
+;   slli a3, a0, 0x30
+;   srli a0, a3, 0x30
 ;   mv a1, zero
 ;   ret
 
@@ -118,15 +118,15 @@ block0(v0: i16):
 
 ; VCode:
 ; block0:
-;   slli a2,a0,48
-;   srai a0,a2,48
+;   slli a3,a0,48
+;   srai a0,a3,48
 ;   srai a1,a0,63
 ;   ret
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
-;   slli a2, a0, 0x30
-;   srai a0, a2, 0x30
+;   slli a3, a0, 0x30
+;   srai a0, a3, 0x30
 ;   srai a1, a0, 0x3f
 ;   ret
 
@@ -138,15 +138,15 @@ block0(v0: i8):
 
 ; VCode:
 ; block0:
-;   slli a2,a0,56
-;   srai a0,a2,56
+;   slli a3,a0,56
+;   srai a0,a3,56
 ;   srai a1,a0,63
 ;   ret
 ;
 ; Disassembled:
 ; block0: ; offset 0x0
-;   slli a2, a0, 0x38
-;   srai a0, a2, 0x38
+;   slli a3, a0, 0x38
+;   srai a0, a3, 0x38
 ;   srai a1, a0, 0x3f
 ;   ret
 

From 10a5ada057441a529fed2762679275eb5e62ec4a Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Wed, 11 Oct 2023 23:40:45 -0700
Subject: [PATCH 03/21] Legalize `stack_store` in ISLE

---
 cranelift/codegen/meta/src/gen_inst.rs          |  5 ++++-
 cranelift/codegen/src/context.rs                |  2 +-
 cranelift/codegen/src/isa/riscv64/legalize.isle | 16 ++++++++++++++++
 cranelift/codegen/src/isa/riscv64/legalize.rs   | 13 +++++++++++++
 cranelift/codegen/src/isle_prelude.rs           |  6 ++++++
 cranelift/codegen/src/prelude.isle              |  4 ++++
 6 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index adecfbdcbeda..69ccb2257790 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -1229,7 +1229,7 @@ impl IsleTarget {
         }
     }
     fn ignore_value_lists(&self) -> bool {
-        matches!(self, IsleTarget::Opt | IsleTarget::Legalize)
+        matches!(self, IsleTarget::Opt)
     }
     fn match_type(&self) -> bool {
         matches!(self, IsleTarget::Opt)
@@ -1627,6 +1627,9 @@ fn gen_insn_constructor(
     common_ctor: &str,
     ret_ty: &str,
 ) {
+    if inst.format.has_value_list {
+        return;
+    }
     // Generate a constructor if this is the mid-end prelude.
     let input_tys = inst
         .operands_in
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index 2a8ddf8aa79b..71790988b44e 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -290,8 +290,8 @@ impl Context {
         self.loop_analysis.clear();
 
         // Run some specific legalizations only.
-        simple_legalize(&mut self.func, &mut self.cfg, isa);
         isa.legalize_function(&mut self.func, &mut self.cfg);
+        simple_legalize(&mut self.func, &mut self.cfg, isa);
         self.verify_if(isa)
     }
 
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.isle b/cranelift/codegen/src/isa/riscv64/legalize.isle
index 9556f4609b4f..a5342e28b81f 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.isle
+++ b/cranelift/codegen/src/isa/riscv64/legalize.isle
@@ -26,6 +26,13 @@
 (extern constructor value_array_2_ctor value_array_2_ctor)
 (decl value_array_3_ctor (Value Value Value) ValueArray3)
 (extern constructor value_array_3_ctor value_array_3_ctor)
+(decl value_list_slice (ValueSlice) ValueList)
+(extern extractor infallible value_list_slice value_list_slice)
+(decl value_slice_unwrap (Value ValueSlice) ValueSlice)
+(extern extractor value_slice_unwrap value_slice_unwrap)
+(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList)
+(extractor (unwrap_head_value_list_1 head tail)
+           (value_list_slice (value_slice_unwrap head tail)))
 
 (decl pure first_result (Inst) Value)
 (extern constructor first_result first_result)
@@ -48,6 +55,8 @@
 ;; all backends in addition to backend-specific legalization.
 (decl partial legalize (Inst) Inst)
 
+;; ========= `{u,s}extend` ====================================================
+
 ;; Transform extend-to-i128 to an `iconcat` instruction
 (rule 0 (legalize (has_type $I128 (uextend x @ (value_type (fits_in_32 _)))))
   (replace_iconcat $I64 (ins_uextend $I64 x) (ins_i64 0)))
@@ -58,3 +67,10 @@
     (replace_iconcat $I64 val64 (ins_sshr $I64 val64 (ins_i64 63)))))
 (rule 1 (legalize (has_type $I128 (sextend x @ (value_type $I64))))
   (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
+
+;; ========= `stack_store` ====================================================
+
+;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0)
+(rule (legalize (stack_store val @ (value_type ty) slot offset))
+  (let ((addr Value (ins_stack_addr $I64 slot offset)))
+    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.rs b/cranelift/codegen/src/isa/riscv64/legalize.rs
index 81f432fb2f98..8be3d839faef 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.rs
+++ b/cranelift/codegen/src/isa/riscv64/legalize.rs
@@ -77,6 +77,19 @@ impl generated::Context for Legalize<'_> {
         }
     }
 
+    fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
+        (list, 0)
+    }
+
+    fn value_slice_unwrap(&mut self, slice: ValueSlice) -> Option<(Value, ValueSlice)> {
+        let (list, off) = slice;
+        if let Some(val) = list.get(off, &self.pos.func.dfg.value_lists) {
+            Some((val, (list, off + 1)))
+        } else {
+            None
+        }
+    }
+
     fn value_array_2_ctor(&mut self, arg0: Value, arg1: Value) -> ValueArray2 {
         [arg0, arg1]
     }
diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs
index 88b6abdb8eff..f9bbb1153748 100644
--- a/cranelift/codegen/src/isle_prelude.rs
+++ b/cranelift/codegen/src/isle_prelude.rs
@@ -828,6 +828,12 @@ macro_rules! isle_common_prelude_methods {
             MemFlags::trusted()
         }
 
+        fn mem_flags_notrap(&mut self) -> MemFlags {
+            let mut mflags = MemFlags::new();
+            mflags.set_notrap();
+            mflags
+        }
+
         #[inline]
         fn intcc_unsigned(&mut self, x: &IntCC) -> IntCC {
             x.unsigned()
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index c61876b8d481..81333bb593da 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -337,6 +337,10 @@
 (decl pure mem_flags_trusted () MemFlags)
 (extern constructor mem_flags_trusted mem_flags_trusted)
 
+;; `MemFlags::notrap`
+(decl pure mem_flags_notrap () MemFlags)
+(extern constructor mem_flags_notrap mem_flags_notrap)
+
 ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Swap args of an IntCC flag.

From 4af131f4a1a4a0851f18af3170d89957e4958629 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 12 Oct 2023 00:09:37 -0700
Subject: [PATCH 04/21] Start a `legalize_shared.isle`

---
 cranelift/codegen/build.rs                    |   6 +-
 .../codegen/src/isa/riscv64/legalize.isle     |  77 +-----------
 .../src/legalizer/legalize_shared.isle        | 111 ++++++++++++++++++
 3 files changed, 120 insertions(+), 74 deletions(-)
 create mode 100644 cranelift/codegen/src/legalizer/legalize_shared.isle

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 7402ed43c50b..5be6665f5c80 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -188,6 +188,10 @@ fn get_isle_compilations(
         make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude_opt.isle"));
     let prelude_lower_isle =
         make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("prelude_lower.isle"));
+    let legalize_shared_isle = make_isle_source_path_relative(
+        &cur_dir,
+        crate_dir.join("src/legalizer/legalize_shared.isle"),
+    );
 
     // Directory for mid-end optimizations.
     let src_opts = make_isle_source_path_relative(&cur_dir, crate_dir.join("src").join("opts"));
@@ -286,7 +290,7 @@ fn get_isle_compilations(
                 output: out_dir.join("legalize_riscv64.rs"),
                 inputs: vec![
                     prelude_isle.clone(),
-                    //..
+                    legalize_shared_isle.clone(),
                     src_isa_risc_v.join("legalize.isle"),
                 ],
                 untracked_inputs: vec![clif_legalize_isle.clone()],
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.isle b/cranelift/codegen/src/isa/riscv64/legalize.isle
index a5342e28b81f..0011a053f07d 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.isle
+++ b/cranelift/codegen/src/isa/riscv64/legalize.isle
@@ -1,76 +1,7 @@
-(type Inst (primitive Inst))
-
-;; Helper used in generated extractors for each CLIF instruction to extract
-;; information from an `Inst`.
-(decl inst_data (InstructionData) Inst)
-(extern extractor infallible inst_data inst_data)
-
-;; Helper used to perform an `ins()` operation which inserts just before the
-;; current instruction being legalized.
-;;
-;; This is invoked by generated constructors for each CLIF instruction with the
-;; `ins_*` prefix.
-(decl ins (Type InstructionData) Inst)
-(extern constructor ins ins)
-
-;; Helper used to perform a `replace()` operation which replaces the current
-;; instruction being legalized.
-;;
-;; This is invoked by generated constructors for each CLIF instruction with the
-;; `replace_*` prefix.
-(decl replace (Type InstructionData) Inst)
-(extern constructor replace replace)
-
-;; Constructors for value arrays.
-(decl value_array_2_ctor (Value Value) ValueArray2)
-(extern constructor value_array_2_ctor value_array_2_ctor)
-(decl value_array_3_ctor (Value Value Value) ValueArray3)
-(extern constructor value_array_3_ctor value_array_3_ctor)
-(decl value_list_slice (ValueSlice) ValueList)
-(extern extractor infallible value_list_slice value_list_slice)
-(decl value_slice_unwrap (Value ValueSlice) ValueSlice)
-(extern extractor value_slice_unwrap value_slice_unwrap)
-(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList)
-(extractor (unwrap_head_value_list_1 head tail)
-           (value_list_slice (value_slice_unwrap head tail)))
-
-(decl pure first_result (Inst) Value)
-(extern constructor first_result first_result)
-(convert Inst Value first_result)
-
-(decl has_type (Type Inst) Inst)
-(extractor (has_type ty inst) (and (result_type ty) inst))
-(decl result_type (Type) Inst)
-(extern extractor result_type result_type)
-
-;; Helper to generate an `iconst` value which has type `I64` and the provided
-;; value.
-(decl ins_i64 (i64) Value)
-(rule (ins_i64 val) (ins_iconst $I64 (imm64 (i64_as_u64 val))))
-
-;; Optionally "legalize" an instruction to a new instruction.
-;;
-;; This is used to transform CLIF into something which can be handled by each
-;; individual backend. There is both common functionality of legalization for
-;; all backends in addition to backend-specific legalization.
-(decl partial legalize (Inst) Inst)
-
 ;; ========= `{u,s}extend` ====================================================
 
-;; Transform extend-to-i128 to an `iconcat` instruction
-(rule 0 (legalize (has_type $I128 (uextend x @ (value_type (fits_in_32 _)))))
-  (replace_iconcat $I64 (ins_uextend $I64 x) (ins_i64 0)))
-(rule 1 (legalize (has_type $I128 (uextend x @ (value_type $I64))))
-  (replace_iconcat $I64 x (ins_i64 0)))
-(rule 0 (legalize (has_type $I128 (sextend x @ (value_type (fits_in_32 _)))))
-  (let ((val64 Value (ins_sextend $I64 x)))
-    (replace_iconcat $I64 val64 (ins_sshr $I64 val64 (ins_i64 63)))))
-(rule 1 (legalize (has_type $I128 (sextend x @ (value_type $I64))))
-  (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
-
-;; ========= `stack_store` ====================================================
+(rule (legalize (has_type $I128 (uextend x)))
+  (replace_uextend_i128 x))
 
-;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0)
-(rule (legalize (stack_store val @ (value_type ty) slot offset))
-  (let ((addr Value (ins_stack_addr $I64 slot offset)))
-    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
+(rule (legalize (has_type $I128 (sextend x)))
+  (replace_sextend_i128 x))
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
new file mode 100644
index 000000000000..ca417a123fc1
--- /dev/null
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -0,0 +1,111 @@
+;; Legalizing CLIF to CLIF
+;;
+;; This file is one of the main entry points for performing "legalizations" that
+;; transform CLIF to CLIF to simplify the IR for one phase or another. One
+;; example of a legalization is replacing "macro instructions" with their
+;; components, such as the `stack_store` CLIF instruction with a `stack_addr`
+;; plus a `store`. This way backends don't need to implement lowering for
+;; all three instructions, only the latter two.
+;;
+;; Backends also have their own forms of legalization as well. For example
+;; instead of natively implementing a `uextend`-to-128-bits backends can choose
+;; to legalize the `uextend` instruction in this case to be replaced with an
+;; `iconcat`. This can help reduce the number of lowerings that a backend needs
+;; while sharing legalization code with other backends.
+;;
+;; This file is organized as shared support for all backends. This contains
+;; the definition of the `legalize` function which is the main entrypoint for
+;; transforming instructions. Some required legalizations are defined in this
+;; file such as legalizing the CLIF `stack_store` pseudo-instruction. Otherwise
+;; platform-neutral helpers are defined in this file that each backend can
+;; opt-in to as desired.
+
+;; ========= Type Definitions =================================================
+
+(type Inst (primitive Inst))
+
+;; ========= Helper extractors/constructors ===================================
+
+;; Helper used in generated extractors for each CLIF instruction to extract
+;; information from an `Inst`.
+(decl inst_data (InstructionData) Inst)
+(extern extractor infallible inst_data inst_data)
+
+;; Helper used to perform an `ins()` operation which inserts just before the
+;; current instruction being legalized.
+;;
+;; This is invoked by generated constructors for each CLIF instruction with the
+;; `ins_*` prefix.
+(decl ins (Type InstructionData) Inst)
+(extern constructor ins ins)
+
+;; Helper used to perform a `replace()` operation which replaces the current
+;; instruction being legalized.
+;;
+;; This is invoked by generated constructors for each CLIF instruction with the
+;; `replace_*` prefix.
+(decl replace (Type InstructionData) Inst)
+(extern constructor replace replace)
+
+;; Constructors for value arrays.
+(decl value_array_2_ctor (Value Value) ValueArray2)
+(extern constructor value_array_2_ctor value_array_2_ctor)
+(decl value_array_3_ctor (Value Value Value) ValueArray3)
+(extern constructor value_array_3_ctor value_array_3_ctor)
+(decl value_list_slice (ValueSlice) ValueList)
+(extern extractor infallible value_list_slice value_list_slice)
+(decl value_slice_unwrap (Value ValueSlice) ValueSlice)
+(extern extractor value_slice_unwrap value_slice_unwrap)
+(decl unwrap_head_value_list_1 (Value ValueSlice) ValueList)
+(extractor (unwrap_head_value_list_1 head tail)
+           (value_list_slice (value_slice_unwrap head tail)))
+
+(decl pure first_result (Inst) Value)
+(extern constructor first_result first_result)
+(convert Inst Value first_result)
+
+(decl has_type (Type Inst) Inst)
+(extractor (has_type ty inst) (and (result_type ty) inst))
+(decl result_type (Type) Inst)
+(extern extractor result_type result_type)
+
+;; Helper to generate an `iconst` value which has type `I64` and the provided
+;; value.
+(decl ins_i64 (i64) Value)
+(rule (ins_i64 val) (ins_iconst $I64 (imm64 (i64_as_u64 val))))
+
+;; Optionally "legalize" an instruction to a new instruction.
+;;
+;; This is used to transform CLIF into something which can be handled by each
+;; individual backend. There is both common functionality of legalization for
+;; all backends in addition to backend-specific legalization.
+;;
+;; Note that this function is defined here and while some legalizations are
+;; performed in this file there are also backend-specific legalizations
+;; throughout each backend as well.
+(decl partial legalize (Inst) Inst)
+
+;; ========= `{u,s}extend` ====================================================
+
+;; Helper to replace a `uextend` to i128 with an `iconcat` instruction instead.
+(decl replace_uextend_i128 (Value) Inst)
+(rule 0 (replace_uextend_i128 x @ (value_type (fits_in_32 _)))
+  (replace_iconcat $I64 (ins_uextend $I64 x) (ins_i64 0)))
+(rule 1 (replace_uextend_i128 x @ (value_type $I64))
+  (replace_iconcat $I64 x (ins_i64 0)))
+
+;; Helper to replace a `sextend` to i128 with an `iconcat` instruction instead.
+(decl replace_sextend_i128 (Value) Inst)
+(rule 0 (replace_sextend_i128 x @ (value_type (fits_in_32 _)))
+  (let ((val64 Value (ins_sextend $I64 x)))
+    (replace_iconcat $I64 val64 (ins_sshr $I64 val64 (ins_i64 63)))))
+(rule 1 (replace_sextend_i128 x @ (value_type $I64))
+  (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
+
+;; ========= `stack_store` ====================================================
+
+;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0)
+(rule (legalize (stack_store val @ (value_type ty) slot offset))
+  (let ((addr Value (ins_stack_addr $I64 slot offset)))
+    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
+

From 52938c18fd201868cddf07d9ba86a32dd99b5b18 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 12 Oct 2023 00:26:47 -0700
Subject: [PATCH 05/21] Add legalizer for all backends

---
 cranelift/codegen/build.rs                    | 30 ++++++
 cranelift/codegen/src/isa/aarch64/legalize.rs | 26 +++++
 cranelift/codegen/src/isa/aarch64/mod.rs      |  6 ++
 cranelift/codegen/src/isa/mod.rs              |  6 +-
 cranelift/codegen/src/isa/riscv64/legalize.rs | 86 ++---------------
 cranelift/codegen/src/isa/s390x/legalize.rs   | 26 +++++
 cranelift/codegen/src/isa/s390x/mod.rs        |  6 ++
 cranelift/codegen/src/isa/x64/legalize.rs     | 30 ++++++
 cranelift/codegen/src/isa/x64/mod.rs          |  6 ++
 cranelift/codegen/src/legalizer/isle.rs       | 96 +++++++++++++++++++
 cranelift/codegen/src/legalizer/mod.rs        |  1 +
 11 files changed, 234 insertions(+), 85 deletions(-)
 create mode 100644 cranelift/codegen/src/isa/aarch64/legalize.rs
 create mode 100644 cranelift/codegen/src/isa/s390x/legalize.rs
 create mode 100644 cranelift/codegen/src/isa/x64/legalize.rs
 create mode 100644 cranelift/codegen/src/legalizer/isle.rs

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 5be6665f5c80..e5167d87bb97 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -249,6 +249,16 @@ fn get_isle_compilations(
                 ],
                 untracked_inputs: vec![clif_lower_isle.clone()],
             },
+            // The x86-64 legalizer.
+            IsleCompilation {
+                output: out_dir.join("legalize_x64.rs"),
+                inputs: vec![
+                    prelude_isle.clone(),
+                    legalize_shared_isle.clone(),
+                    // NB: x86-64-specific legalization here when it exists
+                ],
+                untracked_inputs: vec![clif_legalize_isle.clone()],
+            },
             // The aarch64 instruction selector.
             IsleCompilation {
                 output: out_dir.join("isle_aarch64.rs"),
@@ -262,6 +272,16 @@ fn get_isle_compilations(
                 ],
                 untracked_inputs: vec![clif_lower_isle.clone()],
             },
+            // The aarch64 legalizer.
+            IsleCompilation {
+                output: out_dir.join("legalize_aarch64.rs"),
+                inputs: vec![
+                    prelude_isle.clone(),
+                    legalize_shared_isle.clone(),
+                    // NB: aarch64-specific legalization here when it exists
+                ],
+                untracked_inputs: vec![clif_legalize_isle.clone()],
+            },
             // The s390x instruction selector.
             IsleCompilation {
                 output: out_dir.join("isle_s390x.rs"),
@@ -273,6 +293,16 @@ fn get_isle_compilations(
                 ],
                 untracked_inputs: vec![clif_lower_isle.clone()],
             },
+            // The s390x legalizer.
+            IsleCompilation {
+                output: out_dir.join("legalize_s390x.rs"),
+                inputs: vec![
+                    prelude_isle.clone(),
+                    legalize_shared_isle.clone(),
+                    // NB: s390x-specific legalization here when it exists
+                ],
+                untracked_inputs: vec![clif_legalize_isle.clone()],
+            },
             // The risc-v instruction selector.
             IsleCompilation {
                 output: out_dir.join("isle_riscv64.rs"),
diff --git a/cranelift/codegen/src/isa/aarch64/legalize.rs b/cranelift/codegen/src/isa/aarch64/legalize.rs
new file mode 100644
index 000000000000..811b207f6d21
--- /dev/null
+++ b/cranelift/codegen/src/isa/aarch64/legalize.rs
@@ -0,0 +1,26 @@
+use crate::flowgraph;
+use crate::ir;
+use crate::isa::aarch64::AArch64Backend;
+use crate::legalizer::isle;
+
+// Used by ISLE
+use crate::ir::condcodes::*;
+use crate::ir::immediates::*;
+use crate::ir::types::*;
+use crate::ir::*;
+use crate::machinst::isle::*;
+
+#[allow(dead_code, unused_variables)]
+mod generated {
+    include!(concat!(env!("ISLE_DIR"), "/legalize_aarch64.rs"));
+}
+
+pub fn run(isa: &AArch64Backend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
+    crate::legalizer::isle::run(isa, func, cfg, |cx, i| {
+        generated::constructor_legalize(cx, i)
+    })
+}
+
+impl generated::Context for isle::LegalizeContext<'_, AArch64Backend> {
+    crate::isle_common_legalizer_methods!();
+}
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index 59541d9d1517..e40ac7ad667a 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -1,6 +1,7 @@
 //! ARM 64-bit Instruction Set Architecture.
 
 use crate::dominator_tree::DominatorTree;
+use crate::flowgraph;
 use crate::ir::{Function, Type};
 use crate::isa::aarch64::settings as aarch64_settings;
 #[cfg(feature = "unwind")]
@@ -20,6 +21,7 @@ use target_lexicon::{Aarch64Architecture, Architecture, OperatingSystem, Triple}
 // New backend:
 mod abi;
 pub mod inst;
+mod legalize;
 mod lower;
 mod pcc;
 pub mod settings;
@@ -119,6 +121,10 @@ impl TargetIsa for AArch64Backend {
         16
     }
 
+    fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) {
+        legalize::run(self, func, cfg);
+    }
+
     #[cfg(feature = "unwind")]
     fn emit_unwind_info(
         &self,
diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs
index a9344072e613..1c625264ca2a 100644
--- a/cranelift/codegen/src/isa/mod.rs
+++ b/cranelift/codegen/src/isa/mod.rs
@@ -268,11 +268,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
     /// supported by this backend.
     ///
     /// Note that this is done before optimizations are executed.
-    fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
-        // TODO: there shouldn't be a default impl.
-        let _ = func;
-        let _ = cfg;
-    }
+    fn legalize_function(&self, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph);
 
     /// Compile the given function.
     fn compile_function(
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.rs b/cranelift/codegen/src/isa/riscv64/legalize.rs
index 8be3d839faef..19df44ecd18d 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.rs
+++ b/cranelift/codegen/src/isa/riscv64/legalize.rs
@@ -1,7 +1,7 @@
-use crate::cursor::{Cursor, FuncCursor};
 use crate::flowgraph;
 use crate::ir;
 use crate::isa::riscv64::Riscv64Backend;
+use crate::legalizer::isle;
 
 // Used by ISLE
 use crate::ir::condcodes::*;
@@ -16,85 +16,11 @@ mod generated {
 }
 
 pub fn run(isa: &Riscv64Backend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
-    let mut cx = Legalize {
-        isa,
-        pos: FuncCursor::new(func),
-        cfg,
-        replace: None,
-    };
-    let func_begin = cx.pos.position();
-    cx.pos.set_position(func_begin);
-    while let Some(_block) = cx.pos.next_block() {
-        while let Some(inst) = cx.pos.next_inst() {
-            cx.replace = Some(inst);
-            match generated::constructor_legalize(&mut cx, inst) {
-                Some(_) => {}
-                None => {}
-            }
-        }
-    }
+    crate::legalizer::isle::run(isa, func, cfg, |cx, i| {
+        generated::constructor_legalize(cx, i)
+    })
 }
 
-struct Legalize<'a> {
-    isa: &'a Riscv64Backend,
-    pos: FuncCursor<'a>,
-    cfg: &'a mut flowgraph::ControlFlowGraph,
-    replace: Option<Inst>,
-}
-
-impl generated::Context for Legalize<'_> {
-    crate::isle_common_prelude_methods!();
-
-    fn inst_data(&mut self, inst: Inst) -> InstructionData {
-        self.pos.func.dfg.insts[inst]
-    }
-
-    fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
-        self.pos.ins().build(data.clone(), ty).0
-    }
-
-    fn replace(&mut self, ty: Type, data: &InstructionData) -> Inst {
-        let ins = self.pos.func.dfg.replace(self.replace.unwrap());
-        ins.build(data.clone(), ty).0
-    }
-
-    fn value_type(&mut self, val: Value) -> Type {
-        self.pos.func.dfg.value_type(val)
-    }
-
-    fn first_result(&mut self, inst: Inst) -> Value {
-        let results = self.pos.func.dfg.inst_results(inst);
-        assert_eq!(results.len(), 1,);
-        results[0]
-    }
-
-    fn result_type(&mut self, inst: Inst) -> Option<Type> {
-        let results = self.pos.func.dfg.inst_results(inst);
-        if results.len() == 1 {
-            Some(self.value_type(results[0]))
-        } else {
-            None
-        }
-    }
-
-    fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
-        (list, 0)
-    }
-
-    fn value_slice_unwrap(&mut self, slice: ValueSlice) -> Option<(Value, ValueSlice)> {
-        let (list, off) = slice;
-        if let Some(val) = list.get(off, &self.pos.func.dfg.value_lists) {
-            Some((val, (list, off + 1)))
-        } else {
-            None
-        }
-    }
-
-    fn value_array_2_ctor(&mut self, arg0: Value, arg1: Value) -> ValueArray2 {
-        [arg0, arg1]
-    }
-
-    fn value_array_3_ctor(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3 {
-        [arg0, arg1, arg2]
-    }
+impl generated::Context for isle::LegalizeContext<'_, Riscv64Backend> {
+    crate::isle_common_legalizer_methods!();
 }
diff --git a/cranelift/codegen/src/isa/s390x/legalize.rs b/cranelift/codegen/src/isa/s390x/legalize.rs
new file mode 100644
index 000000000000..2411dbf2d250
--- /dev/null
+++ b/cranelift/codegen/src/isa/s390x/legalize.rs
@@ -0,0 +1,26 @@
+use crate::flowgraph;
+use crate::ir;
+use crate::isa::s390x::S390xBackend;
+use crate::legalizer::isle;
+
+// Used by ISLE
+use crate::ir::condcodes::*;
+use crate::ir::immediates::*;
+use crate::ir::types::*;
+use crate::ir::*;
+use crate::machinst::isle::*;
+
+#[allow(dead_code, unused_variables)]
+mod generated {
+    include!(concat!(env!("ISLE_DIR"), "/legalize_s390x.rs"));
+}
+
+pub fn run(isa: &S390xBackend, func: &mut ir::Function, cfg: &mut flowgraph::ControlFlowGraph) {
+    crate::legalizer::isle::run(isa, func, cfg, |cx, i| {
+        generated::constructor_legalize(cx, i)
+    })
+}
+
+impl generated::Context for isle::LegalizeContext<'_, S390xBackend> {
+    crate::isle_common_legalizer_methods!();
+}
diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs
index 5840450dd7ed..b18efe64b7c2 100644
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -1,6 +1,7 @@
 //! IBM Z 64-bit Instruction Set Architecture.
 
 use crate::dominator_tree::DominatorTree;
+use crate::flowgraph;
 use crate::ir::{Function, Type};
 use crate::isa::s390x::settings as s390x_settings;
 #[cfg(feature = "unwind")]
@@ -20,6 +21,7 @@ use target_lexicon::{Architecture, Triple};
 // New backend:
 mod abi;
 pub(crate) mod inst;
+mod legalize;
 mod lower;
 mod settings;
 
@@ -115,6 +117,10 @@ impl TargetIsa for S390xBackend {
         16
     }
 
+    fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) {
+        legalize::run(self, func, cfg);
+    }
+
     #[cfg(feature = "unwind")]
     fn emit_unwind_info(
         &self,
diff --git a/cranelift/codegen/src/isa/x64/legalize.rs b/cranelift/codegen/src/isa/x64/legalize.rs
new file mode 100644
index 000000000000..87c30553527c
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/legalize.rs
@@ -0,0 +1,30 @@
+use crate::flowgraph;
+use crate::ir;
+use crate::isa::x64::X64Backend;
+use crate::legalizer::isle;
+
+// Used by ISLE
+use crate::ir::condcodes::*;
+use crate::ir::immediates::*;
+use crate::ir::types::*;
+use crate::ir::*;
+use crate::machinst::isle::*;
+
+#[allow(dead_code, unused_variables)]
+mod generated {
+    include!(concat!(env!("ISLE_DIR"), "/legalize_x64.rs"));
+}
+
+pub(crate) fn run(
+    isa: &X64Backend,
+    func: &mut ir::Function,
+    cfg: &mut flowgraph::ControlFlowGraph,
+) {
+    crate::legalizer::isle::run(isa, func, cfg, |cx, i| {
+        generated::constructor_legalize(cx, i)
+    })
+}
+
+impl generated::Context for isle::LegalizeContext<'_, X64Backend> {
+    crate::isle_common_legalizer_methods!();
+}
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index 38a9c668c9d3..d574d804c730 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -4,6 +4,7 @@ pub use self::inst::{args, CallInfo, EmitInfo, EmitState, Inst};
 
 use super::{OwnedTargetIsa, TargetIsa};
 use crate::dominator_tree::DominatorTree;
+use crate::flowgraph;
 use crate::ir::{types, Function, Type};
 #[cfg(feature = "unwind")]
 use crate::isa::unwind::systemv;
@@ -23,6 +24,7 @@ use target_lexicon::Triple;
 mod abi;
 pub mod encoding;
 mod inst;
+mod legalize;
 mod lower;
 mod pcc;
 pub mod settings;
@@ -191,6 +193,10 @@ impl TargetIsa for X64Backend {
     fn has_x86_pmaddubsw_lowering(&self) -> bool {
         self.x64_flags.use_ssse3()
     }
+
+    fn legalize_function(&self, func: &mut Function, cfg: &mut flowgraph::ControlFlowGraph) {
+        legalize::run(self, func, cfg);
+    }
 }
 
 impl fmt::Display for X64Backend {
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
new file mode 100644
index 000000000000..aa88b89f99a5
--- /dev/null
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -0,0 +1,96 @@
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph;
+use crate::ir::{self, Inst};
+
+pub fn run<T>(
+    backend: &T,
+    func: &mut ir::Function,
+    cfg: &mut flowgraph::ControlFlowGraph,
+    constructor_legalize: fn(&mut LegalizeContext<'_, T>, Inst) -> Option<Inst>,
+) {
+    let mut cx = LegalizeContext {
+        backend,
+        pos: FuncCursor::new(func),
+        cfg,
+        replace: None,
+    };
+    let func_begin = cx.pos.position();
+    cx.pos.set_position(func_begin);
+    while let Some(_block) = cx.pos.next_block() {
+        while let Some(inst) = cx.pos.next_inst() {
+            cx.replace = Some(inst);
+            match constructor_legalize(&mut cx, inst) {
+                Some(_) => {}
+                None => {}
+            }
+        }
+    }
+}
+
+pub struct LegalizeContext<'a, T> {
+    pub backend: &'a T,
+    pub pos: FuncCursor<'a>,
+    pub cfg: &'a mut flowgraph::ControlFlowGraph,
+    pub replace: Option<Inst>,
+}
+
+/// Generate common methods for the legalization trait on `LegalizeContext`.
+#[macro_export]
+macro_rules! isle_common_legalizer_methods {
+    () => {
+        crate::isle_common_prelude_methods!();
+
+        fn inst_data(&mut self, inst: Inst) -> InstructionData {
+            self.pos.func.dfg.insts[inst]
+        }
+
+        fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
+            self.pos.ins().build(data.clone(), ty).0
+        }
+
+        fn replace(&mut self, ty: Type, data: &InstructionData) -> Inst {
+            let ins = self.pos.func.dfg.replace(self.replace.unwrap());
+            ins.build(data.clone(), ty).0
+        }
+
+        fn value_type(&mut self, val: Value) -> Type {
+            self.pos.func.dfg.value_type(val)
+        }
+
+        fn first_result(&mut self, inst: Inst) -> Value {
+            let results = self.pos.func.dfg.inst_results(inst);
+            assert_eq!(results.len(), 1,);
+            results[0]
+        }
+
+        fn result_type(&mut self, inst: Inst) -> Option<Type> {
+            let results = self.pos.func.dfg.inst_results(inst);
+            if results.len() == 1 {
+                Some(self.value_type(results[0]))
+            } else {
+                None
+            }
+        }
+
+        fn value_list_slice(&mut self, list: ValueList) -> ValueSlice {
+            (list, 0)
+        }
+
+        fn value_slice_unwrap(&mut self, slice: ValueSlice) -> Option<(Value, ValueSlice)> {
+            let (list, off) = slice;
+            if let Some(val) = list.get(off, &self.pos.func.dfg.value_lists) {
+                Some((val, (list, off + 1)))
+            } else {
+                None
+            }
+        }
+
+        fn value_array_2_ctor(&mut self, arg0: Value, arg1: Value) -> ValueArray2 {
+            [arg0, arg1]
+        }
+
+        fn value_array_3_ctor(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3 {
+            [arg0, arg1, arg2]
+        }
+    };
+}
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index e186b153091d..4a2ebcb028d4 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -22,6 +22,7 @@ use crate::isa::TargetIsa;
 use crate::trace;
 
 mod globalvalue;
+pub mod isle;
 mod table;
 
 use self::globalvalue::expand_global_value;

From f034387bf4dd77009f0e4d5055790336a4c6db04 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 12 Oct 2023 00:28:42 -0700
Subject: [PATCH 06/21] Delete prior legalization of `StackStore`

Now it's done in ISLE!
---
 cranelift/codegen/src/legalizer/mod.rs | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index 4a2ebcb028d4..694c69c17ea3 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -99,25 +99,6 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
                     mflags.set_notrap();
                     pos.func.dfg.replace(inst).load(ty, mflags, addr, 0);
                 }
-                InstructionData::StackStore {
-                    opcode: ir::Opcode::StackStore,
-                    arg,
-                    stack_slot,
-                    offset,
-                } => {
-                    let addr_ty = isa.pointer_type();
-
-                    let mut pos = FuncCursor::new(pos.func).at_inst(inst);
-                    pos.use_srcloc(inst);
-
-                    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
-
-                    // Stack slots are required to be accessible.
-                    // We can't currently ensure that they are aligned.
-                    let mut mflags = MemFlags::new();
-                    mflags.set_notrap();
-                    pos.func.dfg.replace(inst).store(mflags, arg, addr, 0);
-                }
                 InstructionData::DynamicStackLoad {
                     opcode: ir::Opcode::DynamicStackLoad,
                     dynamic_stack_slot,

From 8821900bc137e04b1192594f4be2688cea91fe39 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 12 Oct 2023 00:30:33 -0700
Subject: [PATCH 07/21] Move `StackLoad` legalizing to ISLE

---
 .../src/legalizer/legalize_shared.isle        | 16 +++++-
 cranelift/codegen/src/legalizer/mod.rs        | 56 +------------------
 2 files changed, 16 insertions(+), 56 deletions(-)

diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index ca417a123fc1..4ac2e43b80e2 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -102,10 +102,24 @@
 (rule 1 (replace_sextend_i128 x @ (value_type $I64))
   (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
 
-;; ========= `stack_store` ====================================================
+;; ========= `{dynamic_,}stack_{load,store}` ==================================
+
+;; stack_load(slot, offset) => load(stack_addr(slot, offset), 0)
+(rule (legalize (has_type ty (stack_load slot offset)))
+  (let ((addr Value (ins_stack_addr $I64 slot offset)))
+    (replace_load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
 
 ;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0)
 (rule (legalize (stack_store val @ (value_type ty) slot offset))
   (let ((addr Value (ins_stack_addr $I64 slot offset)))
     (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
 
+;; dynamic_stack_load(slot) => load(dynamic_stack_addr(slot), 0)
+(rule (legalize (has_type ty (dynamic_stack_load slot)))
+  (let ((addr Value (ins_dynamic_stack_addr $I64 slot)))
+    (replace_load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
+
+;; dynamic_stack_store(val, slot) => store(val, dynamic_stack_addr(slot), 0)
+(rule (legalize (dynamic_stack_store val @ (value_type ty) slot))
+  (let ((addr Value (ins_dynamic_stack_addr $I64 slot)))
+    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index 694c69c17ea3..47d7607d7260 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -17,7 +17,7 @@ use crate::cursor::{Cursor, FuncCursor};
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::immediates::Imm64;
 use crate::ir::types::{self, I128, I64};
-use crate::ir::{self, InstBuilder, InstructionData, MemFlags, Value};
+use crate::ir::{self, InstBuilder, InstructionData, Value};
 use crate::isa::TargetIsa;
 use crate::trace;
 
@@ -80,60 +80,6 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
                     opcode: ir::Opcode::GlobalValue,
                     global_value,
                 } => expand_global_value(inst, &mut pos.func, isa, global_value),
-                InstructionData::StackLoad {
-                    opcode: ir::Opcode::StackLoad,
-                    stack_slot,
-                    offset,
-                } => {
-                    let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst));
-                    let addr_ty = isa.pointer_type();
-
-                    let mut pos = FuncCursor::new(pos.func).at_inst(inst);
-                    pos.use_srcloc(inst);
-
-                    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
-
-                    // Stack slots are required to be accessible.
-                    // We can't currently ensure that they are aligned.
-                    let mut mflags = MemFlags::new();
-                    mflags.set_notrap();
-                    pos.func.dfg.replace(inst).load(ty, mflags, addr, 0);
-                }
-                InstructionData::DynamicStackLoad {
-                    opcode: ir::Opcode::DynamicStackLoad,
-                    dynamic_stack_slot,
-                } => {
-                    let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst));
-                    assert!(ty.is_dynamic_vector());
-                    let addr_ty = isa.pointer_type();
-
-                    let mut pos = FuncCursor::new(pos.func).at_inst(inst);
-                    pos.use_srcloc(inst);
-
-                    let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot);
-
-                    // Stack slots are required to be accessible and aligned.
-                    let mflags = MemFlags::trusted();
-                    pos.func.dfg.replace(inst).load(ty, mflags, addr, 0);
-                }
-                InstructionData::DynamicStackStore {
-                    opcode: ir::Opcode::DynamicStackStore,
-                    arg,
-                    dynamic_stack_slot,
-                } => {
-                    pos.use_srcloc(inst);
-                    let addr_ty = isa.pointer_type();
-                    let vector_ty = pos.func.dfg.value_type(arg);
-                    assert!(vector_ty.is_dynamic_vector());
-
-                    let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot);
-
-                    let mut mflags = MemFlags::new();
-                    // Stack slots are required to be accessible and aligned.
-                    mflags.set_notrap();
-                    mflags.set_aligned();
-                    pos.func.dfg.replace(inst).store(mflags, arg, addr, 0);
-                }
                 InstructionData::TableAddr {
                     opcode: ir::Opcode::TableAddr,
                     table,

From 4b60f319cfab925211b5f633f11e1f55f697f1c6 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 09:39:27 -0700
Subject: [PATCH 08/21] Move trapnz-style legalization to ISLE

---
 cranelift/codegen/src/legalizer/isle.rs       | 124 +++++++++++++++++-
 .../src/legalizer/legalize_shared.isle        |  17 +++
 cranelift/codegen/src/legalizer/mod.rs        |   5 +-
 3 files changed, 139 insertions(+), 7 deletions(-)

diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index aa88b89f99a5..4f3353886a21 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -1,11 +1,12 @@
 use crate::cursor::{Cursor, FuncCursor};
-use crate::flowgraph;
-use crate::ir::{self, Inst};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::{self, Inst, InstBuilder};
+use crate::trace;
 
 pub fn run<T>(
     backend: &T,
     func: &mut ir::Function,
-    cfg: &mut flowgraph::ControlFlowGraph,
+    cfg: &mut ControlFlowGraph,
     constructor_legalize: fn(&mut LegalizeContext<'_, T>, Inst) -> Option<Inst>,
 ) {
     let mut cx = LegalizeContext {
@@ -30,7 +31,7 @@ pub fn run<T>(
 pub struct LegalizeContext<'a, T> {
     pub backend: &'a T,
     pub pos: FuncCursor<'a>,
-    pub cfg: &'a mut flowgraph::ControlFlowGraph,
+    pub cfg: &'a mut ControlFlowGraph,
     pub replace: Option<Inst>,
 }
 
@@ -92,5 +93,120 @@ macro_rules! isle_common_legalizer_methods {
         fn value_array_3_ctor(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3 {
             [arg0, arg1, arg2]
         }
+
+        fn expand_trapz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+            crate::legalizer::isle::expand_cond_trap(
+                self.replace.unwrap(),
+                self.pos.func,
+                self.cfg,
+                ir::Opcode::Trapz,
+                arg,
+                *cc,
+            )
+        }
+
+        fn expand_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+            crate::legalizer::isle::expand_cond_trap(
+                self.replace.unwrap(),
+                self.pos.func,
+                self.cfg,
+                ir::Opcode::Trapnz,
+                arg,
+                *cc,
+            )
+        }
+
+        fn expand_resumable_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+            crate::legalizer::isle::expand_cond_trap(
+                self.replace.unwrap(),
+                self.pos.func,
+                self.cfg,
+                ir::Opcode::ResumableTrapnz,
+                arg,
+                *cc,
+            )
+        }
+    };
+}
+
+/// Custom expansion for conditional trap instructions.
+pub fn expand_cond_trap(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    opcode: ir::Opcode,
+    arg: ir::Value,
+    code: ir::TrapCode,
+) -> Inst {
+    trace!(
+        "expanding conditional trap: {:?}: {}",
+        inst,
+        func.dfg.display_inst(inst)
+    );
+
+    // Parse the instruction.
+    let trapz = match opcode {
+        ir::Opcode::Trapz => true,
+        ir::Opcode::Trapnz | ir::Opcode::ResumableTrapnz => false,
+        _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst)),
     };
+
+    // Split the block after `inst`:
+    //
+    //     trapnz arg
+    //     ..
+    //
+    // Becomes:
+    //
+    //     brif arg, new_block_trap, new_block_resume
+    //
+    //   new_block_trap:
+    //     trap
+    //
+    //   new_block_resume:
+    //     ..
+    let old_block = func
+        .layout
+        .inst_block(inst)
+        .expect("Instruction not in layout.");
+    let new_block_trap = func.dfg.make_block();
+    let new_block_resume = func.dfg.make_block();
+
+    // Trapping is a rare event, mark the trapping block as cold.
+    func.layout.set_cold(new_block_trap);
+
+    // Replace trap instruction by the inverted condition.
+    if trapz {
+        func.dfg
+            .replace(inst)
+            .brif(arg, new_block_resume, &[], new_block_trap, &[]);
+    } else {
+        func.dfg
+            .replace(inst)
+            .brif(arg, new_block_trap, &[], new_block_resume, &[]);
+    }
+
+    // Insert the new label and the unconditional trap terminator.
+    let mut pos = FuncCursor::new(func).after_inst(inst);
+    pos.use_srcloc(inst);
+    pos.insert_block(new_block_trap);
+
+    let inst = match opcode {
+        ir::Opcode::Trapz | ir::Opcode::Trapnz => pos.ins().trap(code),
+        ir::Opcode::ResumableTrapnz => {
+            pos.ins().resumable_trap(code);
+            pos.ins().jump(new_block_resume, &[])
+        }
+        _ => unreachable!(),
+    };
+
+    // Insert the new label and resume the execution when the trap fails.
+    pos.insert_block(new_block_resume);
+
+    // Finally update the CFG.
+    cfg.recompute_block(pos.func, old_block);
+    cfg.recompute_block(pos.func, new_block_resume);
+    cfg.recompute_block(pos.func, new_block_trap);
+
+    inst
 }
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index 4ac2e43b80e2..c260af68cfd6 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -123,3 +123,20 @@
 (rule (legalize (dynamic_stack_store val @ (value_type ty) slot))
   (let ((addr Value (ins_dynamic_stack_addr $I64 slot)))
     (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
+
+;; ========= `trapz`, `trapnz`, `resumable_trapnz` ============================
+
+;; Delegate back to Rust to perform the basic-block manipulation necessary to
+;; expand these instructions
+
+(rule (legalize (trapz val code)) (expand_trapz val code))
+(decl expand_trapz (Value TrapCode) Inst)
+(extern constructor expand_trapz expand_trapz)
+
+(rule (legalize (trapnz val code)) (expand_trapnz val code))
+(decl expand_trapnz (Value TrapCode) Inst)
+(extern constructor expand_trapnz expand_trapnz)
+
+(rule (legalize (resumable_trapnz val code)) (expand_resumable_trapnz val code))
+(decl expand_resumable_trapnz (Value TrapCode) Inst)
+(extern constructor expand_resumable_trapnz expand_resumable_trapnz)
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index 47d7607d7260..9b9af32f605f 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -67,12 +67,11 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
             match pos.func.dfg.insts[inst] {
                 // control flow
                 InstructionData::CondTrap {
-                    opcode:
-                        opcode @ (ir::Opcode::Trapnz | ir::Opcode::Trapz | ir::Opcode::ResumableTrapnz),
+                    opcode: ir::Opcode::Trapnz,
                     arg,
                     code,
                 } => {
-                    expand_cond_trap(inst, &mut pos.func, cfg, opcode, arg, code);
+                    expand_cond_trap(inst, &mut pos.func, cfg, ir::Opcode::Trapnz, arg, code);
                 }
 
                 // memory and constants

From 24a78771a7cb4e525c19597103b77f2b9d2fc85a Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 09:40:13 -0700
Subject: [PATCH 09/21] Move `*_imm` and `*_not` legalization into ISLE

---
 .../src/legalizer/legalize_shared.isle        | 87 ++++++++++++++++++
 cranelift/codegen/src/legalizer/mod.rs        | 92 +------------------
 2 files changed, 89 insertions(+), 90 deletions(-)

diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index c260af68cfd6..ec72df479277 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -140,3 +140,90 @@
 (rule (legalize (resumable_trapnz val code)) (expand_resumable_trapnz val code))
 (decl expand_resumable_trapnz (Value TrapCode) Inst)
 (extern constructor expand_resumable_trapnz expand_resumable_trapnz)
+
+;; ========= `*_imm` instrs ===================================================
+
+(rule (legalize (has_type ty (band_imm val imm)))
+  (replace_band ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (bor_imm val imm)))
+  (replace_bor ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (bxor_imm val imm)))
+  (replace_bxor ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (ishl_imm val imm)))
+  (replace_ishl ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (sshr_imm val imm)))
+  (replace_sshr ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (ushr_imm val imm)))
+  (replace_ushr ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (rotl_imm val imm)))
+  (replace_rotl ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (rotr_imm val imm)))
+  (replace_rotr ty val (ins_unsigned_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (iadd_imm val imm)))
+  (replace_iadd ty val (ins_signed_const (lane_type ty) imm)))
+
+;; note the reversed arguments in the `isub`
+(rule (legalize (has_type ty (irsub_imm val imm)))
+  (replace_isub ty (ins_signed_const (lane_type ty) imm) val))
+
+(rule (legalize (has_type ty (imul_imm val imm)))
+  (replace_imul ty val (ins_signed_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (sdiv_imm val imm)))
+  (replace_sdiv ty val (ins_signed_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (srem_imm val imm)))
+  (replace_srem ty val (ins_signed_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (udiv_imm val imm)))
+  (replace_udiv ty val (ins_signed_const (lane_type ty) imm)))
+
+(rule (legalize (has_type ty (urem_imm val imm)))
+  (replace_urem ty val (ins_signed_const (lane_type ty) imm)))
+
+(rule (legalize (icmp_imm cc val @ (value_type ty) imm))
+  (replace_icmp ty cc val (ins_signed_const (lane_type ty) imm)))
+
+(decl ins_unsigned_const (Type Imm64) Value)
+(rule (ins_unsigned_const $I8 (u64_from_imm64 n))
+  (ins_iconst $I8 (imm64 (u64_and n 0xff))))
+(rule (ins_unsigned_const $I16 (u64_from_imm64 n))
+  (ins_iconst $I16 (imm64 (u64_and n 0xffff))))
+(rule (ins_unsigned_const $I32 (u64_from_imm64 n))
+  (ins_iconst $I32 (imm64 (u64_and n 0xffffffff))))
+(rule (ins_unsigned_const $I64 imm)
+  (ins_iconst $I64 imm))
+(rule (ins_unsigned_const $I128 imm)
+  (ins_iconcat $I64 (ins_iconst $I64 imm) (ins_i64 0)))
+
+(decl ins_signed_const (Type Imm64) Value)
+(rule 0 (ins_signed_const (fits_in_64 ty) imm)
+  (ins_unsigned_const ty imm))
+(rule 1 (ins_signed_const $I128 imm)
+  (ins_iconcat $I64
+    (ins_iconst $I64 imm)
+    (ins_iconst $I64 (imm64_sshr $I64 imm (imm64 63)))))
+
+;; ========= `*_not` instrs ===================================================
+
+;; Legalize the fused bitwise-plus-not instructions into simpler
+;; instructions to assist with optimizations. Lowering will
+;; pattern match this sequence regardless when architectures
+;; support the instruction natively.
+
+(rule (legalize (has_type ty (band_not a b)))
+  (replace_band ty a (ins_bnot ty b)))
+
+(rule (legalize (has_type ty (bor_not a b)))
+  (replace_bor ty a (ins_bnot ty b)))
+
+(rule (legalize (has_type ty (bxor_not a b)))
+  (replace_bxor ty a (ins_bnot ty b)))
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index 9b9af32f605f..b1a20c890c8e 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -87,100 +87,12 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
                 } => expand_table_addr(isa, inst, &mut pos.func, table, arg, offset),
 
                 InstructionData::BinaryImm64 { opcode, arg, imm } => {
-                    let is_signed = match opcode {
-                        ir::Opcode::IaddImm
-                        | ir::Opcode::IrsubImm
-                        | ir::Opcode::ImulImm
-                        | ir::Opcode::SdivImm
-                        | ir::Opcode::SremImm => true,
-                        _ => false,
-                    };
-
-                    let imm = imm_const(&mut pos, arg, imm, is_signed);
-                    let replace = pos.func.dfg.replace(inst);
                     match opcode {
-                        // bitops
-                        ir::Opcode::BandImm => {
-                            replace.band(arg, imm);
-                        }
-                        ir::Opcode::BorImm => {
-                            replace.bor(arg, imm);
-                        }
-                        ir::Opcode::BxorImm => {
-                            replace.bxor(arg, imm);
-                        }
-                        // bitshifting
-                        ir::Opcode::IshlImm => {
-                            replace.ishl(arg, imm);
-                        }
-                        ir::Opcode::RotlImm => {
-                            replace.rotl(arg, imm);
-                        }
-                        ir::Opcode::RotrImm => {
-                            replace.rotr(arg, imm);
-                        }
-                        ir::Opcode::SshrImm => {
-                            replace.sshr(arg, imm);
-                        }
-                        ir::Opcode::UshrImm => {
-                            replace.ushr(arg, imm);
-                        }
-                        // math
                         ir::Opcode::IaddImm => {
+                            let imm = imm_const(&mut pos, arg, imm, true);
+                            let replace = pos.func.dfg.replace(inst);
                             replace.iadd(arg, imm);
                         }
-                        ir::Opcode::IrsubImm => {
-                            // note: arg order reversed
-                            replace.isub(imm, arg);
-                        }
-                        ir::Opcode::ImulImm => {
-                            replace.imul(arg, imm);
-                        }
-                        ir::Opcode::SdivImm => {
-                            replace.sdiv(arg, imm);
-                        }
-                        ir::Opcode::SremImm => {
-                            replace.srem(arg, imm);
-                        }
-                        ir::Opcode::UdivImm => {
-                            replace.udiv(arg, imm);
-                        }
-                        ir::Opcode::UremImm => {
-                            replace.urem(arg, imm);
-                        }
-                        _ => prev_pos = pos.position(),
-                    };
-                }
-
-                // comparisons
-                InstructionData::IntCompareImm {
-                    opcode: ir::Opcode::IcmpImm,
-                    cond,
-                    arg,
-                    imm,
-                } => {
-                    let imm = imm_const(&mut pos, arg, imm, true);
-                    pos.func.dfg.replace(inst).icmp(cond, arg, imm);
-                }
-
-                // Legalize the fused bitwise-plus-not instructions into simpler
-                // instructions to assist with optimizations. Lowering will
-                // pattern match this sequence regardless when architectures
-                // support the instruction natively.
-                InstructionData::Binary { opcode, args } => {
-                    match opcode {
-                        ir::Opcode::BandNot => {
-                            let neg = pos.ins().bnot(args[1]);
-                            pos.func.dfg.replace(inst).band(args[0], neg);
-                        }
-                        ir::Opcode::BorNot => {
-                            let neg = pos.ins().bnot(args[1]);
-                            pos.func.dfg.replace(inst).bor(args[0], neg);
-                        }
-                        ir::Opcode::BxorNot => {
-                            let neg = pos.ins().bnot(args[1]);
-                            pos.func.dfg.replace(inst).bxor(args[0], neg);
-                        }
                         _ => prev_pos = pos.position(),
                     };
                 }

From 0f9ff0be352867aed0919bc08147fc714ed7eb5c Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 11:28:18 -0700
Subject: [PATCH 10/21] Move global value legalization into ISLE

Update the `legalize` constructor to provide more control over where to
resume legalization.
---
 cranelift/codegen/src/isa/aarch64/legalize.rs |   3 +-
 cranelift/codegen/src/isa/riscv64/legalize.rs |   3 +-
 cranelift/codegen/src/isa/s390x/legalize.rs   |   3 +-
 cranelift/codegen/src/isa/x64/legalize.rs     |   3 +-
 .../codegen/src/legalizer/globalvalue.rs      |  34 +----
 cranelift/codegen/src/legalizer/isle.rs       |  97 ++++++++++++---
 .../src/legalizer/legalize_shared.isle        | 117 +++++++++++++++++-
 7 files changed, 205 insertions(+), 55 deletions(-)

diff --git a/cranelift/codegen/src/isa/aarch64/legalize.rs b/cranelift/codegen/src/isa/aarch64/legalize.rs
index 811b207f6d21..203c9eebd942 100644
--- a/cranelift/codegen/src/isa/aarch64/legalize.rs
+++ b/cranelift/codegen/src/isa/aarch64/legalize.rs
@@ -4,13 +4,14 @@ use crate::isa::aarch64::AArch64Backend;
 use crate::legalizer::isle;
 
 // Used by ISLE
+use crate::cursor::CursorPosition;
 use crate::ir::condcodes::*;
 use crate::ir::immediates::*;
 use crate::ir::types::*;
 use crate::ir::*;
 use crate::machinst::isle::*;
 
-#[allow(dead_code, unused_variables)]
+#[allow(dead_code, unused_variables, unreachable_patterns)]
 mod generated {
     include!(concat!(env!("ISLE_DIR"), "/legalize_aarch64.rs"));
 }
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.rs b/cranelift/codegen/src/isa/riscv64/legalize.rs
index 19df44ecd18d..d7b2df23d2d4 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.rs
+++ b/cranelift/codegen/src/isa/riscv64/legalize.rs
@@ -4,13 +4,14 @@ use crate::isa::riscv64::Riscv64Backend;
 use crate::legalizer::isle;
 
 // Used by ISLE
+use crate::cursor::CursorPosition;
 use crate::ir::condcodes::*;
 use crate::ir::immediates::*;
 use crate::ir::types::*;
 use crate::ir::*;
 use crate::machinst::isle::*;
 
-#[allow(dead_code, unused_variables)]
+#[allow(dead_code, unused_variables, unreachable_patterns)]
 mod generated {
     include!(concat!(env!("ISLE_DIR"), "/legalize_riscv64.rs"));
 }
diff --git a/cranelift/codegen/src/isa/s390x/legalize.rs b/cranelift/codegen/src/isa/s390x/legalize.rs
index 2411dbf2d250..36c541ebd97a 100644
--- a/cranelift/codegen/src/isa/s390x/legalize.rs
+++ b/cranelift/codegen/src/isa/s390x/legalize.rs
@@ -4,13 +4,14 @@ use crate::isa::s390x::S390xBackend;
 use crate::legalizer::isle;
 
 // Used by ISLE
+use crate::cursor::CursorPosition;
 use crate::ir::condcodes::*;
 use crate::ir::immediates::*;
 use crate::ir::types::*;
 use crate::ir::*;
 use crate::machinst::isle::*;
 
-#[allow(dead_code, unused_variables)]
+#[allow(dead_code, unused_variables, unreachable_patterns)]
 mod generated {
     include!(concat!(env!("ISLE_DIR"), "/legalize_s390x.rs"));
 }
diff --git a/cranelift/codegen/src/isa/x64/legalize.rs b/cranelift/codegen/src/isa/x64/legalize.rs
index 87c30553527c..f9a7aaada42c 100644
--- a/cranelift/codegen/src/isa/x64/legalize.rs
+++ b/cranelift/codegen/src/isa/x64/legalize.rs
@@ -4,13 +4,14 @@ use crate::isa::x64::X64Backend;
 use crate::legalizer::isle;
 
 // Used by ISLE
+use crate::cursor::CursorPosition;
 use crate::ir::condcodes::*;
 use crate::ir::immediates::*;
 use crate::ir::types::*;
 use crate::ir::*;
 use crate::machinst::isle::*;
 
-#[allow(dead_code, unused_variables)]
+#[allow(dead_code, unused_variables, unreachable_patterns)]
 mod generated {
     include!(concat!(env!("ISLE_DIR"), "/legalize_x64.rs"));
 }
diff --git a/cranelift/codegen/src/legalizer/globalvalue.rs b/cranelift/codegen/src/legalizer/globalvalue.rs
index 9837dd59278e..e2660bb8789d 100644
--- a/cranelift/codegen/src/legalizer/globalvalue.rs
+++ b/cranelift/codegen/src/legalizer/globalvalue.rs
@@ -33,24 +33,11 @@ pub fn expand_global_value(
             global_type,
             flags,
         } => load_addr(inst, func, base, offset, global_type, flags, isa),
-        ir::GlobalValueData::Symbol { tls, .. } => symbol(inst, func, global_value, isa, tls),
-        ir::GlobalValueData::DynScaleTargetConst { vector_type } => {
-            const_vector_scale(inst, func, vector_type, isa)
-        }
+        ir::GlobalValueData::Symbol { .. } => unreachable!(),
+        ir::GlobalValueData::DynScaleTargetConst { .. } => unreachable!(),
     }
 }
 
-fn const_vector_scale(inst: ir::Inst, func: &mut ir::Function, ty: ir::Type, isa: &dyn TargetIsa) {
-    assert!(ty.bytes() <= 16);
-
-    // Use a minimum of 128-bits for the base type.
-    let base_bytes = std::cmp::max(ty.bytes(), 16);
-    let scale = (isa.dynamic_vector_bytes(ty) / base_bytes) as i64;
-    assert!(scale > 0);
-    let pos = FuncCursor::new(func).at_inst(inst);
-    pos.func.dfg.replace(inst).iconst(isa.pointer_type(), scale);
-}
-
 /// Expand a `global_value` instruction for a vmctx global.
 fn vmctx_addr(global_value: ir::GlobalValue, inst: ir::Inst, func: &mut ir::Function) {
     // Get the value representing the `vmctx` argument.
@@ -132,20 +119,3 @@ fn load_addr(
         .replace(inst)
         .load(global_type, flags, base_addr, offset);
 }
-
-/// Expand a `global_value` instruction for a symbolic name global.
-fn symbol(
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    gv: ir::GlobalValue,
-    isa: &dyn TargetIsa,
-    tls: bool,
-) {
-    let ptr_ty = isa.pointer_type();
-
-    if tls {
-        func.dfg.replace(inst).tls_value(ptr_ty, gv);
-    } else {
-        func.dfg.replace(inst).symbol_value(ptr_ty, gv);
-    }
-}
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index 4f3353886a21..57ebabc07277 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -1,27 +1,33 @@
-use crate::cursor::{Cursor, FuncCursor};
+use crate::cursor::{Cursor, CursorPosition, FuncCursor};
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::{self, Inst, InstBuilder};
+use crate::isa::TargetIsa;
 use crate::trace;
 
 pub fn run<T>(
     backend: &T,
     func: &mut ir::Function,
     cfg: &mut ControlFlowGraph,
-    constructor_legalize: fn(&mut LegalizeContext<'_, T>, Inst) -> Option<Inst>,
-) {
+    constructor_legalize: fn(&mut LegalizeContext<'_, T>, Inst) -> Option<CursorPosition>,
+) where
+    T: TargetIsa,
+{
+    let pos = FuncCursor::new(func);
     let mut cx = LegalizeContext {
         backend,
-        pos: FuncCursor::new(func),
+        prev_position: pos.position(),
+        pos,
         cfg,
         replace: None,
     };
     let func_begin = cx.pos.position();
     cx.pos.set_position(func_begin);
     while let Some(_block) = cx.pos.next_block() {
+        cx.prev_position = cx.pos.position();
         while let Some(inst) = cx.pos.next_inst() {
             cx.replace = Some(inst);
             match constructor_legalize(&mut cx, inst) {
-                Some(_) => {}
+                Some(pos) => cx.pos.set_position(pos),
                 None => {}
             }
         }
@@ -33,6 +39,7 @@ pub struct LegalizeContext<'a, T> {
     pub pos: FuncCursor<'a>,
     pub cfg: &'a mut ControlFlowGraph,
     pub replace: Option<Inst>,
+    pub prev_position: CursorPosition,
 }
 
 /// Generate common methods for the legalization trait on `LegalizeContext`.
@@ -45,6 +52,10 @@ macro_rules! isle_common_legalizer_methods {
             self.pos.func.dfg.insts[inst]
         }
 
+        fn gv_data(&mut self, gv: GlobalValue) -> GlobalValueData {
+            self.pos.func.global_values[gv].clone()
+        }
+
         fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
             self.pos.ins().build(data.clone(), ty).0
         }
@@ -94,7 +105,7 @@ macro_rules! isle_common_legalizer_methods {
             [arg0, arg1, arg2]
         }
 
-        fn expand_trapz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+        fn expand_trapz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition {
             crate::legalizer::isle::expand_cond_trap(
                 self.replace.unwrap(),
                 self.pos.func,
@@ -105,7 +116,7 @@ macro_rules! isle_common_legalizer_methods {
             )
         }
 
-        fn expand_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+        fn expand_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition {
             crate::legalizer::isle::expand_cond_trap(
                 self.replace.unwrap(),
                 self.pos.func,
@@ -116,7 +127,7 @@ macro_rules! isle_common_legalizer_methods {
             )
         }
 
-        fn expand_resumable_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> Inst {
+        fn expand_resumable_trapnz(&mut self, arg: Value, cc: &ir::TrapCode) -> CursorPosition {
             crate::legalizer::isle::expand_cond_trap(
                 self.replace.unwrap(),
                 self.pos.func,
@@ -126,6 +137,62 @@ macro_rules! isle_common_legalizer_methods {
                 *cc,
             )
         }
+
+        fn pointer_type(&mut self) -> ir::Type {
+            (self.backend as &dyn crate::isa::TargetIsa).pointer_type()
+        }
+
+        fn const_vector_scale(&mut self, ty: ir::Type) -> u64 {
+            assert!(ty.bytes() <= 16);
+
+            // Use a minimum of 128-bits for the base type.
+            let base_bytes = std::cmp::max(ty.bytes(), 16);
+            (self.backend.dynamic_vector_bytes(ty) / base_bytes).into()
+        }
+
+        fn update_inst_facts_with_gv(&mut self, gv: GlobalValue, val: Value) {
+            if let Some(fact) = &self.pos.func.global_value_facts[gv] {
+                if self.pos.func.dfg.facts[val].is_none() {
+                    let fact = fact.clone();
+                    self.pos.func.dfg.facts[val] = Some(fact);
+                }
+            }
+        }
+
+        fn replace_vmctx_addr(&mut self, global_value: ir::GlobalValue) -> CursorPosition {
+            // Get the value representing the `vmctx` argument.
+            let vmctx = self
+                .pos
+                .func
+                .special_param(ir::ArgumentPurpose::VMContext)
+                .expect("Missing vmctx parameter");
+            let inst = self.replace.unwrap();
+
+            // Replace the `global_value` instruction's value with an alias to the vmctx arg.
+            let result = self.pos.func.dfg.first_result(inst);
+            self.pos.func.dfg.clear_results(inst);
+            self.pos.func.dfg.change_to_alias(result, vmctx);
+            self.pos.func.layout.remove_inst(inst);
+
+            // If there was a fact on the GV, then copy it to the vmctx arg
+            // blockparam def.
+            if let Some(fact) = &self.pos.func.global_value_facts[global_value] {
+                if self.pos.func.dfg.facts[vmctx].is_none() {
+                    let fact = fact.clone();
+                    self.pos.func.dfg.facts[vmctx] = Some(fact);
+                }
+            }
+
+            CursorPosition::At(inst)
+        }
+
+        fn cursor_position_at(&mut self, i: Inst) -> CursorPosition {
+            CursorPosition::At(i)
+        }
+
+        fn prev_position(&mut self) -> CursorPosition {
+            self.prev_position
+        }
     };
 }
 
@@ -137,7 +204,7 @@ pub fn expand_cond_trap(
     opcode: ir::Opcode,
     arg: ir::Value,
     code: ir::TrapCode,
-) -> Inst {
+) -> CursorPosition {
     trace!(
         "expanding conditional trap: {:?}: {}",
         inst,
@@ -191,14 +258,16 @@ pub fn expand_cond_trap(
     pos.use_srcloc(inst);
     pos.insert_block(new_block_trap);
 
-    let inst = match opcode {
-        ir::Opcode::Trapz | ir::Opcode::Trapnz => pos.ins().trap(code),
+    match opcode {
+        ir::Opcode::Trapz | ir::Opcode::Trapnz => {
+            pos.ins().trap(code);
+        }
         ir::Opcode::ResumableTrapnz => {
             pos.ins().resumable_trap(code);
-            pos.ins().jump(new_block_resume, &[])
+            pos.ins().jump(new_block_resume, &[]);
         }
         _ => unreachable!(),
-    };
+    }
 
     // Insert the new label and resume the execution when the trap fails.
     pos.insert_block(new_block_resume);
@@ -208,5 +277,5 @@ pub fn expand_cond_trap(
     cfg.recompute_block(pos.func, new_block_resume);
     cfg.recompute_block(pos.func, new_block_trap);
 
-    inst
+    CursorPosition::Before(new_block_resume)
 }
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index ec72df479277..2432e9d5435a 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -23,6 +23,8 @@
 ;; ========= Type Definitions =================================================
 
 (type Inst (primitive Inst))
+(type CursorPosition (primitive CursorPosition))
+(type ExternalName extern (enum))
 
 ;; ========= Helper extractors/constructors ===================================
 
@@ -69,12 +71,26 @@
 (decl result_type (Type) Inst)
 (extern extractor result_type result_type)
 
+(decl cursor_position_at (Inst) CursorPosition)
+(extern constructor cursor_position_at cursor_position_at)
+(convert Inst CursorPosition cursor_position_at)
+
+;; Returns the position of the cursor just before the current instruction was
+;; being legalized. This can be used to re-legalize the sequence of instructions
+;; used to replace another instruction.
+(decl prev_position () CursorPosition)
+(extern constructor prev_position prev_position)
+
+(decl pointer_type () Type)
+(extern constructor pointer_type pointer_type)
+
 ;; Helper to generate an `iconst` value which has type `I64` and the provided
 ;; value.
 (decl ins_i64 (i64) Value)
 (rule (ins_i64 val) (ins_iconst $I64 (imm64 (i64_as_u64 val))))
 
-;; Optionally "legalize" an instruction to a new instruction.
+;; Optionally "legalize" an instruction by replacing the current instruction or
+;; inserting new instructions/blocks.
 ;;
 ;; This is used to transform CLIF into something which can be handled by each
 ;; individual backend. There is both common functionality of legalization for
@@ -83,7 +99,13 @@
 ;; Note that this function is defined here and while some legalizations are
 ;; performed in this file there are also backend-specific legalizations
 ;; throughout each backend as well.
-(decl partial legalize (Inst) Inst)
+;;
+;; The return value of this function is where to resume legalization. By default
+;; this is the current instruction which means that legalization will continue
+;; after the current instruction. If the inserted code itself needs to be
+;; re-legalized then the `(prev_position)` constructor can be used to rerun
+;; over the sequence of replaced instructions.
+(decl partial legalize (Inst) CursorPosition)
 
 ;; ========= `{u,s}extend` ====================================================
 
@@ -130,15 +152,15 @@
 ;; expand these instructions
 
 (rule (legalize (trapz val code)) (expand_trapz val code))
-(decl expand_trapz (Value TrapCode) Inst)
+(decl expand_trapz (Value TrapCode) CursorPosition)
 (extern constructor expand_trapz expand_trapz)
 
 (rule (legalize (trapnz val code)) (expand_trapnz val code))
-(decl expand_trapnz (Value TrapCode) Inst)
+(decl expand_trapnz (Value TrapCode) CursorPosition)
 (extern constructor expand_trapnz expand_trapnz)
 
 (rule (legalize (resumable_trapnz val code)) (expand_resumable_trapnz val code))
-(decl expand_resumable_trapnz (Value TrapCode) Inst)
+(decl expand_resumable_trapnz (Value TrapCode) CursorPosition)
 (extern constructor expand_resumable_trapnz expand_resumable_trapnz)
 
 ;; ========= `*_imm` instrs ===================================================
@@ -227,3 +249,88 @@
 
 (rule (legalize (has_type ty (bxor_not a b)))
   (replace_bxor ty a (ins_bnot ty b)))
+
+;; ========= `global_value` ===================================================
+
+(rule (legalize (global_value gv @ (gv_vmcontext)))
+  (replace_vmctx_addr gv))
+(decl replace_vmctx_addr (GlobalValue) CursorPosition)
+(extern constructor replace_vmctx_addr replace_vmctx_addr)
+
+(rule (legalize (global_value (gv_load base offset ty flags)))
+  (let ((base Value (ins_global_value_copy_facts (pointer_type) base))
+        (_ Inst (replace_load ty flags base offset)))
+    ;; re-legalize the previous instrs to legalize the new `global_value` node
+    (prev_position)))
+
+(rule (legalize (global_value (gv_iadd_imm base offset ty)))
+  (let ((base Value (ins_global_value_copy_facts ty base))
+        (_ Inst (replace_iadd_imm ty base offset)))
+    ;; re-legalize the previous instrs to legalize the new `global_value` node
+    (prev_position)))
+
+(rule (legalize (global_value (gv_const_vector_scale ty)))
+  (replace_iconst (pointer_type) (imm64 (const_vector_scale ty))))
+(decl const_vector_scale (Type) u64)
+(extern constructor const_vector_scale const_vector_scale)
+
+(rule (legalize (global_value gv @ (gv_symbol _ _)))
+  (replace_symbol_value (pointer_type) gv))
+
+(rule (legalize (global_value gv @ (gv_tls_symbol _ _)))
+  (replace_tls_value (pointer_type) gv))
+
+(type GlobalValueData extern (enum
+  (VMContext)
+  (Load
+    (base GlobalValue)
+    (offset Offset32)
+    (global_type Type)
+    (flags MemFlags))
+  (IAddImm
+    (base GlobalValue)
+    (offset Imm64)
+    (global_type Type))
+  (Symbol
+    (name ExternalName)
+    (offset Imm64)
+    (colocated bool)
+    (tls bool))
+  (DynScaleTargetConst
+    (vector_type Type))
+))
+
+;; Convert from `GlobalValue` to `GlobalValueData`
+(decl gv_data (GlobalValueData) GlobalValue)
+(extern extractor infallible gv_data gv_data)
+
+;; Helper extractors for each variant of `GlobalValue`
+(decl gv_vmcontext () GlobalValue)
+(extractor (gv_vmcontext)
+  (gv_data (GlobalValueData.VMContext)))
+(decl gv_load (GlobalValue Offset32 Type MemFlags) GlobalValue)
+(extractor (gv_load base offset ty flags)
+  (gv_data (GlobalValueData.Load base offset ty flags)))
+(decl gv_iadd_imm (GlobalValue Offset32 Type) GlobalValue)
+(extractor (gv_iadd_imm base offset ty)
+  (gv_data (GlobalValueData.IAddImm base offset ty)))
+(decl gv_symbol (Imm64 bool) GlobalValue)
+(extractor (gv_symbol offset colocated)
+  (gv_data (GlobalValueData.Symbol _ offset colocated $false)))
+(decl gv_tls_symbol (Imm64 bool) GlobalValue)
+(extractor (gv_tls_symbol offset colocated)
+  (gv_data (GlobalValueData.Symbol _ offset colocated $true)))
+(decl gv_const_vector_scale (Type) GlobalValue)
+(extractor (gv_const_vector_scale ty)
+  (gv_data (GlobalValueData.DynScaleTargetConst ty)))
+
+;; This is the dual of the `ins_global_value` constructor except that this one
+;; additionally copies PCC facts from the `GlobalValue` onto the returned
+;; `Inst`.
+(decl ins_global_value_copy_facts (Type GlobalValue) Inst)
+(rule (ins_global_value_copy_facts ty gv)
+  (let ((ret Inst (ins_global_value ty gv))
+        (_ Unit (update_inst_facts_with_gv gv ret)))
+    ret))
+(decl update_inst_facts_with_gv (GlobalValue Value) Unit)
+(extern constructor update_inst_facts_with_gv update_inst_facts_with_gv)

From 76cd417a5ab0ac97020a50e27783505c8d48abec Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 12:10:10 -0700
Subject: [PATCH 11/21] Move table_addr legalization into ISLE

This additionally enables deleting all existing Rust-base simple
legalization now that nothing depends on it any more.
---
 .../codegen/meta/src/shared/immediates.rs     |   1 +
 cranelift/codegen/src/context.rs              |   2 -
 .../codegen/src/legalizer/globalvalue.rs      | 121 -----------
 cranelift/codegen/src/legalizer/isle.rs       |  29 ++-
 .../src/legalizer/legalize_shared.isle        |  52 +++++
 cranelift/codegen/src/legalizer/mod.rs        | 194 ------------------
 cranelift/codegen/src/legalizer/table.rs      | 114 ----------
 .../filetests/filetests/isa/x64/table.clif    |   5 +-
 8 files changed, 83 insertions(+), 435 deletions(-)
 delete mode 100644 cranelift/codegen/src/legalizer/globalvalue.rs
 delete mode 100644 cranelift/codegen/src/legalizer/table.rs

diff --git a/cranelift/codegen/meta/src/shared/immediates.rs b/cranelift/codegen/meta/src/shared/immediates.rs
index 5584b5564bb9..1ca7ea96bf5f 100644
--- a/cranelift/codegen/meta/src/shared/immediates.rs
+++ b/cranelift/codegen/meta/src/shared/immediates.rs
@@ -182,6 +182,7 @@ impl Immediates {
                 trapcode_values.insert("int_ovf", "IntegerOverflow");
                 trapcode_values.insert("int_divz", "IntegerDivisionByZero");
                 trapcode_values.insert("bad_toint", "BadConversionToInteger");
+                trapcode_values.insert("table_oob", "TableOutOfBounds");
                 new_enum(
                     "code",
                     "ir::TrapCode",
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index 71790988b44e..fff6eec31be1 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -16,7 +16,6 @@ use crate::egraph::EgraphPass;
 use crate::flowgraph::ControlFlowGraph;
 use crate::ir::Function;
 use crate::isa::TargetIsa;
-use crate::legalizer::simple_legalize;
 use crate::loop_analysis::LoopAnalysis;
 use crate::machinst::{CompiledCode, CompiledCodeStencil};
 use crate::nan_canonicalization::do_nan_canonicalization;
@@ -291,7 +290,6 @@ impl Context {
 
         // Run some specific legalizations only.
         isa.legalize_function(&mut self.func, &mut self.cfg);
-        simple_legalize(&mut self.func, &mut self.cfg, isa);
         self.verify_if(isa)
     }
 
diff --git a/cranelift/codegen/src/legalizer/globalvalue.rs b/cranelift/codegen/src/legalizer/globalvalue.rs
deleted file mode 100644
index e2660bb8789d..000000000000
--- a/cranelift/codegen/src/legalizer/globalvalue.rs
+++ /dev/null
@@ -1,121 +0,0 @@
-//! Legalization of global values.
-//!
-//! This module exports the `expand_global_value` function which transforms a `global_value`
-//! instruction into code that depends on the kind of global value referenced.
-
-use crate::cursor::{Cursor, FuncCursor};
-use crate::ir::{self, pcc::Fact, InstBuilder};
-use crate::isa::TargetIsa;
-
-/// Expand a `global_value` instruction according to the definition of the global value.
-pub fn expand_global_value(
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    isa: &dyn TargetIsa,
-    global_value: ir::GlobalValue,
-) {
-    crate::trace!(
-        "expanding global value: {:?}: {}",
-        inst,
-        func.dfg.display_inst(inst)
-    );
-
-    match func.global_values[global_value] {
-        ir::GlobalValueData::VMContext => vmctx_addr(global_value, inst, func),
-        ir::GlobalValueData::IAddImm {
-            base,
-            offset,
-            global_type,
-        } => iadd_imm_addr(inst, func, base, offset.into(), global_type),
-        ir::GlobalValueData::Load {
-            base,
-            offset,
-            global_type,
-            flags,
-        } => load_addr(inst, func, base, offset, global_type, flags, isa),
-        ir::GlobalValueData::Symbol { .. } => unreachable!(),
-        ir::GlobalValueData::DynScaleTargetConst { .. } => unreachable!(),
-    }
-}
-
-/// Expand a `global_value` instruction for a vmctx global.
-fn vmctx_addr(global_value: ir::GlobalValue, inst: ir::Inst, func: &mut ir::Function) {
-    // Get the value representing the `vmctx` argument.
-    let vmctx = func
-        .special_param(ir::ArgumentPurpose::VMContext)
-        .expect("Missing vmctx parameter");
-
-    // Replace the `global_value` instruction's value with an alias to the vmctx arg.
-    let result = func.dfg.first_result(inst);
-    func.dfg.clear_results(inst);
-    func.dfg.change_to_alias(result, vmctx);
-    func.layout.remove_inst(inst);
-
-    // If there was a fact on the GV, then copy it to the vmctx arg
-    // blockparam def.
-    if let Some(fact) = &func.global_value_facts[global_value] {
-        if func.dfg.facts[vmctx].is_none() {
-            let fact = fact.clone();
-            func.dfg.facts[vmctx] = Some(fact);
-        }
-    }
-}
-
-/// Expand a `global_value` instruction for an iadd_imm global.
-fn iadd_imm_addr(
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    base: ir::GlobalValue,
-    offset: i64,
-    global_type: ir::Type,
-) {
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-
-    // Get the value for the lhs.
-    let lhs = pos.ins().global_value(global_type, base);
-    if let Some(fact) = &pos.func.global_value_facts[base] {
-        pos.func.dfg.facts[lhs] = Some(fact.clone());
-    }
-
-    // Generate the constant and attach a fact to the constant if
-    // there is a fact on the base.
-    let constant = pos.ins().iconst(global_type, offset);
-    if pos.func.global_value_facts[base].is_some() {
-        let bits = u16::try_from(global_type.bits()).unwrap();
-        let unsigned_offset = offset as u64; // Safety: reinterpret i64 bits as u64.
-        pos.func.dfg.facts[constant] = Some(Fact::constant(bits, unsigned_offset));
-    }
-
-    // Simply replace the `global_value` instruction with an `iadd_imm`, reusing the result value.
-    pos.func.dfg.replace(inst).iadd(lhs, constant);
-}
-
-/// Expand a `global_value` instruction for a load global.
-fn load_addr(
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    base: ir::GlobalValue,
-    offset: ir::immediates::Offset32,
-    global_type: ir::Type,
-    flags: ir::MemFlags,
-    isa: &dyn TargetIsa,
-) {
-    // We need to load a pointer from the `base` global value, so insert a new `global_value`
-    // instruction. This depends on the iterative legalization loop. Note that the IR verifier
-    // detects any cycles in the `load` globals.
-    let ptr_ty = isa.pointer_type();
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-    pos.use_srcloc(inst);
-
-    // Get the value for the base.
-    let base_addr = pos.ins().global_value(ptr_ty, base);
-    if let Some(fact) = &pos.func.global_value_facts[base] {
-        pos.func.dfg.facts[base_addr] = Some(fact.clone());
-    }
-
-    // Perform the load.
-    pos.func
-        .dfg
-        .replace(inst)
-        .load(global_type, flags, base_addr, offset);
-}
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index 57ebabc07277..c703021395f0 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -28,7 +28,7 @@ pub fn run<T>(
             cx.replace = Some(inst);
             match constructor_legalize(&mut cx, inst) {
                 Some(pos) => cx.pos.set_position(pos),
-                None => {}
+                None => cx.prev_position = cx.pos.position(),
             }
         }
     }
@@ -183,7 +183,7 @@ macro_rules! isle_common_legalizer_methods {
                 }
             }
 
-            CursorPosition::At(inst)
+            self.prev_position
         }
 
         fn cursor_position_at(&mut self, i: Inst) -> CursorPosition {
@@ -193,6 +193,31 @@ macro_rules! isle_common_legalizer_methods {
         fn prev_position(&mut self) -> CursorPosition {
             self.prev_position
         }
+
+        fn table_bound_gv(&mut self, table: Table) -> GlobalValue {
+            self.pos.func.tables[table].bound_gv
+        }
+
+        fn table_base_gv(&mut self, table: Table) -> GlobalValue {
+            self.pos.func.tables[table].base_gv
+        }
+
+        fn table_element_size(&mut self, table: Table) -> u64 {
+            self.pos.func.tables[table].element_size.into()
+        }
+
+        fn enable_table_access_spectre_mitigation(&mut self) -> bool {
+            self.backend
+                .flags()
+                .enable_table_access_spectre_mitigation()
+        }
+
+        fn replace_with_aliases(&mut self, val: Value) -> Inst {
+            let new_inst = self.pos.func.dfg.value_def(val).inst().unwrap();
+            let inst = self.replace.unwrap();
+            self.pos.func.dfg.replace_with_aliases(inst, new_inst);
+            crate::cursor::Cursor::remove_inst(&mut self.pos)
+        }
     };
 }
 
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index 2432e9d5435a..dfa0f5d0b88c 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -334,3 +334,55 @@
     ret))
 (decl update_inst_facts_with_gv (GlobalValue Value) Unit)
 (extern constructor update_inst_facts_with_gv update_inst_facts_with_gv)
+
+;; ========= `table_addr` =====================================================
+
+(rule (legalize (has_type addr_ty (table_addr table idx @ (value_type idx_ty) offset)))
+  (let (
+    ;; Start with the bounds check. Trap if `index + 1 > bound`.
+    ;; `index > bound - 1` is the same as `index >= bound`.
+    (bound Inst (ins_global_value_copy_facts idx_ty (table_bound_gv table)))
+    (cmp Inst (ins_icmp $I8 (IntCC.UnsignedGreaterThanOrEqual) idx bound))
+    (_ Inst (ins_trapnz $I8 cmp (TrapCode.TableOutOfBounds)))
+
+    ;; Calculate the offset, in bytes, by multiplying `idx` by the byte size
+    ;; of the table element.
+    (idx Value (extend_index_to_addr idx addr_ty))
+    (idx_offset Value (ins_imul_imm addr_ty idx (imm64 (table_element_size table))))
+
+    ;; Calculate the final address by adding the table base address to the
+    ;; element offset to the static offset in the `table_addr` instruction.
+    (base Value (ins_global_value_copy_facts addr_ty (table_base_gv table)))
+    (offset Value (ins_iconst addr_ty (imm64 (i64_as_u64 (offset32_to_i32 offset)))))
+    (elem_addr Value (ins_iadd addr_ty (ins_iadd addr_ty base idx_offset) offset))
+
+    ;; Generate the final `result` and replace the current instruction with that.
+    (result Value (select_table_addr cmp base elem_addr))
+    (_ Inst (replace_with_aliases result))
+  )
+  ;; re-legalize the above instructions
+  (prev_position)))
+
+(decl select_table_addr (Value Value Value) Value)
+(rule (select_table_addr cmp base @ (value_type ty) element_addr)
+  (if-let $true (enable_table_access_spectre_mitigation))
+  (ins_select_spectre_guard ty cmp base element_addr))
+(rule (select_table_addr cmp base @ (value_type ty) element_addr)
+  (if-let $false (enable_table_access_spectre_mitigation))
+  element_addr)
+
+(decl extend_index_to_addr (Value Type) Value)
+(rule 0 (extend_index_to_addr val ty) (ins_uextend ty val))
+(rule 1 (extend_index_to_addr val @ (value_type ty) ty) val)
+
+(decl table_bound_gv (Table) GlobalValue)
+(extern constructor table_bound_gv table_bound_gv)
+(decl table_base_gv (Table) GlobalValue)
+(extern constructor table_base_gv table_base_gv)
+(decl table_element_size (Table) u64)
+(extern constructor table_element_size table_element_size)
+(decl pure enable_table_access_spectre_mitigation () bool)
+(extern constructor enable_table_access_spectre_mitigation enable_table_access_spectre_mitigation)
+
+(decl replace_with_aliases (Value) Inst)
+(extern constructor replace_with_aliases replace_with_aliases)
diff --git a/cranelift/codegen/src/legalizer/mod.rs b/cranelift/codegen/src/legalizer/mod.rs
index b1a20c890c8e..ee56462362b9 100644
--- a/cranelift/codegen/src/legalizer/mod.rs
+++ b/cranelift/codegen/src/legalizer/mod.rs
@@ -1,195 +1 @@
-//! Legalize instructions.
-//!
-//! A legal instruction is one that can be mapped directly to a machine code instruction for the
-//! target ISA. The `legalize_function()` function takes as input any function and transforms it
-//! into an equivalent function using only legal instructions.
-//!
-//! The characteristics of legal instructions depend on the target ISA, so any given instruction
-//! can be legal for one ISA and illegal for another.
-//!
-//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
-//! which provides a legal encoding recipe for every instruction.
-//!
-//! The legalizer does not deal with register allocation constraints. These constraints are derived
-//! from the encoding recipes, and solved later by the register allocator.
-
-use crate::cursor::{Cursor, FuncCursor};
-use crate::flowgraph::ControlFlowGraph;
-use crate::ir::immediates::Imm64;
-use crate::ir::types::{self, I128, I64};
-use crate::ir::{self, InstBuilder, InstructionData, Value};
-use crate::isa::TargetIsa;
-use crate::trace;
-
-mod globalvalue;
 pub mod isle;
-mod table;
-
-use self::globalvalue::expand_global_value;
-use self::table::expand_table_addr;
-
-fn imm_const(pos: &mut FuncCursor, arg: Value, imm: Imm64, is_signed: bool) -> Value {
-    let ty = pos.func.dfg.value_type(arg);
-    match (ty, is_signed) {
-        (I128, true) => {
-            let imm = pos.ins().iconst(I64, imm);
-            pos.ins().sextend(I128, imm)
-        }
-        (I128, false) => {
-            let imm = pos.ins().iconst(I64, imm);
-            pos.ins().uextend(I128, imm)
-        }
-        _ => {
-            let bits = imm.bits();
-            let unsigned = match ty.lane_type() {
-                types::I8 => bits as u8 as i64,
-                types::I16 => bits as u16 as i64,
-                types::I32 => bits as u32 as i64,
-                types::I64 => bits,
-                _ => unreachable!(),
-            };
-            pos.ins().iconst(ty.lane_type(), unsigned)
-        }
-    }
-}
-
-/// Perform a simple legalization by expansion of the function, without
-/// platform-specific transforms.
-pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
-    trace!("Pre-legalization function:\n{}", func.display());
-
-    let mut pos = FuncCursor::new(func);
-    let func_begin = pos.position();
-    pos.set_position(func_begin);
-    while let Some(_block) = pos.next_block() {
-        let mut prev_pos = pos.position();
-        while let Some(inst) = pos.next_inst() {
-            match pos.func.dfg.insts[inst] {
-                // control flow
-                InstructionData::CondTrap {
-                    opcode: ir::Opcode::Trapnz,
-                    arg,
-                    code,
-                } => {
-                    expand_cond_trap(inst, &mut pos.func, cfg, ir::Opcode::Trapnz, arg, code);
-                }
-
-                // memory and constants
-                InstructionData::UnaryGlobalValue {
-                    opcode: ir::Opcode::GlobalValue,
-                    global_value,
-                } => expand_global_value(inst, &mut pos.func, isa, global_value),
-                InstructionData::TableAddr {
-                    opcode: ir::Opcode::TableAddr,
-                    table,
-                    arg,
-                    offset,
-                } => expand_table_addr(isa, inst, &mut pos.func, table, arg, offset),
-
-                InstructionData::BinaryImm64 { opcode, arg, imm } => {
-                    match opcode {
-                        ir::Opcode::IaddImm => {
-                            let imm = imm_const(&mut pos, arg, imm, true);
-                            let replace = pos.func.dfg.replace(inst);
-                            replace.iadd(arg, imm);
-                        }
-                        _ => prev_pos = pos.position(),
-                    };
-                }
-
-                _ => {
-                    prev_pos = pos.position();
-                    continue;
-                }
-            }
-
-            // Legalization implementations require fixpoint loop here.
-            // TODO: fix this.
-            pos.set_position(prev_pos);
-        }
-    }
-
-    trace!("Post-legalization function:\n{}", func.display());
-}
-
-/// Custom expansion for conditional trap instructions.
-fn expand_cond_trap(
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    cfg: &mut ControlFlowGraph,
-    opcode: ir::Opcode,
-    arg: ir::Value,
-    code: ir::TrapCode,
-) {
-    trace!(
-        "expanding conditional trap: {:?}: {}",
-        inst,
-        func.dfg.display_inst(inst)
-    );
-
-    // Parse the instruction.
-    let trapz = match opcode {
-        ir::Opcode::Trapz => true,
-        ir::Opcode::Trapnz | ir::Opcode::ResumableTrapnz => false,
-        _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst)),
-    };
-
-    // Split the block after `inst`:
-    //
-    //     trapnz arg
-    //     ..
-    //
-    // Becomes:
-    //
-    //     brif arg, new_block_trap, new_block_resume
-    //
-    //   new_block_trap:
-    //     trap
-    //
-    //   new_block_resume:
-    //     ..
-    let old_block = func
-        .layout
-        .inst_block(inst)
-        .expect("Instruction not in layout.");
-    let new_block_trap = func.dfg.make_block();
-    let new_block_resume = func.dfg.make_block();
-
-    // Trapping is a rare event, mark the trapping block as cold.
-    func.layout.set_cold(new_block_trap);
-
-    // Replace trap instruction by the inverted condition.
-    if trapz {
-        func.dfg
-            .replace(inst)
-            .brif(arg, new_block_resume, &[], new_block_trap, &[]);
-    } else {
-        func.dfg
-            .replace(inst)
-            .brif(arg, new_block_trap, &[], new_block_resume, &[]);
-    }
-
-    // Insert the new label and the unconditional trap terminator.
-    let mut pos = FuncCursor::new(func).after_inst(inst);
-    pos.use_srcloc(inst);
-    pos.insert_block(new_block_trap);
-
-    match opcode {
-        ir::Opcode::Trapz | ir::Opcode::Trapnz => {
-            pos.ins().trap(code);
-        }
-        ir::Opcode::ResumableTrapnz => {
-            pos.ins().resumable_trap(code);
-            pos.ins().jump(new_block_resume, &[]);
-        }
-        _ => unreachable!(),
-    }
-
-    // Insert the new label and resume the execution when the trap fails.
-    pos.insert_block(new_block_resume);
-
-    // Finally update the CFG.
-    cfg.recompute_block(pos.func, old_block);
-    cfg.recompute_block(pos.func, new_block_resume);
-    cfg.recompute_block(pos.func, new_block_trap);
-}
diff --git a/cranelift/codegen/src/legalizer/table.rs b/cranelift/codegen/src/legalizer/table.rs
deleted file mode 100644
index db72508a2680..000000000000
--- a/cranelift/codegen/src/legalizer/table.rs
+++ /dev/null
@@ -1,114 +0,0 @@
-//! Legalization of tables.
-//!
-//! This module exports the `expand_table_addr` function which transforms a `table_addr`
-//! instruction into code that depends on the kind of table referenced.
-
-use crate::cursor::{Cursor, FuncCursor};
-use crate::ir::condcodes::IntCC;
-use crate::ir::immediates::Offset32;
-use crate::ir::{self, InstBuilder};
-use crate::isa::TargetIsa;
-
-/// Expand a `table_addr` instruction according to the definition of the table.
-pub fn expand_table_addr(
-    isa: &dyn TargetIsa,
-    inst: ir::Inst,
-    func: &mut ir::Function,
-    table: ir::Table,
-    index: ir::Value,
-    element_offset: Offset32,
-) {
-    let bound_gv = func.tables[table].bound_gv;
-    let index_ty = func.dfg.value_type(index);
-    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-    pos.use_srcloc(inst);
-
-    // Start with the bounds check. Trap if `index + 1 > bound`.
-    let bound = pos.ins().global_value(index_ty, bound_gv);
-
-    // `index > bound - 1` is the same as `index >= bound`.
-    let oob = pos
-        .ins()
-        .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
-    pos.ins().trapnz(oob, ir::TrapCode::TableOutOfBounds);
-
-    // If Spectre mitigations are enabled, we will use a comparison to
-    // short-circuit the computed table element address to the start
-    // of the table on the misspeculation path when out-of-bounds.
-    let spectre_oob_cmp = if isa.flags().enable_table_access_spectre_mitigation() {
-        Some((index, bound))
-    } else {
-        None
-    };
-
-    compute_addr(
-        inst,
-        table,
-        addr_ty,
-        index,
-        index_ty,
-        element_offset,
-        pos.func,
-        spectre_oob_cmp,
-    );
-}
-
-/// Emit code for the base address computation of a `table_addr` instruction.
-fn compute_addr(
-    inst: ir::Inst,
-    table: ir::Table,
-    addr_ty: ir::Type,
-    mut index: ir::Value,
-    index_ty: ir::Type,
-    element_offset: Offset32,
-    func: &mut ir::Function,
-    spectre_oob_cmp: Option<(ir::Value, ir::Value)>,
-) {
-    let mut pos = FuncCursor::new(func).at_inst(inst);
-    pos.use_srcloc(inst);
-
-    // Convert `index` to `addr_ty`.
-    if index_ty != addr_ty {
-        index = pos.ins().uextend(addr_ty, index);
-    }
-
-    // Add the table base address base
-    let base_gv = pos.func.tables[table].base_gv;
-    let base = pos.ins().global_value(addr_ty, base_gv);
-
-    let element_size = pos.func.tables[table].element_size;
-    let mut offset;
-    let element_size: u64 = element_size.into();
-    if element_size == 1 {
-        offset = index;
-    } else if element_size.is_power_of_two() {
-        offset = pos
-            .ins()
-            .ishl_imm(index, i64::from(element_size.trailing_zeros()));
-    } else {
-        offset = pos.ins().imul_imm(index, element_size as i64);
-    }
-
-    let element_addr = if element_offset == Offset32::new(0) {
-        pos.ins().iadd(base, offset)
-    } else {
-        let imm: i64 = element_offset.into();
-        offset = pos.ins().iadd(base, offset);
-        pos.ins().iadd_imm(offset, imm)
-    };
-
-    let element_addr = if let Some((index, bound)) = spectre_oob_cmp {
-        let cond = pos
-            .ins()
-            .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
-        // If out-of-bounds, choose the table base on the misspeculation path.
-        pos.ins().select_spectre_guard(cond, base, element_addr)
-    } else {
-        element_addr
-    };
-    let new_inst = pos.func.dfg.value_def(element_addr).inst().unwrap();
-
-    pos.func.dfg.replace_with_aliases(inst, new_inst);
-    pos.remove_inst();
-}
diff --git a/cranelift/filetests/filetests/isa/x64/table.clif b/cranelift/filetests/filetests/isa/x64/table.clif
index c3c217fc9a06..1b69a3e85999 100644
--- a/cranelift/filetests/filetests/isa/x64/table.clif
+++ b/cranelift/filetests/filetests/isa/x64/table.clif
@@ -1,6 +1,7 @@
 test compile precise-output
 set enable_safepoints=true
 set enable_table_access_spectre_mitigation=true
+set opt_level=speed
 
 target x86_64
 
@@ -24,8 +25,8 @@ block0(v0: i32, v1: r64, v2: i64):
 ;   cmpl    %r11d, %edi
 ;   jnb     label2; j label1
 ; block1:
-;   movl    %edi, %ecx
 ;   movq    0(%rdx), %rax
+;   movl    %edi, %ecx
 ;   lea     0(%rax,%rcx,1), %rcx
 ;   cmpl    %r11d, %edi
 ;   cmovnbq %rax, %rcx, %rcx
@@ -45,8 +46,8 @@ block0(v0: i32, v1: r64, v2: i64):
 ;   cmpl %r11d, %edi
 ;   jae 0x28
 ; block2: ; offset 0x11
-;   movl %edi, %ecx
 ;   movq (%rdx), %rax
+;   movl %edi, %ecx
 ;   addq %rax, %rcx
 ;   cmpl %r11d, %edi
 ;   cmovaeq %rax, %rcx

From 54cba636b05f2d5ba900f9725473ba03b3574724 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 12:25:13 -0700
Subject: [PATCH 12/21] Add a simple test for two-legalizations

Ensure that instructions aren't skipped
---
 .../legalizer/back-to-back-legalize.clif       | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif

diff --git a/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif
new file mode 100644
index 000000000000..4404386af90a
--- /dev/null
+++ b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif
@@ -0,0 +1,18 @@
+test legalizer
+target x86_64
+
+function %foo() -> i32 {
+block0:
+    v1 = iconst.i32 1
+    v2 = iadd_imm v1, 1
+    v3 = iadd_imm v2, 1
+    return v3
+}
+
+; check: v1 = iconst.i32 1
+; check: v4 = iconst.i32 1
+; check: v2 = iadd v1, v4
+; check: v5 = iconst.i32 1
+; check: v3 = iadd v2, v5
+; check: return v3
+

From e0e2191d063234c1b9c92b651b53496d05126337 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 12:31:35 -0700
Subject: [PATCH 13/21] Add a test for nan canonicalization legalizations

---
 .../legalizer/canonicalize-nans.clif          | 189 ++++++++++++++++++
 1 file changed, 189 insertions(+)
 create mode 100644 cranelift/filetests/filetests/legalizer/canonicalize-nans.clif

diff --git a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
new file mode 100644
index 000000000000..2750356a6010
--- /dev/null
+++ b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
@@ -0,0 +1,189 @@
+test optimize precise-output
+set enable_nan_canonicalization=true
+target x86_64
+
+function %ceil(f32) -> f32 {
+block0(v0: f32):
+  v1 = ceil v0
+  return v1
+}
+
+; function %ceil(f32) -> f32 fast {
+; block0(v0: f32):
+;     v2 = ceil v0
+;     v3 = fcmp ne v2, v2
+;     v4 = f32const +NaN
+;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     return v1
+; }
+
+
+function %floor(f64) -> f64 {
+block0(v0: f64):
+  v1 = floor v0
+  return v1
+}
+
+; function %floor(f64) -> f64 fast {
+; block0(v0: f64):
+;     v2 = floor v0
+;     v3 = fcmp ne v2, v2
+;     v4 = f64const +NaN
+;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     return v1
+; }
+
+function %nearest(f32x4) -> f32x4 {
+block0(v0: f32x4):
+  v1 = nearest v0
+  return v1
+}
+
+; function %nearest(f32x4) -> f32x4 fast {
+; block0(v0: f32x4):
+;     v2 = nearest v0
+;     v3 = fcmp ne v2, v2
+;     v4 = f32const +NaN
+;     v5 = splat.f32x4 v4  ; v4 = +NaN
+;     v6 = bitcast.f32x4 v3
+;     v1 = bitselect v6, v5, v2
+;     return v1
+; }
+
+function %sqrt(f64x2) -> f64x2 {
+block0(v0: f64x2):
+  v1 = sqrt v0
+  return v1
+}
+
+; function %sqrt(f64x2) -> f64x2 fast {
+; block0(v0: f64x2):
+;     v2 = sqrt v0
+;     v3 = fcmp ne v2, v2
+;     v4 = f64const +NaN
+;     v5 = splat.f64x2 v4  ; v4 = +NaN
+;     v6 = bitcast.f64x2 v3
+;     v1 = bitselect v6, v5, v2
+;     return v1
+; }
+
+function %trunc(f32) -> f32 {
+block0(v0: f32):
+  v1 = trunc v0
+  return v1
+}
+
+; function %trunc(f32) -> f32 fast {
+; block0(v0: f32):
+;     v2 = trunc v0
+;     v3 = fcmp ne v2, v2
+;     v4 = f32const +NaN
+;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     return v1
+; }
+
+function %fadd(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fadd v0, v1
+  return v2
+}
+
+; function %fadd(f32, f32) -> f32 fast {
+; block0(v0: f32, v1: f32):
+;     v3 = fadd v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f32const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fsub(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+  v2 = fsub v0, v1
+  return v2
+}
+
+; function %fsub(f32, f32) -> f32 fast {
+; block0(v0: f32, v1: f32):
+;     v3 = fsub v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f32const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fdiv(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fdiv v0, v1
+  return v2
+}
+
+; function %fdiv(f64, f64) -> f64 fast {
+; block0(v0: f64, v1: f64):
+;     v3 = fdiv v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f64const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fmul(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmul v0, v1
+  return v2
+}
+
+; function %fmul(f64, f64) -> f64 fast {
+; block0(v0: f64, v1: f64):
+;     v3 = fmul v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f64const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fmax(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmax v0, v1
+  return v2
+}
+
+; function %fmax(f64, f64) -> f64 fast {
+; block0(v0: f64, v1: f64):
+;     v3 = fmax v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f64const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fmin(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+  v2 = fmin v0, v1
+  return v2
+}
+
+; function %fmin(f64, f64) -> f64 fast {
+; block0(v0: f64, v1: f64):
+;     v3 = fmin v0, v1
+;     v4 = fcmp ne v3, v3
+;     v5 = f64const +NaN
+;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     return v2
+; }
+
+function %fma(f64, f64, f64) -> f64 {
+block0(v0: f64, v1: f64, v2: f64):
+  v3 = fma v0, v1, v2
+  return v3
+}
+
+; function %fma(f64, f64, f64) -> f64 fast {
+; block0(v0: f64, v1: f64, v2: f64):
+;     v4 = fma v0, v1, v2
+;     v5 = fcmp ne v4, v4
+;     v6 = f64const +NaN
+;     v3 = select v5, v6, v4  ; v6 = +NaN
+;     return v3
+; }
+

From 8d09a9040f741dafa467474358ab3cbd5c8d5f0d Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 13:02:29 -0700
Subject: [PATCH 14/21] Move NaN canonicalization into ISLE

---
 cranelift/codegen/src/context.rs              |  10 --
 cranelift/codegen/src/isle_prelude.rs         |   4 +
 cranelift/codegen/src/legalizer/isle.rs       |  34 +++++-
 .../src/legalizer/legalize_shared.isle        |  75 +++++++++++++
 cranelift/codegen/src/lib.rs                  |   1 -
 cranelift/codegen/src/nan_canonicalization.rs | 106 ------------------
 cranelift/codegen/src/prelude.isle            |   4 +
 .../legalizer/canonicalize-nans.clif          |  11 +-
 cranelift/fuzzgen/src/lib.rs                  |   5 +-
 9 files changed, 120 insertions(+), 130 deletions(-)
 delete mode 100644 cranelift/codegen/src/nan_canonicalization.rs

diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index fff6eec31be1..7128984bb599 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -18,7 +18,6 @@ use crate::ir::Function;
 use crate::isa::TargetIsa;
 use crate::loop_analysis::LoopAnalysis;
 use crate::machinst::{CompiledCode, CompiledCodeStencil};
-use crate::nan_canonicalization::do_nan_canonicalization;
 use crate::remove_constant_phis::do_remove_constant_phis;
 use crate::result::{CodegenResult, CompileResult};
 use crate::settings::{FlagsOrIsa, OptLevel};
@@ -168,9 +167,6 @@ impl Context {
         );
 
         self.compute_cfg();
-        if isa.flags().enable_nan_canonicalization() {
-            self.canonicalize_nans(isa)?;
-        }
 
         self.legalize(isa)?;
 
@@ -275,12 +271,6 @@ impl Context {
         Ok(())
     }
 
-    /// Perform NaN canonicalizing rewrites on the function.
-    pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
-        do_nan_canonicalization(&mut self.func);
-        self.verify_if(isa)
-    }
-
     /// Run the legalizer for `isa` on the function.
     pub fn legalize(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
         // Legalization invalidates the domtree and loop_analysis by mutating the CFG.
diff --git a/cranelift/codegen/src/isle_prelude.rs b/cranelift/codegen/src/isle_prelude.rs
index f9bbb1153748..552f6745ff0a 100644
--- a/cranelift/codegen/src/isle_prelude.rs
+++ b/cranelift/codegen/src/isle_prelude.rs
@@ -834,6 +834,10 @@ macro_rules! isle_common_prelude_methods {
             mflags
         }
 
+        fn mem_flags_empty(&mut self) -> MemFlags {
+            MemFlags::new()
+        }
+
         #[inline]
         fn intcc_unsigned(&mut self, x: &IntCC) -> IntCC {
             x.unsigned()
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index c703021395f0..30ad0632d5fc 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -25,10 +25,17 @@ pub fn run<T>(
     while let Some(_block) = cx.pos.next_block() {
         cx.prev_position = cx.pos.position();
         while let Some(inst) = cx.pos.next_inst() {
+            trace!("legalizing {}", cx.pos.func.dfg.display_inst(inst));
             cx.replace = Some(inst);
             match constructor_legalize(&mut cx, inst) {
-                Some(pos) => cx.pos.set_position(pos),
-                None => cx.prev_position = cx.pos.position(),
+                Some(pos) => {
+                    trace!("moving to {pos:?}");
+                    cx.pos.set_position(pos);
+                }
+                None => {
+                    trace!("fallthrough");
+                    cx.prev_position = cx.pos.position();
+                }
             }
         }
     }
@@ -57,12 +64,16 @@ macro_rules! isle_common_legalizer_methods {
         }
 
         fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
-            self.pos.ins().build(data.clone(), ty).0
+            let ret = self.pos.ins().build(data.clone(), ty).0;
+            crate::trace!("ins {}", self.pos.func.dfg.display_inst(ret));
+            ret
         }
 
         fn replace(&mut self, ty: Type, data: &InstructionData) -> Inst {
             let ins = self.pos.func.dfg.replace(self.replace.unwrap());
-            ins.build(data.clone(), ty).0
+            let ret = ins.build(data.clone(), ty).0;
+            crate::trace!("replace {}", self.pos.func.dfg.display_inst(ret));
+            ret
         }
 
         fn value_type(&mut self, val: Value) -> Type {
@@ -218,6 +229,21 @@ macro_rules! isle_common_legalizer_methods {
             self.pos.func.dfg.replace_with_aliases(inst, new_inst);
             crate::cursor::Cursor::remove_inst(&mut self.pos)
         }
+
+        fn enable_nan_canonicalization(&mut self) -> bool {
+            self.backend.flags().enable_nan_canonicalization()
+        }
+
+        fn canon_nan32(&mut self) -> Value {
+            static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000;
+            self.pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN))
+        }
+
+        fn canon_nan64(&mut self) -> Value {
+            static CANON_64BIT_NAN: u64 =
+                0b0111111111111000000000000000000000000000000000000000000000000000;
+            self.pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN))
+        }
     };
 }
 
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index dfa0f5d0b88c..e62e3d62fd9f 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -386,3 +386,78 @@
 
 (decl replace_with_aliases (Value) Inst)
 (extern constructor replace_with_aliases replace_with_aliases)
+
+;; ========= NaN canonicalization =============================================
+
+(rule (legalize (has_type ty (ceil val)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_ceil ty val)))
+
+(rule (legalize (has_type ty (floor val)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_floor ty val)))
+
+(rule (legalize (has_type ty (nearest val)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_nearest ty val)))
+
+(rule (legalize (has_type ty (trunc val)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_trunc ty val)))
+
+(rule (legalize (has_type ty (sqrt val)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_sqrt ty val)))
+
+(rule (legalize (has_type ty (fadd a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fadd ty a b)))
+
+(rule (legalize (has_type ty (fsub a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fsub ty a b)))
+
+(rule (legalize (has_type ty (fmul a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fmul ty a b)))
+
+(rule (legalize (has_type ty (fdiv a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fdiv ty a b)))
+
+(rule (legalize (has_type ty (fmin a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fmin ty a b)))
+
+(rule (legalize (has_type ty (fmax a b)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fmax ty a b)))
+
+(rule (legalize (has_type ty (fma a b c)))
+  (if-let $true (enable_nan_canonicalization))
+  (replace_canonicalized ty (ins_fma ty a b c)))
+
+(decl replace_canonicalized (Type Value) Inst)
+(rule (replace_canonicalized $F32 val)
+  (replace_select $F32 (ins_fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan32) val))
+(rule (replace_canonicalized $F64 val)
+  (replace_select $F64 (ins_fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan64) val))
+(rule (replace_canonicalized $F32X4 val)
+  (let ((cmp Value (ins_fcmp $I32X4 (FloatCC.NotEqual) val val))
+        (canon Value (ins_splat $F32X4 (canon_nan32)))
+        (mask Value (ins_bitcast $F32X4 (mem_flags_empty) cmp)))
+    (replace_bitselect $F32X4 mask canon val)))
+(rule (replace_canonicalized $F64X2 val)
+  (let ((canon Value (ins_splat $F64X2 (canon_nan64)))
+        (cmp Value (ins_fcmp $I64X2 (FloatCC.NotEqual) val val))
+        (mask Value (ins_bitcast $F64X2 (mem_flags_empty) cmp)))
+    (replace_bitselect $F64X2 mask canon val)))
+
+(decl pure enable_nan_canonicalization () bool)
+(extern constructor enable_nan_canonicalization enable_nan_canonicalization)
+
+(decl canon_nan32 () Value)
+(extern constructor canon_nan32 canon_nan32)
+
+(decl canon_nan64 () Value)
+(extern constructor canon_nan64 canon_nan64)
diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs
index c856e1a1b645..9d2e9200c159 100644
--- a/cranelift/codegen/src/lib.rs
+++ b/cranelift/codegen/src/lib.rs
@@ -77,7 +77,6 @@ mod inst_predicates;
 mod isle_prelude;
 mod iterators;
 mod legalizer;
-mod nan_canonicalization;
 mod opts;
 mod remove_constant_phis;
 mod result;
diff --git a/cranelift/codegen/src/nan_canonicalization.rs b/cranelift/codegen/src/nan_canonicalization.rs
deleted file mode 100644
index 49415a86b0d6..000000000000
--- a/cranelift/codegen/src/nan_canonicalization.rs
+++ /dev/null
@@ -1,106 +0,0 @@
-//! A NaN-canonicalizing rewriting pass. Patch floating point arithmetic
-//! instructions that may return a NaN result with a sequence of operations
-//! that will replace nondeterministic NaN's with a single canonical NaN value.
-
-use crate::cursor::{Cursor, FuncCursor};
-use crate::ir::condcodes::FloatCC;
-use crate::ir::immediates::{Ieee32, Ieee64};
-use crate::ir::types;
-use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value};
-use crate::opts::MemFlags;
-use crate::timing;
-
-// Canonical 32-bit and 64-bit NaN values.
-static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000;
-static CANON_64BIT_NAN: u64 = 0b0111111111111000000000000000000000000000000000000000000000000000;
-
-/// Perform the NaN canonicalization pass.
-pub fn do_nan_canonicalization(func: &mut Function) {
-    let _tt = timing::canonicalize_nans();
-    let mut pos = FuncCursor::new(func);
-    while let Some(_block) = pos.next_block() {
-        while let Some(inst) = pos.next_inst() {
-            if is_fp_arith(&mut pos, inst) {
-                add_nan_canon_seq(&mut pos, inst);
-            }
-        }
-    }
-}
-
-/// Returns true/false based on whether the instruction is a floating-point
-/// arithmetic operation. This ignores operations like `fneg`, `fabs`, or
-/// `fcopysign` that only operate on the sign bit of a floating point value.
-fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool {
-    match pos.func.dfg.insts[inst] {
-        InstructionData::Unary { opcode, .. } => {
-            opcode == Opcode::Ceil
-                || opcode == Opcode::Floor
-                || opcode == Opcode::Nearest
-                || opcode == Opcode::Sqrt
-                || opcode == Opcode::Trunc
-        }
-        InstructionData::Binary { opcode, .. } => {
-            opcode == Opcode::Fadd
-                || opcode == Opcode::Fdiv
-                || opcode == Opcode::Fmax
-                || opcode == Opcode::Fmin
-                || opcode == Opcode::Fmul
-                || opcode == Opcode::Fsub
-        }
-        InstructionData::Ternary { opcode, .. } => opcode == Opcode::Fma,
-        _ => false,
-    }
-}
-
-/// Append a sequence of canonicalizing instructions after the given instruction.
-fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
-    // Select the instruction result, result type. Replace the instruction
-    // result and step forward before inserting the canonicalization sequence.
-    let val = pos.func.dfg.first_result(inst);
-    let val_type = pos.func.dfg.value_type(val);
-    let new_res = pos.func.dfg.replace_result(val, val_type);
-    let _next_inst = pos.next_inst().expect("block missing terminator!");
-
-    // Insert a comparison instruction, to check if `inst_res` is NaN. Select
-    // the canonical NaN value if `val` is NaN, assign the result to `inst`.
-    let is_nan = pos.ins().fcmp(FloatCC::NotEqual, new_res, new_res);
-
-    let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| {
-        pos.ins()
-            .with_result(val)
-            .select(is_nan, canon_nan, new_res);
-    };
-    let vector_select = |pos: &mut FuncCursor, canon_nan: Value| {
-        let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan);
-        pos.ins()
-            .with_result(val)
-            .bitselect(is_nan, canon_nan, new_res);
-    };
-
-    match val_type {
-        types::F32 => {
-            let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
-            scalar_select(pos, canon_nan);
-        }
-        types::F64 => {
-            let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
-            scalar_select(pos, canon_nan);
-        }
-        types::F32X4 => {
-            let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
-            let canon_nan = pos.ins().splat(types::F32X4, canon_nan);
-            vector_select(pos, canon_nan);
-        }
-        types::F64X2 => {
-            let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
-            let canon_nan = pos.ins().splat(types::F64X2, canon_nan);
-            vector_select(pos, canon_nan);
-        }
-        _ => {
-            // Panic if the type given was not an IEEE floating point type.
-            panic!("Could not canonicalize NaN: Unexpected result type found.");
-        }
-    }
-
-    pos.prev_inst(); // Step backwards so the pass does not skip instructions.
-}
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index 81333bb593da..a2cf1be788b4 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -341,6 +341,10 @@
 (decl pure mem_flags_notrap () MemFlags)
 (extern constructor mem_flags_notrap mem_flags_notrap)
 
+;; `MemFlags::new`
+(decl pure mem_flags_empty () MemFlags)
+(extern constructor mem_flags_empty mem_flags_empty)
+
 ;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Swap args of an IntCC flag.
diff --git a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
index 2750356a6010..6bc1b88e0659 100644
--- a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
+++ b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
@@ -17,7 +17,6 @@ block0(v0: f32):
 ;     return v1
 ; }
 
-
 function %floor(f64) -> f64 {
 block0(v0: f64):
   v1 = floor v0
@@ -59,11 +58,11 @@ block0(v0: f64x2):
 ; function %sqrt(f64x2) -> f64x2 fast {
 ; block0(v0: f64x2):
 ;     v2 = sqrt v0
-;     v3 = fcmp ne v2, v2
-;     v4 = f64const +NaN
-;     v5 = splat.f64x2 v4  ; v4 = +NaN
-;     v6 = bitcast.f64x2 v3
-;     v1 = bitselect v6, v5, v2
+;     v3 = f64const +NaN
+;     v4 = splat.f64x2 v3  ; v3 = +NaN
+;     v5 = fcmp ne v2, v2
+;     v6 = bitcast.f64x2 v5
+;     v1 = bitselect v6, v4, v2
 ;     return v1
 ; }
 
diff --git a/cranelift/fuzzgen/src/lib.rs b/cranelift/fuzzgen/src/lib.rs
index d71658182492..80257473fb27 100644
--- a/cranelift/fuzzgen/src/lib.rs
+++ b/cranelift/fuzzgen/src/lib.rs
@@ -122,6 +122,7 @@ where
         let flags = settings::Flags::new({
             let mut builder = settings::builder();
             builder.set("enable_verifier", "false").unwrap();
+            builder.set("enable_nan_canonicalization", "true").unwrap();
             builder
         });
 
@@ -129,9 +130,7 @@ where
             .expect("Unable to build a TargetIsa for the current host")
             .finish(flags)
             .expect("Failed to build TargetISA");
-
-        ctx.canonicalize_nans(isa.as_ref())
-            .expect("Failed NaN canonicalization pass");
+        isa.legalize_function(&mut ctx.func, &mut ctx.cfg);
 
         // Run the int_divz pass
         //

From 0099414124e58af60165075f3bd2faa1e4c30e71 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 13:58:41 -0700
Subject: [PATCH 15/21] x64: Replace `CvtUint64ToFloatSeq` with legalization

This should hopefully be a bit more easy to read and more understandable
because it's done at a higher level than before. Additionally this fixes
minor issues such as using AVX instructions when available and
additionally breaking false dependencies with a leading `xorps` of the
destination register.
---
 cranelift/codegen/build.rs                    |   2 +-
 cranelift/codegen/src/isa/x64/inst.isle       |  16 --
 cranelift/codegen/src/isa/x64/inst/emit.rs    | 148 ------------------
 cranelift/codegen/src/isa/x64/inst/mod.rs     |  36 -----
 cranelift/codegen/src/isa/x64/legalize.isle   |   8 +
 cranelift/codegen/src/isa/x64/legalize.rs     |  82 +++++++++-
 cranelift/codegen/src/isa/x64/lower.isle      |   3 +-
 .../filetests/filetests/isa/x64/fcvt.clif     |  55 +++++--
 8 files changed, 129 insertions(+), 221 deletions(-)
 create mode 100644 cranelift/codegen/src/isa/x64/legalize.isle

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index e5167d87bb97..1ac9af1bf46a 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -255,7 +255,7 @@ fn get_isle_compilations(
                 inputs: vec![
                     prelude_isle.clone(),
                     legalize_shared_isle.clone(),
-                    // NB: x86-64-specific legalization here when it exists
+                    src_isa_x64.join("legalize.isle"),
                 ],
                 untracked_inputs: vec![clif_legalize_isle.clone()],
             },
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index e55867ab45ab..784ec11623b7 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -454,13 +454,6 @@
                          (dst WritableXmm)
                          (src2_size OperandSize))
 
-       ;; Converts an unsigned int64 to a float32/float64.
-       (CvtUint64ToFloatSeq (dst_size OperandSize) ;; 4 or 8
-                            (src Gpr)
-                            (dst WritableXmm)
-                            (tmp_gpr1 WritableGpr)
-                            (tmp_gpr2 WritableGpr))
-
        ;; Converts a scalar xmm to a signed int32/int64.
        (CvtFloatToSintSeq (dst_size OperandSize)
                           (src_size OperandSize)
@@ -2095,15 +2088,6 @@
             (_ Unit (emit (MInst.CvtIntToFloatVex op src1 src2 dst size))))
         dst))
 
-(decl cvt_u64_to_float_seq (Type Gpr) Xmm)
-(rule (cvt_u64_to_float_seq ty src)
-      (let ((size OperandSize (raw_operand_size_of_type ty))
-            (dst WritableXmm (temp_writable_xmm))
-            (tmp_gpr1 WritableGpr (temp_writable_gpr))
-            (tmp_gpr2 WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.CvtUint64ToFloatSeq size src dst tmp_gpr1 tmp_gpr2))))
-        dst))
-
 (decl cvt_float_to_uint_seq (Type Value bool) Gpr)
 (rule (cvt_float_to_uint_seq out_ty src @ (value_type src_ty) is_saturating)
       (let ((out_size OperandSize (raw_operand_size_of_type out_ty))
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index 6bd65154eb23..7a53f6eb4b07 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -15,33 +15,6 @@ use crate::isa::x64::inst::*;
 use crate::machinst::{inst_common, MachBuffer, MachInstEmit, MachLabel, Reg, Writable};
 use core::convert::TryInto;
 
-/// A small helper to generate a signed conversion instruction.
-fn emit_signed_cvt(
-    sink: &mut MachBuffer<Inst>,
-    info: &EmitInfo,
-    state: &mut EmitState,
-    // Required to be RealRegs.
-    src: Reg,
-    dst: Writable<Reg>,
-    to_f64: bool,
-) {
-    // Handle an unsigned int, which is the "easy" case: a signed conversion will do the
-    // right thing.
-    let op = if to_f64 {
-        SseOpcode::Cvtsi2sd
-    } else {
-        SseOpcode::Cvtsi2ss
-    };
-    Inst::CvtIntToFloat {
-        op,
-        dst: Writable::from_reg(Xmm::new(dst.to_reg()).unwrap()),
-        src1: Xmm::new(dst.to_reg()).unwrap(),
-        src2: GprMem::new(RegMem::reg(src)).unwrap(),
-        src2_size: OperandSize::Size64,
-    }
-    .emit(&[], sink, info, state);
-}
-
 /// Emits a one way conditional jump if CC is set (true).
 fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
     let cond_start = sink.cur_offset();
@@ -3300,127 +3273,6 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
-        Inst::CvtUint64ToFloatSeq {
-            dst_size,
-            src,
-            dst,
-            tmp_gpr1,
-            tmp_gpr2,
-        } => {
-            let src = allocs.next(src.to_reg());
-            let dst = allocs.next(dst.to_reg().to_reg());
-            let tmp_gpr1 = allocs.next(tmp_gpr1.to_reg().to_reg());
-            let tmp_gpr2 = allocs.next(tmp_gpr2.to_reg().to_reg());
-
-            // Note: this sequence is specific to 64-bit mode; a 32-bit mode would require a
-            // different sequence.
-            //
-            // Emit the following sequence:
-            //
-            //  cmp 0, %src
-            //  jl handle_negative
-            //
-            //  ;; handle positive, which can't overflow
-            //  cvtsi2sd/cvtsi2ss %src, %dst
-            //  j done
-            //
-            //  ;; handle negative: see below for an explanation of what it's doing.
-            //  handle_negative:
-            //  mov %src, %tmp_gpr1
-            //  shr $1, %tmp_gpr1
-            //  mov %src, %tmp_gpr2
-            //  and $1, %tmp_gpr2
-            //  or %tmp_gpr1, %tmp_gpr2
-            //  cvtsi2sd/cvtsi2ss %tmp_gpr2, %dst
-            //  addsd/addss %dst, %dst
-            //
-            //  done:
-
-            assert_ne!(src, tmp_gpr1);
-            assert_ne!(src, tmp_gpr2);
-            assert_ne!(tmp_gpr1, tmp_gpr2);
-
-            let handle_negative = sink.get_label();
-            let done = sink.get_label();
-
-            // If x seen as a signed int64 is not negative, a signed-conversion will do the right
-            // thing.
-            // TODO use tst src, src here.
-            let inst = Inst::cmp_rmi_r(OperandSize::Size64, RegMemImm::imm(0), src);
-            inst.emit(&[], sink, info, state);
-
-            one_way_jmp(sink, CC::L, handle_negative);
-
-            // Handle a positive int64, which is the "easy" case: a signed conversion will do the
-            // right thing.
-            emit_signed_cvt(
-                sink,
-                info,
-                state,
-                src,
-                Writable::from_reg(dst),
-                *dst_size == OperandSize::Size64,
-            );
-
-            let inst = Inst::jmp_known(done);
-            inst.emit(&[], sink, info, state);
-
-            sink.bind_label(handle_negative, state.ctrl_plane_mut());
-
-            // Divide x by two to get it in range for the signed conversion, keep the LSB, and
-            // scale it back up on the FP side.
-            let inst = Inst::gen_move(Writable::from_reg(tmp_gpr1), src, types::I64);
-            inst.emit(&[], sink, info, state);
-
-            // tmp_gpr1 := src >> 1
-            let inst = Inst::shift_r(
-                OperandSize::Size64,
-                ShiftKind::ShiftRightLogical,
-                Imm8Gpr::new(Imm8Reg::Imm8 { imm: 1 }).unwrap(),
-                tmp_gpr1,
-                Writable::from_reg(tmp_gpr1),
-            );
-            inst.emit(&[], sink, info, state);
-
-            let inst = Inst::gen_move(Writable::from_reg(tmp_gpr2), src, types::I64);
-            inst.emit(&[], sink, info, state);
-
-            let inst = Inst::alu_rmi_r(
-                OperandSize::Size64,
-                AluRmiROpcode::And,
-                RegMemImm::imm(1),
-                Writable::from_reg(tmp_gpr2),
-            );
-            inst.emit(&[], sink, info, state);
-
-            let inst = Inst::alu_rmi_r(
-                OperandSize::Size64,
-                AluRmiROpcode::Or,
-                RegMemImm::reg(tmp_gpr1),
-                Writable::from_reg(tmp_gpr2),
-            );
-            inst.emit(&[], sink, info, state);
-
-            emit_signed_cvt(
-                sink,
-                info,
-                state,
-                tmp_gpr2,
-                Writable::from_reg(dst),
-                *dst_size == OperandSize::Size64,
-            );
-
-            let add_op = if *dst_size == OperandSize::Size64 {
-                SseOpcode::Addsd
-            } else {
-                SseOpcode::Addss
-            };
-            let inst = Inst::xmm_rm_r(add_op, RegMem::reg(dst), Writable::from_reg(dst));
-            inst.emit(&[], sink, info, state);
-
-            sink.bind_label(done, state.ctrl_plane_mut());
-        }
-
         Inst::CvtFloatToSintSeq {
             src_size,
             dst_size,
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 19bda166c19f..3c8abd648f00 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -108,7 +108,6 @@ impl Inst {
             | Inst::CmpRmiR { .. }
             | Inst::CvtFloatToSintSeq { .. }
             | Inst::CvtFloatToUintSeq { .. }
-            | Inst::CvtUint64ToFloatSeq { .. }
             | Inst::Div { .. }
             | Inst::Div8 { .. }
             | Inst::Fence { .. }
@@ -1326,29 +1325,6 @@ impl PrettyPrint for Inst {
                 format!("{op} {src1}, {src2}, {dst}")
             }
 
-            Inst::CvtUint64ToFloatSeq {
-                src,
-                dst,
-                dst_size,
-                tmp_gpr1,
-                tmp_gpr2,
-                ..
-            } => {
-                let src = pretty_print_reg(src.to_reg(), 8, allocs);
-                let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
-                let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8, allocs);
-                let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8, allocs);
-                let op = ljustify(format!(
-                    "u64_to_{}_seq",
-                    if *dst_size == OperandSize::Size64 {
-                        "f64"
-                    } else {
-                        "f32"
-                    }
-                ));
-                format!("{op} {src}, {dst}, {tmp_gpr1}, {tmp_gpr2}")
-            }
-
             Inst::CvtFloatToSintSeq {
                 src,
                 dst,
@@ -2208,18 +2184,6 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
             collector.reg_use(src1.to_reg());
             src2.get_operands(collector);
         }
-        Inst::CvtUint64ToFloatSeq {
-            src,
-            dst,
-            tmp_gpr1,
-            tmp_gpr2,
-            ..
-        } => {
-            collector.reg_use(src.to_reg());
-            collector.reg_early_def(dst.to_writable_reg());
-            collector.reg_early_def(tmp_gpr1.to_writable_reg());
-            collector.reg_early_def(tmp_gpr2.to_writable_reg());
-        }
         Inst::CvtFloatToSintSeq {
             src,
             dst,
diff --git a/cranelift/codegen/src/isa/x64/legalize.isle b/cranelift/codegen/src/isa/x64/legalize.isle
new file mode 100644
index 000000000000..d824b5392bce
--- /dev/null
+++ b/cranelift/codegen/src/isa/x64/legalize.isle
@@ -0,0 +1,8 @@
+;; ========= `fcvt_from_uint` =================================================
+
+(rule (legalize (has_type ty (fcvt_from_uint v @ (value_type $I64))))
+  (x64_replace_fcvt_from_u64 ty v))
+
+(decl x64_replace_fcvt_from_u64 (Type Value) CursorPosition)
+(extern constructor x64_replace_fcvt_from_u64 x64_replace_fcvt_from_u64)
+
diff --git a/cranelift/codegen/src/isa/x64/legalize.rs b/cranelift/codegen/src/isa/x64/legalize.rs
index f9a7aaada42c..84342f3f23f2 100644
--- a/cranelift/codegen/src/isa/x64/legalize.rs
+++ b/cranelift/codegen/src/isa/x64/legalize.rs
@@ -4,7 +4,7 @@ use crate::isa::x64::X64Backend;
 use crate::legalizer::isle;
 
 // Used by ISLE
-use crate::cursor::CursorPosition;
+use crate::cursor::{Cursor, CursorPosition};
 use crate::ir::condcodes::*;
 use crate::ir::immediates::*;
 use crate::ir::types::*;
@@ -28,4 +28,84 @@ pub(crate) fn run(
 
 impl generated::Context for isle::LegalizeContext<'_, X64Backend> {
     crate::isle_common_legalizer_methods!();
+
+    /// On x64 there's no native instruction for converting an unsigned 64-bit
+    /// number into a float, only a signed 64-bit number. To handle this
+    /// `fcvt_from_uint` instructions are replaced with this sequence below.
+    ///
+    /// If the input value interpreted as a signed number is positive then
+    /// `fcvt_from_sint` can be used because it will produce the same result.
+    ///
+    /// If the input value interpreted as a signed number is negative then
+    /// the rough idea is to shift it to the right by one to halve it, use
+    /// `fcvt_from_sint` on the result, and then double the result with an
+    /// `fadd` afterwards.
+    ///
+    /// TODO: the previous incarnation of this code has a "see below for an
+    /// explanation" comment for what it was doing and I don't know why the
+    /// lower bit is preserved in the shifted-right temporary. I presume that
+    /// has to do with something about how a u64 is so large than f64 doesn't
+    /// have the precision to hold the lowest bit so this trick produces the
+    /// right result "because of float precision weirdness". If someone knows
+    /// better it'd be good to fill out this comment.
+    fn x64_replace_fcvt_from_u64(&mut self, ty: Type, uint: Value) -> CursorPosition {
+        let inst = self.replace.unwrap();
+        let old_block = self
+            .pos
+            .func
+            .layout
+            .inst_block(inst)
+            .expect("inst not in layout");
+        let block_is_nonnegative = self.pos.func.dfg.make_block();
+        let block_is_negative = self.pos.func.dfg.make_block();
+        let block_resume = self.pos.func.dfg.make_block();
+
+        // Test whether the input is negative when interpreted as a signed
+        // number, and proceed from here.
+        let zero = self.pos.ins().iconst(I64, 0);
+        let is_negative = self.pos.ins().icmp(IntCC::SignedLessThan, uint, zero);
+        self.pos.ins().brif(
+            is_negative,
+            block_is_negative,
+            &[],
+            block_is_nonnegative,
+            &[],
+        );
+
+        // If the input value was negative "do the trick". More-or-less use a
+        // signed conversion on a half-value and then double it afterwards.
+        self.pos.insert_block(block_is_negative);
+        let one = self.pos.ins().iconst(I64, 1);
+        let ushr = self.pos.ins().ushr(uint, one);
+        let band = self.pos.ins().band(uint, one);
+        let bor = self.pos.ins().bor(ushr, band);
+        let fcvt = self.pos.ins().fcvt_from_sint(ty, bor);
+        let fadd = self.pos.ins().fadd(fcvt, fcvt);
+        self.pos.ins().jump(block_resume, &[fadd]);
+
+        // If the input value was positive then `fcvt_from_sint` can be used
+        // since it'd produce the same result.
+        self.pos.insert_block(block_is_nonnegative);
+        let fcvt = self.pos.ins().fcvt_from_sint(ty, uint);
+        self.pos.ins().jump(block_resume, &[fcvt]);
+
+        // At the join point remove the previous `fcvt_from_uint` instruction
+        // and change its result to alias our new result which is a parameter to
+        // this block.
+        self.pos.insert_block(block_resume);
+        let result = self.pos.func.dfg.append_block_param(block_resume, ty);
+        let prev_result = self.pos.func.dfg.first_result(inst);
+        self.pos.func.dfg.clear_results(inst);
+        self.pos.func.dfg.change_to_alias(prev_result, result);
+        self.pos.func.layout.remove_inst(inst);
+
+        // Finally update the CFG.
+        self.cfg.recompute_block(self.pos.func, old_block);
+        self.cfg
+            .recompute_block(self.pos.func, block_is_nonnegative);
+        self.cfg.recompute_block(self.pos.func, block_is_negative);
+        self.cfg.recompute_block(self.pos.func, block_resume);
+
+        CursorPosition::Before(block_resume)
+    }
 }
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index e0d17fe627a4..84ce6041a737 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -3409,8 +3409,7 @@
 (rule 1 (lower (has_type $F64 (fcvt_from_uint val @ (value_type (fits_in_32 (ty_int ty))))))
       (x64_cvtsi2sd $I64 (xmm_zero $F64X2) (extend_to_gpr val $I64 (ExtendKind.Zero))))
 
-(rule (lower (has_type ty (fcvt_from_uint val @ (value_type $I64))))
-      (cvt_u64_to_float_seq ty val))
+;; NB: fcvt_from_uint for 64-bit values is handled in `legalize.isle.`
 
 ;; Algorithm uses unpcklps to help create a float that is equivalent
 ;; 0x1.0p52 + double(src). 0x1.0p52 is unique because at this exponent
diff --git a/cranelift/filetests/filetests/isa/x64/fcvt.clif b/cranelift/filetests/filetests/isa/x64/fcvt.clif
index 2533ccb85fa8..75ee658ccdff 100644
--- a/cranelift/filetests/filetests/isa/x64/fcvt.clif
+++ b/cranelift/filetests/filetests/isa/x64/fcvt.clif
@@ -285,12 +285,29 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64):
 ;   cvtsi2ss %xmm6, %r8, %xmm6
 ;   uninit  %xmm7
 ;   xorps   %xmm7, %xmm7, %xmm7
-;   movl    %edx, %r8d
-;   cvtsi2ss %xmm7, %r8, %xmm7
-;   u64_to_f32_seq %rcx, %xmm4, %r8, %rdx
+;   movl    %edx, %edx
+;   cvtsi2ss %xmm7, %rdx, %xmm7
+;   testq   %rcx, %rcx
+;   jl      label2; j label1
+; block1:
+;   uninit  %xmm3
+;   xorps   %xmm3, %xmm3, %xmm3
+;   cvtsi2ss %xmm3, %rcx, %xmm3
+;   jmp     label3
+; block2:
+;   movq    %rcx, %rdx
+;   shrq    $1, %rdx, %rdx
+;   andq    %rcx, $1, %rcx
+;   orq     %rdx, %rcx, %rdx
+;   uninit  %xmm3
+;   xorps   %xmm3, %xmm3, %xmm3
+;   cvtsi2ss %xmm3, %rdx, %xmm3
+;   addss   %xmm3, %xmm3, %xmm3
+;   jmp     label3
+; block3:
 ;   addss   %xmm0, %xmm6, %xmm0
 ;   addss   %xmm0, %xmm7, %xmm0
-;   addss   %xmm0, %xmm4, %xmm0
+;   addss   %xmm0, %xmm3, %xmm0
 ;   movq    %rbp, %rsp
 ;   popq    %rbp
 ;   ret
@@ -307,22 +324,26 @@ block0(v0: i8, v1: i16, v2: i32, v3: i64):
 ;   movzwq %si, %r8
 ;   cvtsi2ssq %r8, %xmm6
 ;   xorps %xmm7, %xmm7
-;   movl %edx, %r8d
-;   cvtsi2ssq %r8, %xmm7
-;   cmpq $0, %rcx
-;   jl 0x3b
-;   cvtsi2ssq %rcx, %xmm4
-;   jmp 0x55
-;   movq %rcx, %r8
-;   shrq $1, %r8
+;   movl %edx, %edx
+;   cvtsi2ssq %rdx, %xmm7
+;   testq %rcx, %rcx
+;   jl 0x3c
+; block2: ; offset 0x2f
+;   xorps %xmm3, %xmm3
+;   cvtsi2ssq %rcx, %xmm3
+;   jmp 0x56
+; block3: ; offset 0x3c
 ;   movq %rcx, %rdx
-;   andq $1, %rdx
-;   orq %r8, %rdx
-;   cvtsi2ssq %rdx, %xmm4
-;   addss %xmm4, %xmm4
+;   shrq $1, %rdx
+;   andq $1, %rcx
+;   orq %rcx, %rdx
+;   xorps %xmm3, %xmm3
+;   cvtsi2ssq %rdx, %xmm3
+;   addss %xmm3, %xmm3
+; block4: ; offset 0x56
 ;   addss %xmm6, %xmm0
 ;   addss %xmm7, %xmm0
-;   addss %xmm4, %xmm0
+;   addss %xmm3, %xmm0
 ;   movq %rbp, %rsp
 ;   popq %rbp
 ;   retq

From 799747ccb4db390d89e4619a4573991550b52b91 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 15:41:58 -0700
Subject: [PATCH 16/21] Delete unused function

---
 cranelift/codegen/src/isa/x64/inst/mod.rs | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 3c8abd648f00..95f4f72b072d 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -473,26 +473,6 @@ impl Inst {
         }
     }
 
-    pub(crate) fn shift_r(
-        size: OperandSize,
-        kind: ShiftKind,
-        num_bits: Imm8Gpr,
-        src: Reg,
-        dst: Writable<Reg>,
-    ) -> Inst {
-        if let Imm8Reg::Imm8 { imm: num_bits } = num_bits.clone().to_imm8_reg() {
-            debug_assert!(num_bits < size.to_bits());
-        }
-        debug_assert!(dst.to_reg().class() == RegClass::Int);
-        Inst::ShiftR {
-            size,
-            kind,
-            src: Gpr::new(src).unwrap(),
-            num_bits,
-            dst: WritableGpr::from_writable_reg(dst).unwrap(),
-        }
-    }
-
     /// Does a comparison of dst - src for operands of size `size`, as stated by the machine
     /// instruction semantics. Be careful with the order of parameters!
     pub(crate) fn cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {

From 42f91d50a5349e60aa981a39e01da7362400ba51 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 15:53:56 -0700
Subject: [PATCH 17/21] Add back in method for tests

---
 cranelift/codegen/src/isa/x64/inst/mod.rs | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 95f4f72b072d..dca711e3a11b 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -473,6 +473,27 @@ impl Inst {
         }
     }
 
+    #[cfg(test)]
+    pub(crate) fn shift_r(
+        size: OperandSize,
+        kind: ShiftKind,
+        num_bits: Imm8Gpr,
+        src: Reg,
+        dst: Writable<Reg>,
+    ) -> Inst {
+        if let Imm8Reg::Imm8 { imm: num_bits } = num_bits.clone().to_imm8_reg() {
+            debug_assert!(num_bits < size.to_bits());
+        }
+        debug_assert!(dst.to_reg().class() == RegClass::Int);
+        Inst::ShiftR {
+            size,
+            kind,
+            src: Gpr::new(src).unwrap(),
+            num_bits,
+            dst: WritableGpr::from_writable_reg(dst).unwrap(),
+        }
+    }
+
     /// Does a comparison of dst - src for operands of size `size`, as stated by the machine
     /// instruction semantics. Be careful with the order of parameters!
     pub(crate) fn cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {

From cb2d4b0ca20a585cf53677f452bc72a9213676ef Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Sat, 21 Oct 2023 16:34:43 -0700
Subject: [PATCH 18/21] Fix x64 test issue arising

---
 cranelift/codegen/src/opts/extends.isle          | 6 ++++++
 cranelift/filetests/filetests/egraph/vector.clif | 7 +++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/cranelift/codegen/src/opts/extends.isle b/cranelift/codegen/src/opts/extends.isle
index a1a4dfda0596..83ded0d7479f 100644
--- a/cranelift/codegen/src/opts/extends.isle
+++ b/cranelift/codegen/src/opts/extends.isle
@@ -32,3 +32,9 @@
 ;; actually doing the extend in the first place.
 (rule (simplify (ireduce ty (sextend _ x @ (value_type ty)))) x)
 (rule (simplify (ireduce ty (uextend _ x @ (value_type ty)))) x)
+
+;; Ignore extens in `fcvt_from_{uint,sint}` since extension won't change the
+;; result and backends can sometimes have more optimal lowerings if it's known
+;; that the type is smaller.
+(rule (simplify (fcvt_from_uint ty (uextend _ x))) (fcvt_from_uint ty x))
+(rule (simplify (fcvt_from_uint ty (sextend _ x))) (fcvt_from_uint ty x))
diff --git a/cranelift/filetests/filetests/egraph/vector.clif b/cranelift/filetests/filetests/egraph/vector.clif
index 4ecbaa83b318..a9cb2cbe7746 100644
--- a/cranelift/filetests/filetests/egraph/vector.clif
+++ b/cranelift/filetests/filetests/egraph/vector.clif
@@ -18,10 +18,9 @@ block0(v0: i32):
   v2 = splat.i64x2 v1
   v3 = fcvt_from_uint.f64x2 v2
   return v3
-  ; check: v1 = uextend.i64 v0
-  ; check: v4 = fcvt_from_uint.f64 v1
-  ; check: v5 = splat.f64x2 v4
-  ; check: return v5
+  ; check: v5 = fcvt_from_uint.f64 v0
+  ; check: v7 = splat.f64x2 v5
+  ; check: return v7
 }
 
 function %f3(i32) -> f64x2 {

From 96cb6f0e9415943dee96b92add46288545e83178 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 13 Nov 2023 13:01:19 -0800
Subject: [PATCH 19/21] Fix issues with rebase

---
 cranelift/codegen/src/isa/x64/pcc.rs               | 12 ------------
 cranelift/codegen/src/legalizer/isle.rs            |  9 +++++++++
 .../codegen/src/legalizer/legalize_shared.isle     | 14 +++++++++++---
 cranelift/isle/isle/src/codegen.rs                 |  1 -
 4 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/cranelift/codegen/src/isa/x64/pcc.rs b/cranelift/codegen/src/isa/x64/pcc.rs
index 97afc57466bd..90d9740f0f38 100644
--- a/cranelift/codegen/src/isa/x64/pcc.rs
+++ b/cranelift/codegen/src/isa/x64/pcc.rs
@@ -661,18 +661,6 @@ pub(crate) fn check(
             ensure_no_fact(vcode, dst.to_writable_reg().to_reg())
         }
 
-        Inst::CvtUint64ToFloatSeq {
-            dst,
-            tmp_gpr1,
-            tmp_gpr2,
-            ..
-        } => {
-            ensure_no_fact(vcode, dst.to_writable_reg().to_reg())?;
-            ensure_no_fact(vcode, tmp_gpr1.to_writable_reg().to_reg())?;
-            ensure_no_fact(vcode, tmp_gpr2.to_writable_reg().to_reg())?;
-            Ok(())
-        }
-
         Inst::CvtFloatToSintSeq {
             dst,
             tmp_gpr,
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index 30ad0632d5fc..a0b5b5cea2bc 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -170,6 +170,15 @@ macro_rules! isle_common_legalizer_methods {
             }
         }
 
+        fn update_const_facts(&mut self, ty: Type, imm: Imm64, gv: GlobalValue, constant: Value) {
+            if self.pos.func.global_value_facts[gv].is_none() {
+                return;
+            }
+            let bits = u16::try_from(ty.bits()).unwrap();
+            let unsigned_offset = imm.bits() as u64; // Safety: reinterpret i64 bits as u64.
+            self.pos.func.dfg.facts[constant] = Some(Fact::constant(bits, unsigned_offset));
+        }
+
         fn replace_vmctx_addr(&mut self, global_value: ir::GlobalValue) -> CursorPosition {
             // Get the value representing the `vmctx` argument.
             let vmctx = self
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index e62e3d62fd9f..a84adbcc256a 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -263,9 +263,10 @@
     ;; re-legalize the previous instrs to legalize the new `global_value` node
     (prev_position)))
 
-(rule (legalize (global_value (gv_iadd_imm base offset ty)))
-  (let ((base Value (ins_global_value_copy_facts ty base))
-        (_ Inst (replace_iadd_imm ty base offset)))
+(rule (legalize (global_value (gv_iadd_imm gv_base offset ty)))
+  (let ((base Value (ins_global_value_copy_facts ty gv_base))
+        (offset Value (ins_iconst_with_fact ty offset gv_base))
+        (_ Inst (replace_iadd ty base offset)))
     ;; re-legalize the previous instrs to legalize the new `global_value` node
     (prev_position)))
 
@@ -332,8 +333,15 @@
   (let ((ret Inst (ins_global_value ty gv))
         (_ Unit (update_inst_facts_with_gv gv ret)))
     ret))
+(decl ins_iconst_with_fact (Type Imm64 GlobalValue) Inst)
+(rule (ins_iconst_with_fact ty imm gv)
+  (let ((ret Inst (ins_iconst ty imm))
+        (_ Unit (update_const_facts ty imm gv ret)))
+    ret))
 (decl update_inst_facts_with_gv (GlobalValue Value) Unit)
 (extern constructor update_inst_facts_with_gv update_inst_facts_with_gv)
+(decl update_const_facts (Type Imm64 GlobalValue Value) Unit)
+(extern constructor update_const_facts update_const_facts)
 
 ;; ========= `table_addr` =====================================================
 
diff --git a/cranelift/isle/isle/src/codegen.rs b/cranelift/isle/isle/src/codegen.rs
index d5eae5a09235..45ea723bdc49 100644
--- a/cranelift/isle/isle/src/codegen.rs
+++ b/cranelift/isle/isle/src/codegen.rs
@@ -135,7 +135,6 @@ impl<'a> Codegen<'a> {
         }
 
         writeln!(code, "\nuse super::*;  // Pulls in all external types.").unwrap();
-        writeln!(code, "use std::marker::PhantomData;").unwrap();
     }
 
     fn generate_trait_sig(&self, code: &mut String, indent: &str, sig: &ExternalSig) {

From c619784d1312b70a7ef95fd02a632a6bc014f4cb Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 13 Nov 2023 13:40:26 -0800
Subject: [PATCH 20/21] Only build necessary ISLE compilations

Use Cargo features to disable ISLE compiles if they're not needed.
---
 cranelift/codegen/build.rs | 239 ++++++++++++++++++++-----------------
 1 file changed, 127 insertions(+), 112 deletions(-)

diff --git a/cranelift/codegen/build.rs b/cranelift/codegen/build.rs
index 1ac9af1bf46a..77ae456ac978 100644
--- a/cranelift/codegen/build.rs
+++ b/cranelift/codegen/build.rs
@@ -15,6 +15,7 @@
 // current directory is used to find the sources.
 
 use cranelift_codegen_meta as meta;
+use cranelift_codegen_meta::isa::Isa;
 use cranelift_isle::error::Errors;
 
 use std::env;
@@ -28,7 +29,7 @@ fn main() {
     let out_dir = env::var("OUT_DIR").expect("The OUT_DIR environment variable must be set");
     let target_triple = env::var("TARGET").expect("The TARGET environment variable must be set");
 
-    let mut isas = meta::isa::Isa::all()
+    let mut isas = Isa::all()
         .iter()
         .cloned()
         .filter(|isa| {
@@ -82,7 +83,7 @@ fn main() {
     }
 
     if &std::env::var("SKIP_ISLE").unwrap_or("0".to_string()) != "1" {
-        if let Err(err) = build_isle(crate_dir, isle_dir) {
+        if let Err(err) = build_isle(crate_dir, isle_dir, &isas) {
             eprintln!("Error: {}", err);
             process::exit(1);
         }
@@ -175,6 +176,7 @@ struct IsleCompilation {
 fn get_isle_compilations(
     crate_dir: &std::path::Path,
     out_dir: &std::path::Path,
+    isas: &[Isa],
 ) -> Result<IsleCompilations, std::io::Error> {
     let cur_dir = std::env::current_dir()?;
 
@@ -218,122 +220,135 @@ fn get_isle_compilations(
     // N.B.: add any new compilation outputs to
     // `scripts/force-rebuild-isle.sh` if they do not fit the pattern
     // `cranelift/codegen/src/isa/*/lower/isle/generated_code.rs`!
-    Ok(IsleCompilations {
-        items: vec![
-            // The mid-end optimization rules.
-            IsleCompilation {
-                output: out_dir.join("isle_opt.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    prelude_opt_isle,
-                    src_opts.join("arithmetic.isle"),
-                    src_opts.join("bitops.isle"),
-                    src_opts.join("cprop.isle"),
-                    src_opts.join("extends.isle"),
-                    src_opts.join("icmp.isle"),
-                    src_opts.join("remat.isle"),
-                    src_opts.join("selects.isle"),
-                    src_opts.join("shifts.isle"),
-                    src_opts.join("vector.isle"),
-                ],
-                untracked_inputs: vec![clif_opt_isle],
-            },
-            // The x86-64 instruction selector.
-            IsleCompilation {
-                output: out_dir.join("isle_x64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    prelude_lower_isle.clone(),
-                    src_isa_x64.join("inst.isle"),
-                    src_isa_x64.join("lower.isle"),
-                ],
-                untracked_inputs: vec![clif_lower_isle.clone()],
-            },
-            // The x86-64 legalizer.
-            IsleCompilation {
-                output: out_dir.join("legalize_x64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    legalize_shared_isle.clone(),
-                    src_isa_x64.join("legalize.isle"),
-                ],
-                untracked_inputs: vec![clif_legalize_isle.clone()],
-            },
-            // The aarch64 instruction selector.
-            IsleCompilation {
-                output: out_dir.join("isle_aarch64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    prelude_lower_isle.clone(),
-                    src_isa_aarch64.join("inst.isle"),
-                    src_isa_aarch64.join("inst_neon.isle"),
-                    src_isa_aarch64.join("lower.isle"),
-                    src_isa_aarch64.join("lower_dynamic_neon.isle"),
-                ],
-                untracked_inputs: vec![clif_lower_isle.clone()],
-            },
-            // The aarch64 legalizer.
-            IsleCompilation {
-                output: out_dir.join("legalize_aarch64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    legalize_shared_isle.clone(),
-                    // NB: aarch64-specific legalization here when it exists
-                ],
-                untracked_inputs: vec![clif_legalize_isle.clone()],
-            },
-            // The s390x instruction selector.
-            IsleCompilation {
-                output: out_dir.join("isle_s390x.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    prelude_lower_isle.clone(),
-                    src_isa_s390x.join("inst.isle"),
-                    src_isa_s390x.join("lower.isle"),
-                ],
-                untracked_inputs: vec![clif_lower_isle.clone()],
-            },
-            // The s390x legalizer.
-            IsleCompilation {
-                output: out_dir.join("legalize_s390x.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    legalize_shared_isle.clone(),
-                    // NB: s390x-specific legalization here when it exists
-                ],
-                untracked_inputs: vec![clif_legalize_isle.clone()],
-            },
-            // The risc-v instruction selector.
-            IsleCompilation {
-                output: out_dir.join("isle_riscv64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    prelude_lower_isle.clone(),
-                    src_isa_risc_v.join("inst.isle"),
-                    src_isa_risc_v.join("inst_vector.isle"),
-                    src_isa_risc_v.join("lower.isle"),
-                ],
-                untracked_inputs: vec![clif_lower_isle.clone()],
-            },
-            // The risc-v legalizer.
-            IsleCompilation {
-                output: out_dir.join("legalize_riscv64.rs"),
-                inputs: vec![
-                    prelude_isle.clone(),
-                    legalize_shared_isle.clone(),
-                    src_isa_risc_v.join("legalize.isle"),
-                ],
-                untracked_inputs: vec![clif_legalize_isle.clone()],
-            },
-        ],
-    })
+    let mut items = vec![
+        // The mid-end optimization rules.
+        IsleCompilation {
+            output: out_dir.join("isle_opt.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                prelude_opt_isle,
+                src_opts.join("arithmetic.isle"),
+                src_opts.join("bitops.isle"),
+                src_opts.join("cprop.isle"),
+                src_opts.join("extends.isle"),
+                src_opts.join("icmp.isle"),
+                src_opts.join("remat.isle"),
+                src_opts.join("selects.isle"),
+                src_opts.join("shifts.isle"),
+                src_opts.join("vector.isle"),
+            ],
+            untracked_inputs: vec![clif_opt_isle],
+        },
+    ];
+
+    if isas.contains(&Isa::X86) {
+        // The x86-64 instruction selector.
+        items.push(IsleCompilation {
+            output: out_dir.join("isle_x64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                prelude_lower_isle.clone(),
+                src_isa_x64.join("inst.isle"),
+                src_isa_x64.join("lower.isle"),
+            ],
+            untracked_inputs: vec![clif_lower_isle.clone()],
+        });
+        // The x86-64 legalizer.
+        items.push(IsleCompilation {
+            output: out_dir.join("legalize_x64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                legalize_shared_isle.clone(),
+                src_isa_x64.join("legalize.isle"),
+            ],
+            untracked_inputs: vec![clif_legalize_isle.clone()],
+        });
+    }
+
+    if isas.contains(&Isa::Arm64) {
+        // The aarch64 instruction selector.
+        items.push(IsleCompilation {
+            output: out_dir.join("isle_aarch64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                prelude_lower_isle.clone(),
+                src_isa_aarch64.join("inst.isle"),
+                src_isa_aarch64.join("inst_neon.isle"),
+                src_isa_aarch64.join("lower.isle"),
+                src_isa_aarch64.join("lower_dynamic_neon.isle"),
+            ],
+            untracked_inputs: vec![clif_lower_isle.clone()],
+        });
+        // The aarch64 legalizer.
+        items.push(IsleCompilation {
+            output: out_dir.join("legalize_aarch64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                legalize_shared_isle.clone(),
+                // NB: aarch64-specific legalization here when it exists
+            ],
+            untracked_inputs: vec![clif_legalize_isle.clone()],
+        });
+    }
+
+    if isas.contains(&Isa::S390x) {
+        // The s390x instruction selector.
+        items.push(IsleCompilation {
+            output: out_dir.join("isle_s390x.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                prelude_lower_isle.clone(),
+                src_isa_s390x.join("inst.isle"),
+                src_isa_s390x.join("lower.isle"),
+            ],
+            untracked_inputs: vec![clif_lower_isle.clone()],
+        });
+        // The s390x legalizer.
+        items.push(IsleCompilation {
+            output: out_dir.join("legalize_s390x.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                legalize_shared_isle.clone(),
+                // NB: s390x-specific legalization here when it exists
+            ],
+            untracked_inputs: vec![clif_legalize_isle.clone()],
+        });
+    }
+
+    if isas.contains(&Isa::Riscv64) {
+        // The risc-v instruction selector.
+        items.push(IsleCompilation {
+            output: out_dir.join("isle_riscv64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                prelude_lower_isle.clone(),
+                src_isa_risc_v.join("inst.isle"),
+                src_isa_risc_v.join("inst_vector.isle"),
+                src_isa_risc_v.join("lower.isle"),
+            ],
+            untracked_inputs: vec![clif_lower_isle.clone()],
+        });
+        // The risc-v legalizer.
+        items.push(IsleCompilation {
+            output: out_dir.join("legalize_riscv64.rs"),
+            inputs: vec![
+                prelude_isle.clone(),
+                legalize_shared_isle.clone(),
+                src_isa_risc_v.join("legalize.isle"),
+            ],
+            untracked_inputs: vec![clif_legalize_isle.clone()],
+        });
+    }
+
+    Ok(IsleCompilations { items })
 }
 
 fn build_isle(
     crate_dir: &std::path::Path,
     isle_dir: &std::path::Path,
+    isas: &[Isa],
 ) -> Result<(), Box<dyn std::error::Error + 'static>> {
-    let isle_compilations = get_isle_compilations(crate_dir, isle_dir)?;
+    let isle_compilations = get_isle_compilations(crate_dir, isle_dir, isas)?;
 
     let mut had_error = false;
     for compilation in &isle_compilations.items {

From c8ed85e02e8520d5ea8ccd26f26b0240873e543a Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 13 Nov 2023 13:45:53 -0800
Subject: [PATCH 21/21] Unify `ins_*` and `replace_*` constructors

Use the last-inserted instruction as the one to replace all the original
values of the original instruction with.
---
 cranelift/codegen/meta/src/gen_inst.rs        |  38 +-
 cranelift/codegen/src/ir/dfg.rs               |   7 +
 .../codegen/src/isa/riscv64/legalize.isle     |   8 +-
 cranelift/codegen/src/isa/x64/legalize.isle   |   2 +-
 cranelift/codegen/src/legalizer/isle.rs       |  38 +-
 .../src/legalizer/legalize_shared.isle        | 350 +++++++++---------
 .../filetests/filetests/egraph/shifts.clif    |  60 +--
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  21 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  44 +--
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  21 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  44 +--
 ..._guard_yes_spectre_i32_access_0_offset.wat |  45 +--
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  35 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  58 +--
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  49 ++-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  35 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  58 +--
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  24 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  24 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  24 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  24 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  49 ++-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  35 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  45 ++-
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  49 ++-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  35 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  45 ++-
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  20 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  38 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  20 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  38 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  47 ++-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  31 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  51 +--
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  44 +--
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  31 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  51 +--
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  18 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  18 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  18 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  18 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  44 +--
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  42 ++-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  45 +--
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  44 +--
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  42 ++-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  45 +--
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  25 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  25 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  28 +-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  40 +-
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  28 +-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  40 +-
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  24 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  24 +-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  24 +-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  24 +-
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  14 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  14 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  20 +-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  18 +-
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  20 +-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  18 +-
 ...rd_no_spectre_i32_access_0x1000_offset.wat |  14 +-
 ...ard_no_spectre_i8_access_0x1000_offset.wat |  14 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  20 +-
 ...d_yes_spectre_i32_access_0x1000_offset.wat |  18 +-
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  20 +-
 ...rd_yes_spectre_i8_access_0x1000_offset.wat |  18 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  36 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  36 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  36 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  44 +--
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  30 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  44 +--
 ...o_spectre_i32_access_0xffff0000_offset.wat |  26 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  26 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  30 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  40 +-
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  30 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  40 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  30 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  30 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  32 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  18 +-
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  28 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  18 +-
 ...o_spectre_i32_access_0xffff0000_offset.wat |  22 +-
 ...no_spectre_i8_access_0xffff0000_offset.wat |  22 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  28 +-
 ...s_spectre_i32_access_0xffff0000_offset.wat |  28 +-
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  28 +-
 ...es_spectre_i8_access_0xffff0000_offset.wat |  28 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  28 +-
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  28 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  18 +-
 ...0_guard_yes_spectre_i8_access_0_offset.wat |  18 +-
 ..._guard_yes_spectre_i32_access_0_offset.wat |  18 +-
 ...f_guard_yes_spectre_i8_access_0_offset.wat |  18 +-
 .../legalizer/back-to-back-legalize.clif      |   8 +-
 .../filetests/legalizer/bxor_imm.clif         |   4 +-
 .../legalizer/canonicalize-nans.clif          |  36 +-
 .../wasm/duplicate-loads-dynamic-memory.wat   |  28 +-
 .../wasm/duplicate-loads-static-memory.wat    |  14 +-
 ...re-access-same-index-different-offsets.wat |  46 ++-
 ...re-access-same-index-different-offsets.wat |  36 +-
 106 files changed, 1820 insertions(+), 1672 deletions(-)

diff --git a/cranelift/codegen/meta/src/gen_inst.rs b/cranelift/codegen/meta/src/gen_inst.rs
index 69ccb2257790..e7f7019dc07e 100644
--- a/cranelift/codegen/meta/src/gen_inst.rs
+++ b/cranelift/codegen/meta/src/gen_inst.rs
@@ -1232,7 +1232,7 @@ impl IsleTarget {
         matches!(self, IsleTarget::Opt)
     }
     fn match_type(&self) -> bool {
-        matches!(self, IsleTarget::Opt)
+        matches!(self, IsleTarget::Opt | IsleTarget::Legalize)
     }
 }
 
@@ -1585,14 +1585,13 @@ fn gen_common_isle(
             // which delegates to a `make_inst` helper. Note that this
             // constructor has the same signature as the extractor above, so
             // don't give it a prefix.
-            IsleTarget::Opt => gen_insn_constructor(fmt, inst, "", "make_inst", "Value"),
+            IsleTarget::Opt => gen_insn_constructor(fmt, inst, "make_inst"),
 
             // During legalization new instructions are either inserted just
             // before the current instruction with `ins_*` or they're replacing
             // the current instruction with `replace_*`.
             IsleTarget::Legalize => {
-                gen_insn_constructor(fmt, inst, "ins_", "ins", "Inst");
-                gen_insn_constructor(fmt, inst, "replace_", "replace", "Inst");
+                gen_insn_constructor(fmt, inst, "ins");
             }
         }
 
@@ -1620,44 +1619,17 @@ fn gen_isle_enum(name: &str, mut variants: Vec<&str>, fmt: &mut Formatter) {
     fmt.empty_line();
 }
 
-fn gen_insn_constructor(
-    fmt: &mut Formatter,
-    inst: &Instruction,
-    prefix: &str,
-    common_ctor: &str,
-    ret_ty: &str,
-) {
+fn gen_insn_constructor(fmt: &mut Formatter, inst: &Instruction, common_ctor: &str) {
     if inst.format.has_value_list {
         return;
     }
-    // Generate a constructor if this is the mid-end prelude.
-    let input_tys = inst
-        .operands_in
-        .iter()
-        .map(|o| {
-            let ty = o.kind.rust_type;
-            if ty == "&[Value]" {
-                "ValueSlice"
-            } else {
-                ty.rsplit("::").next().unwrap()
-            }
-        })
-        .collect::<Vec<_>>()
-        .join(" ");
-    if prefix != "" {
-        fmtln!(
-            fmt,
-            "(decl {prefix}{} (Type {input_tys}) {ret_ty})",
-            inst.name
-        );
-    }
     let input_args = inst
         .operands_in
         .iter()
         .map(|o| o.name)
         .collect::<Vec<_>>()
         .join(" ");
-    fmtln!(fmt, "(rule ({prefix}{} ty {input_args})", inst.name);
+    fmtln!(fmt, "(rule ({} ty {input_args})", inst.name);
     fmt.indent(|fmt| {
         let mut s = format!(
             "({common_ctor} ty (InstructionData.{} (Opcode.{})",
diff --git a/cranelift/codegen/src/ir/dfg.rs b/cranelift/codegen/src/ir/dfg.rs
index feb7466c3f37..f806fea55c04 100644
--- a/cranelift/codegen/src/ir/dfg.rs
+++ b/cranelift/codegen/src/ir/dfg.rs
@@ -459,6 +459,13 @@ impl DataFlowGraph {
             debug_assert_ne!(ty, types::INVALID);
 
             self.values[dest] = ValueData::Alias { ty, original }.into();
+
+            if let Some(fact) = &self.facts[dest] {
+                if self.facts[original].is_none() {
+                    let fact = fact.clone();
+                    self.facts[original] = Some(fact);
+                }
+            }
         }
 
         self.clear_results(dest_inst);
diff --git a/cranelift/codegen/src/isa/riscv64/legalize.isle b/cranelift/codegen/src/isa/riscv64/legalize.isle
index 0011a053f07d..df9a92e8c074 100644
--- a/cranelift/codegen/src/isa/riscv64/legalize.isle
+++ b/cranelift/codegen/src/isa/riscv64/legalize.isle
@@ -1,7 +1,7 @@
 ;; ========= `{u,s}extend` ====================================================
 
-(rule (legalize (has_type $I128 (uextend x)))
-  (replace_uextend_i128 x))
+(rule (legalize (uextend $I128 x))
+  (uextend_i128 x))
 
-(rule (legalize (has_type $I128 (sextend x)))
-  (replace_sextend_i128 x))
+(rule (legalize (sextend $I128 x))
+  (sextend_i128 x))
diff --git a/cranelift/codegen/src/isa/x64/legalize.isle b/cranelift/codegen/src/isa/x64/legalize.isle
index d824b5392bce..5091001351fa 100644
--- a/cranelift/codegen/src/isa/x64/legalize.isle
+++ b/cranelift/codegen/src/isa/x64/legalize.isle
@@ -1,6 +1,6 @@
 ;; ========= `fcvt_from_uint` =================================================
 
-(rule (legalize (has_type ty (fcvt_from_uint v @ (value_type $I64))))
+(rule (legalize (fcvt_from_uint ty v @ (value_type $I64)))
   (x64_replace_fcvt_from_u64 ty v))
 
 (decl x64_replace_fcvt_from_u64 (Type Value) CursorPosition)
diff --git a/cranelift/codegen/src/legalizer/isle.rs b/cranelift/codegen/src/legalizer/isle.rs
index a0b5b5cea2bc..1c32f06f3354 100644
--- a/cranelift/codegen/src/legalizer/isle.rs
+++ b/cranelift/codegen/src/legalizer/isle.rs
@@ -19,6 +19,7 @@ pub fn run<T>(
         pos,
         cfg,
         replace: None,
+        last_ins: None,
     };
     let func_begin = cx.pos.position();
     cx.pos.set_position(func_begin);
@@ -27,13 +28,19 @@ pub fn run<T>(
         while let Some(inst) = cx.pos.next_inst() {
             trace!("legalizing {}", cx.pos.func.dfg.display_inst(inst));
             cx.replace = Some(inst);
+            cx.last_ins = None;
             match constructor_legalize(&mut cx, inst) {
                 Some(pos) => {
                     trace!("moving to {pos:?}");
+                    if let Some(ins) = cx.last_ins.take() {
+                        cx.pos.func.dfg.replace_with_aliases(inst, ins);
+                        cx.pos.remove_inst();
+                    }
                     cx.pos.set_position(pos);
                 }
                 None => {
                     trace!("fallthrough");
+                    debug_assert!(cx.last_ins.is_none());
                     cx.prev_position = cx.pos.position();
                 }
             }
@@ -45,7 +52,16 @@ pub struct LegalizeContext<'a, T> {
     pub backend: &'a T,
     pub pos: FuncCursor<'a>,
     pub cfg: &'a mut ControlFlowGraph,
+    /// Instruction that is currently being legalized by this context and will
+    /// be replaced if an instruction is inserted.
     pub replace: Option<Inst>,
+    /// Last-inserted instruction.
+    ///
+    /// If this is set then all the values of `replace` will be turned into
+    /// aliases to the results of this instruction.
+    pub last_ins: Option<Inst>,
+    /// The previous position before this context started legalizing, used to
+    /// rewind backwards and re-legalize instructions that were just added.
     pub prev_position: CursorPosition,
 }
 
@@ -55,8 +71,9 @@ macro_rules! isle_common_legalizer_methods {
     () => {
         crate::isle_common_prelude_methods!();
 
-        fn inst_data(&mut self, inst: Inst) -> InstructionData {
-            self.pos.func.dfg.insts[inst]
+        fn inst_data(&mut self, inst: Inst) -> (Type, InstructionData) {
+            let ty = self.pos.func.dfg.ctrl_typevar(inst);
+            (ty, self.pos.func.dfg.insts[inst])
         }
 
         fn gv_data(&mut self, gv: GlobalValue) -> GlobalValueData {
@@ -66,13 +83,7 @@ macro_rules! isle_common_legalizer_methods {
         fn ins(&mut self, ty: Type, data: &InstructionData) -> Inst {
             let ret = self.pos.ins().build(data.clone(), ty).0;
             crate::trace!("ins {}", self.pos.func.dfg.display_inst(ret));
-            ret
-        }
-
-        fn replace(&mut self, ty: Type, data: &InstructionData) -> Inst {
-            let ins = self.pos.func.dfg.replace(self.replace.unwrap());
-            let ret = ins.build(data.clone(), ty).0;
-            crate::trace!("replace {}", self.pos.func.dfg.display_inst(ret));
+            self.last_ins = Some(ret);
             ret
         }
 
@@ -179,7 +190,7 @@ macro_rules! isle_common_legalizer_methods {
             self.pos.func.dfg.facts[constant] = Some(Fact::constant(bits, unsigned_offset));
         }
 
-        fn replace_vmctx_addr(&mut self, global_value: ir::GlobalValue) -> CursorPosition {
+        fn vmctx_addr(&mut self, global_value: ir::GlobalValue) -> CursorPosition {
             // Get the value representing the `vmctx` argument.
             let vmctx = self
                 .pos
@@ -232,13 +243,6 @@ macro_rules! isle_common_legalizer_methods {
                 .enable_table_access_spectre_mitigation()
         }
 
-        fn replace_with_aliases(&mut self, val: Value) -> Inst {
-            let new_inst = self.pos.func.dfg.value_def(val).inst().unwrap();
-            let inst = self.replace.unwrap();
-            self.pos.func.dfg.replace_with_aliases(inst, new_inst);
-            crate::cursor::Cursor::remove_inst(&mut self.pos)
-        }
-
         fn enable_nan_canonicalization(&mut self) -> bool {
             self.backend.flags().enable_nan_canonicalization()
         }
diff --git a/cranelift/codegen/src/legalizer/legalize_shared.isle b/cranelift/codegen/src/legalizer/legalize_shared.isle
index a84adbcc256a..5fdebd9e5d3f 100644
--- a/cranelift/codegen/src/legalizer/legalize_shared.isle
+++ b/cranelift/codegen/src/legalizer/legalize_shared.isle
@@ -30,25 +30,16 @@
 
 ;; Helper used in generated extractors for each CLIF instruction to extract
 ;; information from an `Inst`.
-(decl inst_data (InstructionData) Inst)
+(decl inst_data (Type InstructionData) Inst)
 (extern extractor infallible inst_data inst_data)
 
 ;; Helper used to perform an `ins()` operation which inserts just before the
 ;; current instruction being legalized.
 ;;
-;; This is invoked by generated constructors for each CLIF instruction with the
-;; `ins_*` prefix.
+;; This is invoked by generated constructors for each CLIF instruction.
 (decl ins (Type InstructionData) Inst)
 (extern constructor ins ins)
 
-;; Helper used to perform a `replace()` operation which replaces the current
-;; instruction being legalized.
-;;
-;; This is invoked by generated constructors for each CLIF instruction with the
-;; `replace_*` prefix.
-(decl replace (Type InstructionData) Inst)
-(extern constructor replace replace)
-
 ;; Constructors for value arrays.
 (decl value_array_2_ctor (Value Value) ValueArray2)
 (extern constructor value_array_2_ctor value_array_2_ctor)
@@ -87,7 +78,7 @@
 ;; Helper to generate an `iconst` value which has type `I64` and the provided
 ;; value.
 (decl ins_i64 (i64) Value)
-(rule (ins_i64 val) (ins_iconst $I64 (imm64 (i64_as_u64 val))))
+(rule (ins_i64 val) (iconst $I64 (imm64 (i64_as_u64 val))))
 
 ;; Optionally "legalize" an instruction by replacing the current instruction or
 ;; inserting new instructions/blocks.
@@ -110,129 +101,129 @@
 ;; ========= `{u,s}extend` ====================================================
 
 ;; Helper to replace a `uextend` to i128 with an `iconcat` instruction instead.
-(decl replace_uextend_i128 (Value) Inst)
-(rule 0 (replace_uextend_i128 x @ (value_type (fits_in_32 _)))
-  (replace_iconcat $I64 (ins_uextend $I64 x) (ins_i64 0)))
-(rule 1 (replace_uextend_i128 x @ (value_type $I64))
-  (replace_iconcat $I64 x (ins_i64 0)))
+(decl uextend_i128 (Value) Inst)
+(rule 0 (uextend_i128 x @ (value_type (fits_in_32 _)))
+  (iconcat $I64 (uextend $I64 x) (ins_i64 0)))
+(rule 1 (uextend_i128 x @ (value_type $I64))
+  (iconcat $I64 x (ins_i64 0)))
 
 ;; Helper to replace a `sextend` to i128 with an `iconcat` instruction instead.
-(decl replace_sextend_i128 (Value) Inst)
-(rule 0 (replace_sextend_i128 x @ (value_type (fits_in_32 _)))
-  (let ((val64 Value (ins_sextend $I64 x)))
-    (replace_iconcat $I64 val64 (ins_sshr $I64 val64 (ins_i64 63)))))
-(rule 1 (replace_sextend_i128 x @ (value_type $I64))
-  (replace_iconcat $I64 x (ins_sshr $I64 x (ins_i64 63))))
+(decl sextend_i128 (Value) Inst)
+(rule 0 (sextend_i128 x @ (value_type (fits_in_32 _)))
+  (let ((val64 Value (sextend $I64 x)))
+    (iconcat $I64 val64 (sshr $I64 val64 (ins_i64 63)))))
+(rule 1 (sextend_i128 x @ (value_type $I64))
+  (iconcat $I64 x (sshr $I64 x (ins_i64 63))))
 
 ;; ========= `{dynamic_,}stack_{load,store}` ==================================
 
 ;; stack_load(slot, offset) => load(stack_addr(slot, offset), 0)
-(rule (legalize (has_type ty (stack_load slot offset)))
-  (let ((addr Value (ins_stack_addr $I64 slot offset)))
-    (replace_load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
+(rule (legalize (stack_load ty slot offset))
+  (let ((addr Value (stack_addr $I64 slot offset)))
+    (load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
 
 ;; stack_store(val, slot, offset) => store(val, stack_addr(slot, offset), 0)
-(rule (legalize (stack_store val @ (value_type ty) slot offset))
-  (let ((addr Value (ins_stack_addr $I64 slot offset)))
-    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
+(rule (legalize (stack_store ty val slot offset))
+  (let ((addr Value (stack_addr $I64 slot offset)))
+    (store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
 
 ;; dynamic_stack_load(slot) => load(dynamic_stack_addr(slot), 0)
-(rule (legalize (has_type ty (dynamic_stack_load slot)))
-  (let ((addr Value (ins_dynamic_stack_addr $I64 slot)))
-    (replace_load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
+(rule (legalize (dynamic_stack_load ty slot))
+  (let ((addr Value (dynamic_stack_addr $I64 slot)))
+    (load ty (mem_flags_notrap) addr (i32_to_offset32 0))))
 
 ;; dynamic_stack_store(val, slot) => store(val, dynamic_stack_addr(slot), 0)
-(rule (legalize (dynamic_stack_store val @ (value_type ty) slot))
-  (let ((addr Value (ins_dynamic_stack_addr $I64 slot)))
-    (replace_store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
+(rule (legalize (dynamic_stack_store ty val slot))
+  (let ((addr Value (dynamic_stack_addr $I64 slot)))
+    (store ty (mem_flags_notrap) val addr (i32_to_offset32 0))))
 
 ;; ========= `trapz`, `trapnz`, `resumable_trapnz` ============================
 
 ;; Delegate back to Rust to perform the basic-block manipulation necessary to
 ;; expand these instructions
 
-(rule (legalize (trapz val code)) (expand_trapz val code))
+(rule (legalize (trapz _ val code)) (expand_trapz val code))
 (decl expand_trapz (Value TrapCode) CursorPosition)
 (extern constructor expand_trapz expand_trapz)
 
-(rule (legalize (trapnz val code)) (expand_trapnz val code))
+(rule (legalize (trapnz _ val code)) (expand_trapnz val code))
 (decl expand_trapnz (Value TrapCode) CursorPosition)
 (extern constructor expand_trapnz expand_trapnz)
 
-(rule (legalize (resumable_trapnz val code)) (expand_resumable_trapnz val code))
+(rule (legalize (resumable_trapnz _ val code)) (expand_resumable_trapnz val code))
 (decl expand_resumable_trapnz (Value TrapCode) CursorPosition)
 (extern constructor expand_resumable_trapnz expand_resumable_trapnz)
 
 ;; ========= `*_imm` instrs ===================================================
 
-(rule (legalize (has_type ty (band_imm val imm)))
-  (replace_band ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (band_imm ty val imm))
+  (band ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (bor_imm val imm)))
-  (replace_bor ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (bor_imm ty val imm))
+  (bor ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (bxor_imm val imm)))
-  (replace_bxor ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (bxor_imm ty val imm))
+  (bxor ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (ishl_imm val imm)))
-  (replace_ishl ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (ishl_imm ty val imm))
+  (ishl ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (sshr_imm val imm)))
-  (replace_sshr ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (sshr_imm ty val imm))
+  (sshr ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (ushr_imm val imm)))
-  (replace_ushr ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (ushr_imm ty val imm))
+  (ushr ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (rotl_imm val imm)))
-  (replace_rotl ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (rotl_imm ty val imm))
+  (rotl ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (rotr_imm val imm)))
-  (replace_rotr ty val (ins_unsigned_const (lane_type ty) imm)))
+(rule (legalize (rotr_imm ty val imm))
+  (rotr ty val (unsigned_const (lane_type ty) imm)))
 
-(rule (legalize (has_type ty (iadd_imm val imm)))
-  (replace_iadd ty val (ins_signed_const (lane_type ty) imm)))
+(rule (legalize (iadd_imm ty val imm))
+  (iadd ty val (signed_const (lane_type ty) imm)))
 
 ;; note the reversed arguments in the `isub`
-(rule (legalize (has_type ty (irsub_imm val imm)))
-  (replace_isub ty (ins_signed_const (lane_type ty) imm) val))
-
-(rule (legalize (has_type ty (imul_imm val imm)))
-  (replace_imul ty val (ins_signed_const (lane_type ty) imm)))
-
-(rule (legalize (has_type ty (sdiv_imm val imm)))
-  (replace_sdiv ty val (ins_signed_const (lane_type ty) imm)))
-
-(rule (legalize (has_type ty (srem_imm val imm)))
-  (replace_srem ty val (ins_signed_const (lane_type ty) imm)))
-
-(rule (legalize (has_type ty (udiv_imm val imm)))
-  (replace_udiv ty val (ins_signed_const (lane_type ty) imm)))
-
-(rule (legalize (has_type ty (urem_imm val imm)))
-  (replace_urem ty val (ins_signed_const (lane_type ty) imm)))
-
-(rule (legalize (icmp_imm cc val @ (value_type ty) imm))
-  (replace_icmp ty cc val (ins_signed_const (lane_type ty) imm)))
-
-(decl ins_unsigned_const (Type Imm64) Value)
-(rule (ins_unsigned_const $I8 (u64_from_imm64 n))
-  (ins_iconst $I8 (imm64 (u64_and n 0xff))))
-(rule (ins_unsigned_const $I16 (u64_from_imm64 n))
-  (ins_iconst $I16 (imm64 (u64_and n 0xffff))))
-(rule (ins_unsigned_const $I32 (u64_from_imm64 n))
-  (ins_iconst $I32 (imm64 (u64_and n 0xffffffff))))
-(rule (ins_unsigned_const $I64 imm)
-  (ins_iconst $I64 imm))
-(rule (ins_unsigned_const $I128 imm)
-  (ins_iconcat $I64 (ins_iconst $I64 imm) (ins_i64 0)))
-
-(decl ins_signed_const (Type Imm64) Value)
-(rule 0 (ins_signed_const (fits_in_64 ty) imm)
-  (ins_unsigned_const ty imm))
-(rule 1 (ins_signed_const $I128 imm)
-  (ins_iconcat $I64
-    (ins_iconst $I64 imm)
-    (ins_iconst $I64 (imm64_sshr $I64 imm (imm64 63)))))
+(rule (legalize (irsub_imm ty val imm))
+  (isub ty (signed_const (lane_type ty) imm) val))
+
+(rule (legalize (imul_imm ty val imm))
+  (imul ty val (signed_const (lane_type ty) imm)))
+
+(rule (legalize (sdiv_imm ty val imm))
+  (sdiv ty val (signed_const (lane_type ty) imm)))
+
+(rule (legalize (srem_imm ty val imm))
+  (srem ty val (signed_const (lane_type ty) imm)))
+
+(rule (legalize (udiv_imm ty val imm))
+  (udiv ty val (signed_const (lane_type ty) imm)))
+
+(rule (legalize (urem_imm ty val imm))
+  (urem ty val (signed_const (lane_type ty) imm)))
+
+(rule (legalize (icmp_imm ty cc val imm))
+  (icmp ty cc val (signed_const (lane_type ty) imm)))
+
+(decl unsigned_const (Type Imm64) Value)
+(rule (unsigned_const $I8 (u64_from_imm64 n))
+  (iconst $I8 (imm64 (u64_and n 0xff))))
+(rule (unsigned_const $I16 (u64_from_imm64 n))
+  (iconst $I16 (imm64 (u64_and n 0xffff))))
+(rule (unsigned_const $I32 (u64_from_imm64 n))
+  (iconst $I32 (imm64 (u64_and n 0xffffffff))))
+(rule (unsigned_const $I64 imm)
+  (iconst $I64 imm))
+(rule (unsigned_const $I128 imm)
+  (iconcat $I64 (iconst $I64 imm) (ins_i64 0)))
+
+(decl signed_const (Type Imm64) Value)
+(rule 0 (signed_const (fits_in_64 ty) imm)
+  (unsigned_const ty imm))
+(rule 1 (signed_const $I128 imm)
+  (iconcat $I64
+    (iconst $I64 imm)
+    (iconst $I64 (imm64_sshr $I64 imm (imm64 63)))))
 
 ;; ========= `*_not` instrs ===================================================
 
@@ -241,45 +232,45 @@
 ;; pattern match this sequence regardless when architectures
 ;; support the instruction natively.
 
-(rule (legalize (has_type ty (band_not a b)))
-  (replace_band ty a (ins_bnot ty b)))
+(rule (legalize (band_not ty a b))
+  (band ty a (bnot ty b)))
 
-(rule (legalize (has_type ty (bor_not a b)))
-  (replace_bor ty a (ins_bnot ty b)))
+(rule (legalize (bor_not ty a b))
+  (bor ty a (bnot ty b)))
 
-(rule (legalize (has_type ty (bxor_not a b)))
-  (replace_bxor ty a (ins_bnot ty b)))
+(rule (legalize (bxor_not ty a b))
+  (bxor ty a (bnot ty b)))
 
 ;; ========= `global_value` ===================================================
 
-(rule (legalize (global_value gv @ (gv_vmcontext)))
-  (replace_vmctx_addr gv))
-(decl replace_vmctx_addr (GlobalValue) CursorPosition)
-(extern constructor replace_vmctx_addr replace_vmctx_addr)
+(rule (legalize (global_value _ gv @ (gv_vmcontext)))
+  (vmctx_addr gv))
+(decl vmctx_addr (GlobalValue) CursorPosition)
+(extern constructor vmctx_addr vmctx_addr)
 
-(rule (legalize (global_value (gv_load base offset ty flags)))
-  (let ((base Value (ins_global_value_copy_facts (pointer_type) base))
-        (_ Inst (replace_load ty flags base offset)))
+(rule (legalize (global_value _ (gv_load base offset ty flags)))
+  (let ((base Value (global_value_copy_facts (pointer_type) base))
+        (_ Inst (load ty flags base offset)))
     ;; re-legalize the previous instrs to legalize the new `global_value` node
     (prev_position)))
 
-(rule (legalize (global_value (gv_iadd_imm gv_base offset ty)))
-  (let ((base Value (ins_global_value_copy_facts ty gv_base))
-        (offset Value (ins_iconst_with_fact ty offset gv_base))
-        (_ Inst (replace_iadd ty base offset)))
+(rule (legalize (global_value _ (gv_iadd_imm gv_base offset ty)))
+  (let ((base Value (global_value_copy_facts ty gv_base))
+        (offset Value (iconst_with_fact ty offset gv_base))
+        (_ Inst (iadd ty base offset)))
     ;; re-legalize the previous instrs to legalize the new `global_value` node
     (prev_position)))
 
-(rule (legalize (global_value (gv_const_vector_scale ty)))
-  (replace_iconst (pointer_type) (imm64 (const_vector_scale ty))))
+(rule (legalize (global_value _ (gv_const_vector_scale ty)))
+  (iconst (pointer_type) (imm64 (const_vector_scale ty))))
 (decl const_vector_scale (Type) u64)
 (extern constructor const_vector_scale const_vector_scale)
 
-(rule (legalize (global_value gv @ (gv_symbol _ _)))
-  (replace_symbol_value (pointer_type) gv))
+(rule (legalize (global_value _ gv @ (gv_symbol _ _)))
+  (symbol_value (pointer_type) gv))
 
-(rule (legalize (global_value gv @ (gv_tls_symbol _ _)))
-  (replace_tls_value (pointer_type) gv))
+(rule (legalize (global_value _ gv @ (gv_tls_symbol _ _)))
+  (tls_value (pointer_type) gv))
 
 (type GlobalValueData extern (enum
   (VMContext)
@@ -325,17 +316,17 @@
 (extractor (gv_const_vector_scale ty)
   (gv_data (GlobalValueData.DynScaleTargetConst ty)))
 
-;; This is the dual of the `ins_global_value` constructor except that this one
+;; This is the dual of the `global_value` constructor except that this one
 ;; additionally copies PCC facts from the `GlobalValue` onto the returned
 ;; `Inst`.
-(decl ins_global_value_copy_facts (Type GlobalValue) Inst)
-(rule (ins_global_value_copy_facts ty gv)
-  (let ((ret Inst (ins_global_value ty gv))
+(decl global_value_copy_facts (Type GlobalValue) Inst)
+(rule (global_value_copy_facts ty gv)
+  (let ((ret Inst (global_value ty gv))
         (_ Unit (update_inst_facts_with_gv gv ret)))
     ret))
-(decl ins_iconst_with_fact (Type Imm64 GlobalValue) Inst)
-(rule (ins_iconst_with_fact ty imm gv)
-  (let ((ret Inst (ins_iconst ty imm))
+(decl iconst_with_fact (Type Imm64 GlobalValue) Inst)
+(rule (iconst_with_fact ty imm gv)
+  (let ((ret Inst (iconst ty imm))
         (_ Unit (update_const_facts ty imm gv ret)))
     ret))
 (decl update_inst_facts_with_gv (GlobalValue Value) Unit)
@@ -345,28 +336,28 @@
 
 ;; ========= `table_addr` =====================================================
 
-(rule (legalize (has_type addr_ty (table_addr table idx @ (value_type idx_ty) offset)))
+(rule (legalize (table_addr addr_ty table idx @ (value_type idx_ty) offset))
   (let (
     ;; Start with the bounds check. Trap if `index + 1 > bound`.
     ;; `index > bound - 1` is the same as `index >= bound`.
-    (bound Inst (ins_global_value_copy_facts idx_ty (table_bound_gv table)))
-    (cmp Inst (ins_icmp $I8 (IntCC.UnsignedGreaterThanOrEqual) idx bound))
-    (_ Inst (ins_trapnz $I8 cmp (TrapCode.TableOutOfBounds)))
+    (bound Inst (global_value_copy_facts idx_ty (table_bound_gv table)))
+    (cmp Inst (icmp $I8 (IntCC.UnsignedGreaterThanOrEqual) idx bound))
+    (_ Inst (trapnz $I8 cmp (TrapCode.TableOutOfBounds)))
 
     ;; Calculate the offset, in bytes, by multiplying `idx` by the byte size
     ;; of the table element.
     (idx Value (extend_index_to_addr idx addr_ty))
-    (idx_offset Value (ins_imul_imm addr_ty idx (imm64 (table_element_size table))))
+    (mul_imm Value (iconst addr_ty (imm64 (table_element_size table))))
+    (idx_offset Value (imul addr_ty idx mul_imm))
 
     ;; Calculate the final address by adding the table base address to the
     ;; element offset to the static offset in the `table_addr` instruction.
-    (base Value (ins_global_value_copy_facts addr_ty (table_base_gv table)))
-    (offset Value (ins_iconst addr_ty (imm64 (i64_as_u64 (offset32_to_i32 offset)))))
-    (elem_addr Value (ins_iadd addr_ty (ins_iadd addr_ty base idx_offset) offset))
+    (base Value (global_value_copy_facts addr_ty (table_base_gv table)))
+    (offset Value (iconst addr_ty (imm64 (i64_as_u64 (offset32_to_i32 offset)))))
+    (elem_addr Value (iadd addr_ty (iadd addr_ty base idx_offset) offset))
 
     ;; Generate the final `result` and replace the current instruction with that.
-    (result Value (select_table_addr cmp base elem_addr))
-    (_ Inst (replace_with_aliases result))
+    (_ Value (select_table_addr cmp base elem_addr))
   )
   ;; re-legalize the above instructions
   (prev_position)))
@@ -374,13 +365,13 @@
 (decl select_table_addr (Value Value Value) Value)
 (rule (select_table_addr cmp base @ (value_type ty) element_addr)
   (if-let $true (enable_table_access_spectre_mitigation))
-  (ins_select_spectre_guard ty cmp base element_addr))
+  (select_spectre_guard ty cmp base element_addr))
 (rule (select_table_addr cmp base @ (value_type ty) element_addr)
   (if-let $false (enable_table_access_spectre_mitigation))
   element_addr)
 
 (decl extend_index_to_addr (Value Type) Value)
-(rule 0 (extend_index_to_addr val ty) (ins_uextend ty val))
+(rule 0 (extend_index_to_addr val ty) (uextend ty val))
 (rule 1 (extend_index_to_addr val @ (value_type ty) ty) val)
 
 (decl table_bound_gv (Table) GlobalValue)
@@ -392,74 +383,71 @@
 (decl pure enable_table_access_spectre_mitigation () bool)
 (extern constructor enable_table_access_spectre_mitigation enable_table_access_spectre_mitigation)
 
-(decl replace_with_aliases (Value) Inst)
-(extern constructor replace_with_aliases replace_with_aliases)
-
 ;; ========= NaN canonicalization =============================================
 
-(rule (legalize (has_type ty (ceil val)))
+(rule (legalize (ceil ty val))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_ceil ty val)))
+  (canonicalized ty (ceil ty val)))
 
-(rule (legalize (has_type ty (floor val)))
+(rule (legalize (floor ty val))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_floor ty val)))
+  (canonicalized ty (floor ty val)))
 
-(rule (legalize (has_type ty (nearest val)))
+(rule (legalize (nearest ty val))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_nearest ty val)))
+  (canonicalized ty (nearest ty val)))
 
-(rule (legalize (has_type ty (trunc val)))
+(rule (legalize (trunc ty val))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_trunc ty val)))
+  (canonicalized ty (trunc ty val)))
 
-(rule (legalize (has_type ty (sqrt val)))
+(rule (legalize (sqrt ty val))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_sqrt ty val)))
+  (canonicalized ty (sqrt ty val)))
 
-(rule (legalize (has_type ty (fadd a b)))
+(rule (legalize (fadd ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fadd ty a b)))
+  (canonicalized ty (fadd ty a b)))
 
-(rule (legalize (has_type ty (fsub a b)))
+(rule (legalize (fsub ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fsub ty a b)))
+  (canonicalized ty (fsub ty a b)))
 
-(rule (legalize (has_type ty (fmul a b)))
+(rule (legalize (fmul ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fmul ty a b)))
+  (canonicalized ty (fmul ty a b)))
 
-(rule (legalize (has_type ty (fdiv a b)))
+(rule (legalize (fdiv ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fdiv ty a b)))
+  (canonicalized ty (fdiv ty a b)))
 
-(rule (legalize (has_type ty (fmin a b)))
+(rule (legalize (fmin ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fmin ty a b)))
+  (canonicalized ty (fmin ty a b)))
 
-(rule (legalize (has_type ty (fmax a b)))
+(rule (legalize (fmax ty a b))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fmax ty a b)))
+  (canonicalized ty (fmax ty a b)))
 
-(rule (legalize (has_type ty (fma a b c)))
+(rule (legalize (fma ty a b c))
   (if-let $true (enable_nan_canonicalization))
-  (replace_canonicalized ty (ins_fma ty a b c)))
-
-(decl replace_canonicalized (Type Value) Inst)
-(rule (replace_canonicalized $F32 val)
-  (replace_select $F32 (ins_fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan32) val))
-(rule (replace_canonicalized $F64 val)
-  (replace_select $F64 (ins_fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan64) val))
-(rule (replace_canonicalized $F32X4 val)
-  (let ((cmp Value (ins_fcmp $I32X4 (FloatCC.NotEqual) val val))
-        (canon Value (ins_splat $F32X4 (canon_nan32)))
-        (mask Value (ins_bitcast $F32X4 (mem_flags_empty) cmp)))
-    (replace_bitselect $F32X4 mask canon val)))
-(rule (replace_canonicalized $F64X2 val)
-  (let ((canon Value (ins_splat $F64X2 (canon_nan64)))
-        (cmp Value (ins_fcmp $I64X2 (FloatCC.NotEqual) val val))
-        (mask Value (ins_bitcast $F64X2 (mem_flags_empty) cmp)))
-    (replace_bitselect $F64X2 mask canon val)))
+  (canonicalized ty (fma ty a b c)))
+
+(decl canonicalized (Type Value) Inst)
+(rule (canonicalized $F32 val)
+  (select $F32 (fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan32) val))
+(rule (canonicalized $F64 val)
+  (select $F64 (fcmp $I8 (FloatCC.NotEqual) val val) (canon_nan64) val))
+(rule (canonicalized $F32X4 val)
+  (let ((cmp Value (fcmp $I32X4 (FloatCC.NotEqual) val val))
+        (canon Value (splat $F32X4 (canon_nan32)))
+        (mask Value (bitcast $F32X4 (mem_flags_empty) cmp)))
+    (bitselect $F32X4 mask canon val)))
+(rule (canonicalized $F64X2 val)
+  (let ((canon Value (splat $F64X2 (canon_nan64)))
+        (cmp Value (fcmp $I64X2 (FloatCC.NotEqual) val val))
+        (mask Value (bitcast $F64X2 (mem_flags_empty) cmp)))
+    (bitselect $F64X2 mask canon val)))
 
 (decl pure enable_nan_canonicalization () bool)
 (extern constructor enable_nan_canonicalization enable_nan_canonicalization)
diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif
index f2b992771bde..20733f0bd121 100644
--- a/cranelift/filetests/filetests/egraph/shifts.clif
+++ b/cranelift/filetests/filetests/egraph/shifts.clif
@@ -221,9 +221,9 @@ block0(v0: i32):
     v1 = ishl_imm v0, 24
     v2 = sshr_imm v1, 24
     return v2
-    ; check: v5 = ireduce.i8 v0
-    ; check: v6 = sextend.i32 v5
-    ; check: return v6
+    ; check: v7 = ireduce.i8 v0
+    ; check: v8 = sextend.i32 v7
+    ; check: return v8
 }
 
 function %i32_shl_sshr_16_to_ireduce(i32) -> i32 {
@@ -231,9 +231,9 @@ block0(v0: i32):
     v1 = ishl_imm v0, 16
     v2 = sshr_imm v1, 16
     return v2
-    ; check: v5 = ireduce.i16 v0
-    ; check: v6 = sextend.i32 v5
-    ; check: return v6
+    ; check: v7 = ireduce.i16 v0
+    ; check: v8 = sextend.i32 v7
+    ; check: return v8
 }
 
 function %i64_shl_sshr_8_to_ireduce(i64) -> i64 {
@@ -241,9 +241,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 56
     v2 = sshr_imm v1, 56
     return v2
-    ; check: v5 = ireduce.i8 v0
-    ; check: v6 = sextend.i64 v5
-    ; check: return v6
+    ; check: v7 = ireduce.i8 v0
+    ; check: v8 = sextend.i64 v7
+    ; check: return v8
 }
 
 function %i64_shl_sshr_16_to_ireduce(i64) -> i64 {
@@ -251,9 +251,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 48
     v2 = sshr_imm v1, 48
     return v2
-    ; check: v5 = ireduce.i16 v0
-    ; check: v6 = sextend.i64 v5
-    ; check: return v6
+    ; check: v7 = ireduce.i16 v0
+    ; check: v8 = sextend.i64 v7
+    ; check: return v8
 }
 
 function %i64_shl_sshr_32_to_ireduce(i64) -> i64 {
@@ -261,9 +261,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 32
     v2 = sshr_imm v1, 32
     return v2
-    ; check: v5 = ireduce.i32 v0
-    ; check: v6 = sextend.i64 v5
-    ; check: return v6
+    ; check: v7 = ireduce.i32 v0
+    ; check: v8 = sextend.i64 v7
+    ; check: return v8
 }
 
 function %i32_shl_ushr_8_to_ireduce(i32) -> i32 {
@@ -271,9 +271,9 @@ block0(v0: i32):
     v1 = ishl_imm v0, 24
     v2 = ushr_imm v1, 24
     return v2
-    ; check: v7 = ireduce.i8 v0
-    ; check: v8 = uextend.i32 v7
-    ; check: return v8
+    ; check: v9 = ireduce.i8 v0
+    ; check: v10 = uextend.i32 v9
+    ; check: return v10
 }
 
 function %i32_shl_ushr_16_to_ireduce(i32) -> i32 {
@@ -281,9 +281,9 @@ block0(v0: i32):
     v1 = ishl_imm v0, 16
     v2 = ushr_imm v1, 16
     return v2
-    ; check: v7 = ireduce.i16 v0
-    ; check: v8 = uextend.i32 v7
-    ; check: return v8
+    ; check: v9 = ireduce.i16 v0
+    ; check: v10 = uextend.i32 v9
+    ; check: return v10
 }
 
 function %i64_shl_ushr_8_to_ireduce(i64) -> i64 {
@@ -291,9 +291,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 56
     v2 = ushr_imm v1, 56
     return v2
-    ; check: v7 = ireduce.i8 v0
-    ; check: v8 = uextend.i64 v7
-    ; check: return v8
+    ; check: v9 = ireduce.i8 v0
+    ; check: v10 = uextend.i64 v9
+    ; check: return v10
 }
 
 function %i64_shl_ushr_16_to_ireduce(i64) -> i64 {
@@ -301,9 +301,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 48
     v2 = ushr_imm v1, 48
     return v2
-    ; check: v7 = ireduce.i16 v0
-    ; check: v8 = uextend.i64 v7
-    ; check: return v8
+    ; check: v9 = ireduce.i16 v0
+    ; check: v10 = uextend.i64 v9
+    ; check: return v10
 }
 
 function %i64_shl_ushr_32_to_ireduce(i64) -> i64 {
@@ -311,9 +311,9 @@ block0(v0: i64):
     v1 = ishl_imm v0, 32
     v2 = ushr_imm v1, 32
     return v2
-    ; check: v7 = ireduce.i32 v0
-    ; check: v8 = uextend.i64 v7
-    ; check: return v8
+    ; check: v9 = ireduce.i32 v0
+    ; check: v10 = uextend.i64 v9
+    ; check: return v10
 }
 
 function %ishl_amt_type_ireduce(i8, i16) -> i8 {
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
index 04cfec907e23..aebf4b89f1de 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -50,9 +50,10 @@
 ;;   clgr %r4, %r2
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   lghi %r5, 4096
-;;   strv %r3, 0(%r5,%r4)
+;;   lg %r5, 0(%r5)
+;;   agr %r5, %r4
+;;   lghi %r2, 4096
+;;   strv %r3, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -65,14 +66,16 @@
 ;; block0:
 ;;   lgr %r4, %r3
 ;;   llgfr %r3, %r2
-;;   lg %r2, 8(%r4)
-;;   aghi %r2, -4100
-;;   clgr %r3, %r2
+;;   lgr %r5, %r4
+;;   lg %r4, 8(%r5)
+;;   aghi %r4, -4100
+;;   clgr %r3, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   lghi %r4, 4096
-;;   lrv %r2, 0(%r4,%r3)
+;;   lg %r5, 0(%r5)
+;;   agr %r5, %r3
+;;   lghi %r2, 4096
+;;   lrv %r2, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 92de57a3f35a..9129071d021e 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -53,23 +52,22 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r7, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r5, %r2
+;;   llgfr %r5, %r2
+;;   lgr %r10, %r2
 ;;   llilf %r2, 4294901764
-;;   algfr %r2, %r5
+;;   algfr %r2, %r10
 ;;   jgnle .+2 # trap=heap_oob
-;;   lgr %r5, %r7
-;;   lg %r7, 8(%r5)
-;;   clgr %r2, %r7
+;;   lg %r8, 8(%r4)
+;;   clgr %r2, %r8
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   llilh %r2, 65535
-;;   strv %r3, 0(%r2,%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   llilh %r5, 65535
+;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;; block3:
 ;;   .word 0x0000 # trap=heap_oob
@@ -78,20 +76,18 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r4, %r2
-;;   llilf %r2, 4294901764
-;;   algfr %r2, %r4
+;;   llgfr %r4, %r2
+;;   llilf %r5, 4294901764
+;;   algfr %r5, %r2
 ;;   jgnle .+2 # trap=heap_oob
-;;   lgr %r4, %r5
-;;   lg %r5, 8(%r4)
-;;   clgr %r2, %r5
+;;   lg %r2, 8(%r3)
+;;   clgr %r5, %r2
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   llilh %r2, 65535
-;;   lrv %r2, 0(%r2,%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r4
+;;   llilh %r4, 65535
+;;   lrv %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
index fd04d8dfce5f..dfd611fb7c60 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -50,9 +50,10 @@
 ;;   clgr %r4, %r2
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   lghi %r5, 4096
-;;   stc %r3, 0(%r5,%r4)
+;;   lg %r5, 0(%r5)
+;;   agr %r5, %r4
+;;   lghi %r2, 4096
+;;   stc %r3, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -65,14 +66,16 @@
 ;; block0:
 ;;   lgr %r4, %r3
 ;;   llgfr %r3, %r2
-;;   lg %r2, 8(%r4)
-;;   aghi %r2, -4097
-;;   clgr %r3, %r2
+;;   lgr %r5, %r4
+;;   lg %r4, 8(%r5)
+;;   aghi %r4, -4097
+;;   clgr %r3, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   lghi %r4, 4096
-;;   llc %r2, 0(%r4,%r3)
+;;   lg %r5, 0(%r5)
+;;   agr %r5, %r3
+;;   lghi %r2, 4096
+;;   llc %r2, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
index 559b03c3e981..7a4c2ef807d8 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -53,23 +52,22 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r7, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r5, %r2
+;;   llgfr %r5, %r2
+;;   lgr %r10, %r2
 ;;   llilf %r2, 4294901761
-;;   algfr %r2, %r5
+;;   algfr %r2, %r10
 ;;   jgnle .+2 # trap=heap_oob
-;;   lgr %r5, %r7
-;;   lg %r7, 8(%r5)
-;;   clgr %r2, %r7
+;;   lg %r8, 8(%r4)
+;;   clgr %r2, %r8
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   llilh %r2, 65535
-;;   stc %r3, 0(%r2,%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   llilh %r5, 65535
+;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;; block3:
 ;;   .word 0x0000 # trap=heap_oob
@@ -78,20 +76,18 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r4, %r2
-;;   llilf %r2, 4294901761
-;;   algfr %r2, %r4
+;;   llgfr %r4, %r2
+;;   llilf %r5, 4294901761
+;;   algfr %r5, %r2
 ;;   jgnle .+2 # trap=heap_oob
-;;   lgr %r4, %r5
-;;   lg %r5, 8(%r4)
-;;   clgr %r2, %r5
+;;   lg %r2, 8(%r3)
+;;   clgr %r5, %r2
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   llilh %r2, 65535
-;;   llc %r2, 0(%r2,%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r4
+;;   llilh %r4, 65535
+;;   llc %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
index d52c68aed627..ae1529e1467a 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -54,37 +53,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r2, %r5
-;;   lg %r5, 8(%r2)
-;;   aghik %r6, %r5, -4
-;;   lgr %r5, %r4
-;;   ag %r5, 0(%r2)
-;;   lghi %r2, 0
-;;   clgr %r4, %r6
-;;   locgrh %r5, %r2
-;;   strv %r3, 0(%r5)
+;;   llgfr %r7, %r2
+;;   lg %r5, 8(%r4)
+;;   aghi %r5, -4
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r7
+;;   lghi %r4, 0
+;;   clgr %r7, %r5
+;;   locgrh %r2, %r4
+;;   strv %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r2, %r4
-;;   lg %r4, 8(%r2)
-;;   aghi %r4, -4
-;;   lgr %r5, %r3
-;;   ag %r5, 0(%r2)
-;;   lghi %r2, 0
-;;   clgr %r3, %r4
-;;   locgrh %r5, %r2
-;;   lrv %r2, 0(%r5)
+;;   llgfr %r4, %r2
+;;   lg %r5, 8(%r3)
+;;   aghi %r5, -4
+;;   lg %r2, 0(%r3)
+;;   agr %r2, %r4
+;;   lghi %r3, 0
+;;   clgr %r4, %r5
+;;   locgrh %r2, %r3
+;;   lrv %r2, 0(%r2)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
index dc74058fbcd6..441d61a0a6ba 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -56,32 +55,32 @@
 ;;   llgfr %r5, %r2
 ;;   lg %r2, 8(%r4)
 ;;   aghi %r2, -4100
-;;   lgr %r7, %r5
-;;   ag %r7, 0(%r4)
-;;   aghik %r4, %r7, 4096
-;;   lghi %r7, 0
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   aghi %r4, 4096
+;;   lghi %r8, 0
 ;;   clgr %r5, %r2
-;;   locgrh %r4, %r7
+;;   locgrh %r4, %r8
 ;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r4, %r2
-;;   lg %r5, 8(%r3)
-;;   aghi %r5, -4100
-;;   lgr %r2, %r4
-;;   ag %r2, 0(%r3)
-;;   aghi %r2, 4096
-;;   lghi %r3, 0
-;;   clgr %r4, %r5
-;;   locgrh %r2, %r3
-;;   lrv %r2, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r3)
+;;   aghi %r2, -4100
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   aghi %r3, 4096
+;;   lghi %r4, 0
+;;   clgr %r5, %r2
+;;   locgrh %r3, %r4
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index 5d1c7f1916ee..e891ab649b04 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -41,7 +41,8 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r10, %r15, 80(%r15)
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
 ;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
@@ -50,43 +51,50 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r14, %r2
-;;   llilf %r5, 4294901764
-;;   algfr %r5, %r2
+;;   llgfr %r10, %r2
+;;   llilf %r9, 4294901764
+;;   algfr %r9, %r2
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r10, 8(%r4)
-;;   ag %r14, 0(%r4)
-;;   llilh %r4, 65535
-;;   agrk %r2, %r14, %r4
-;;   lghi %r4, 0
-;;   clgr %r5, %r10
-;;   locgrh %r2, %r4
-;;   strv %r3, 0(%r2)
+;;   lg %r5, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r10
+;;   llilh %r2, 65535
+;;   agr %r4, %r2
+;;   lghi %r2, 0
+;;   clgr %r9, %r5
+;;   locgrh %r4, %r2
+;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r10, %r15, 80(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r14, %r2
-;;   llilf %r5, 4294901764
-;;   algfr %r5, %r2
+;;   llgfr %r5, %r2
+;;   llilf %r9, 4294901764
+;;   algfr %r9, %r2
 ;;   jgnle .+2 # trap=heap_oob
 ;;   lg %r4, 8(%r3)
-;;   ag %r14, 0(%r3)
-;;   llilh %r3, 65535
-;;   agrk %r2, %r14, %r3
-;;   lghi %r3, 0
-;;   clgr %r5, %r4
-;;   locgrh %r2, %r3
-;;   lrv %r2, 0(%r2)
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   llilh %r2, 65535
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
+;;   clgr %r9, %r4
+;;   locgrh %r3, %r5
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 334a42b8eee9..99d184cbf083 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,7 +41,9 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r8, %r15, 64(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -52,32 +54,43 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
+;;   llgfr %r7, %r2
 ;;   lg %r5, 8(%r4)
-;;   lgr %r8, %r4
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r8)
-;;   lghi %r14, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r4, %r14
-;;   stc %r3, 0(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r7
+;;   lghi %r6, 0
+;;   clgr %r7, %r5
+;;   locgrhe %r2, %r6
+;;   stc %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r8, %r15, 64(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lg %r5, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   lghi %r3, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r4, %r3
-;;   llc %r2, 0(%r4)
+;;   llgfr %r6, %r2
+;;   lg %r4, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r5, %r3, %r6
+;;   lghi %r2, 0
+;;   clgr %r6, %r4
+;;   locgrhe %r5, %r2
+;;   llc %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
index 44be0e98eb28..7979de1f2a26 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -56,32 +55,32 @@
 ;;   llgfr %r5, %r2
 ;;   lg %r2, 8(%r4)
 ;;   aghi %r2, -4097
-;;   lgr %r7, %r5
-;;   ag %r7, 0(%r4)
-;;   aghik %r4, %r7, 4096
-;;   lghi %r7, 0
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   aghi %r4, 4096
+;;   lghi %r8, 0
 ;;   clgr %r5, %r2
-;;   locgrh %r4, %r7
+;;   locgrh %r4, %r8
 ;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r4, %r2
-;;   lg %r5, 8(%r3)
-;;   aghi %r5, -4097
-;;   lgr %r2, %r4
-;;   ag %r2, 0(%r3)
-;;   aghi %r2, 4096
-;;   lghi %r3, 0
-;;   clgr %r4, %r5
-;;   locgrh %r2, %r3
-;;   llc %r2, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r3)
+;;   aghi %r2, -4097
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   aghi %r3, 4096
+;;   lghi %r4, 0
+;;   clgr %r5, %r2
+;;   locgrh %r3, %r4
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 93550319358b..c7827f26dbe8 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -41,7 +41,8 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r10, %r15, 80(%r15)
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
 ;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
@@ -50,43 +51,50 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r14, %r2
-;;   llilf %r5, 4294901761
-;;   algfr %r5, %r2
+;;   llgfr %r10, %r2
+;;   llilf %r9, 4294901761
+;;   algfr %r9, %r2
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r10, 8(%r4)
-;;   ag %r14, 0(%r4)
-;;   llilh %r4, 65535
-;;   agrk %r2, %r14, %r4
-;;   lghi %r4, 0
-;;   clgr %r5, %r10
-;;   locgrh %r2, %r4
-;;   stc %r3, 0(%r2)
+;;   lg %r5, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r10
+;;   llilh %r2, 65535
+;;   agr %r4, %r2
+;;   lghi %r2, 0
+;;   clgr %r9, %r5
+;;   locgrh %r4, %r2
+;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r10, %r15, 80(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r14, %r2
-;;   llilf %r5, 4294901761
-;;   algfr %r5, %r2
+;;   llgfr %r5, %r2
+;;   llilf %r9, 4294901761
+;;   algfr %r9, %r2
 ;;   jgnle .+2 # trap=heap_oob
 ;;   lg %r4, 8(%r3)
-;;   ag %r14, 0(%r3)
-;;   llilh %r3, 65535
-;;   agrk %r2, %r14, %r3
-;;   lghi %r3, 0
-;;   clgr %r5, %r4
-;;   locgrh %r2, %r3
-;;   llc %r2, 0(%r2)
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   llilh %r2, 65535
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
+;;   clgr %r9, %r4
+;;   locgrh %r3, %r5
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
index 898fac6ffcc6..a8274eefe16d 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -43,13 +43,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lg %r2, 8(%r5)
-;;   clgr %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r5, 8(%r4)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
 ;;   lghi %r5, 4096
 ;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
@@ -62,15 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r3
-;;   llgfr %r3, %r2
-;;   lg %r5, 8(%r4)
-;;   clgr %r3, %r5
+;;   llgfr %r2, %r2
+;;   lg %r4, 8(%r3)
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   lghi %r4, 4096
-;;   lrv %r2, 0(%r4,%r3)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   lrv %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 47d8b57e247f..2b0c428e21f6 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -43,13 +43,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lg %r2, 8(%r5)
-;;   clgr %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r5, 8(%r4)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
 ;;   llilh %r5, 65535
 ;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
@@ -62,15 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r3
-;;   llgfr %r3, %r2
-;;   lg %r5, 8(%r4)
-;;   clgr %r3, %r5
+;;   llgfr %r2, %r2
+;;   lg %r4, 8(%r3)
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   llilh %r4, 65535
-;;   lrv %r2, 0(%r4,%r3)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   llilh %r5, 65535
+;;   lrv %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
index d3bac7e16fac..622e48fe73cc 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -43,13 +43,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lg %r2, 8(%r5)
-;;   clgr %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r5, 8(%r4)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
 ;;   lghi %r5, 4096
 ;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
@@ -62,15 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r3
-;;   llgfr %r3, %r2
-;;   lg %r5, 8(%r4)
-;;   clgr %r3, %r5
+;;   llgfr %r2, %r2
+;;   lg %r4, 8(%r3)
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   lghi %r4, 4096
-;;   llc %r2, 0(%r4,%r3)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   llc %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
index 50469a1d1b84..56c149aee5cc 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -43,13 +43,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lg %r2, 8(%r5)
-;;   clgr %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r5, 8(%r4)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
 ;;   llilh %r5, 65535
 ;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
@@ -62,15 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r3
-;;   llgfr %r3, %r2
-;;   lg %r5, 8(%r4)
-;;   clgr %r3, %r5
+;;   llgfr %r2, %r2
+;;   lg %r4, 8(%r3)
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r3, 0(%r4)
-;;   llilh %r4, 65535
-;;   llc %r2, 0(%r4,%r3)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   llilh %r5, 65535
+;;   llc %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index 3f0878a849f0..88e073836a22 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,7 +41,9 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r8, %r15, 64(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -52,32 +54,43 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
+;;   llgfr %r7, %r2
 ;;   lg %r5, 8(%r4)
-;;   lgr %r8, %r4
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r8)
-;;   lghi %r14, 0
-;;   clgr %r2, %r5
-;;   locgrh %r4, %r14
-;;   strv %r3, 0(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r7
+;;   lghi %r6, 0
+;;   clgr %r7, %r5
+;;   locgrh %r2, %r6
+;;   strv %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r8, %r15, 64(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lg %r5, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   lghi %r3, 0
-;;   clgr %r2, %r5
-;;   locgrh %r4, %r3
-;;   lrv %r2, 0(%r4)
+;;   llgfr %r6, %r2
+;;   lg %r4, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r5, %r3, %r6
+;;   lghi %r2, 0
+;;   clgr %r6, %r4
+;;   locgrh %r5, %r2
+;;   lrv %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
index 2d0a997c86de..268b4ab4580f 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -54,33 +53,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r6, %r2
+;;   llgfr %r8, %r2
 ;;   lg %r5, 8(%r4)
-;;   lgr %r2, %r6
-;;   ag %r2, 0(%r4)
-;;   aghi %r2, 4096
-;;   lghi %r4, 0
-;;   clgr %r6, %r5
-;;   locgrh %r2, %r4
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r8
+;;   aghik %r2, %r4, 4096
+;;   lghi %r7, 0
+;;   clgr %r8, %r5
+;;   locgrh %r2, %r7
 ;;   strv %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lg %r4, 8(%r3)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghi %r5, 4096
+;;   llgfr %r4, %r2
+;;   lg %r5, 8(%r3)
+;;   lg %r2, 0(%r3)
+;;   agr %r2, %r4
+;;   aghi %r2, 4096
 ;;   lghi %r3, 0
-;;   clgr %r2, %r4
-;;   locgrh %r5, %r3
-;;   lrv %r2, 0(%r5)
+;;   clgr %r4, %r5
+;;   locgrh %r2, %r3
+;;   lrv %r2, 0(%r2)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index d013a5d8a469..48acc69e20cc 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -53,35 +52,35 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r7, %r2
-;;   lg %r5, 8(%r4)
-;;   lgr %r2, %r7
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   agr %r2, %r4
-;;   lghi %r4, 0
-;;   clgr %r7, %r5
-;;   locgrh %r2, %r4
-;;   strv %r3, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   llilh %r8, 65535
+;;   agr %r4, %r8
+;;   lghi %r8, 0
+;;   clgr %r5, %r2
+;;   locgrh %r4, %r8
+;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r4, %r2
-;;   lg %r5, 8(%r3)
-;;   lgr %r2, %r4
-;;   ag %r2, 0(%r3)
-;;   llilh %r3, 65535
-;;   agr %r2, %r3
-;;   lghi %r3, 0
-;;   clgr %r4, %r5
-;;   locgrh %r2, %r3
-;;   lrv %r2, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   llilh %r4, 65535
+;;   agr %r3, %r4
+;;   lghi %r4, 0
+;;   clgr %r5, %r2
+;;   locgrh %r3, %r4
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 47f15780d729..32fbdfd47665 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,7 +41,9 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r8, %r15, 64(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -52,32 +54,43 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
+;;   llgfr %r7, %r2
 ;;   lg %r5, 8(%r4)
-;;   lgr %r8, %r4
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r8)
-;;   lghi %r14, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r4, %r14
-;;   stc %r3, 0(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r7
+;;   lghi %r6, 0
+;;   clgr %r7, %r5
+;;   locgrhe %r2, %r6
+;;   stc %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r8, %r15, 64(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lg %r5, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   lghi %r3, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r4, %r3
-;;   llc %r2, 0(%r4)
+;;   llgfr %r6, %r2
+;;   lg %r4, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r5, %r3, %r6
+;;   lghi %r2, 0
+;;   clgr %r6, %r4
+;;   locgrhe %r5, %r2
+;;   llc %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
index 342e1136474d..a71acb56f3a5 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -54,33 +53,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r6, %r2
+;;   llgfr %r8, %r2
 ;;   lg %r5, 8(%r4)
-;;   lgr %r2, %r6
-;;   ag %r2, 0(%r4)
-;;   aghi %r2, 4096
-;;   lghi %r4, 0
-;;   clgr %r6, %r5
-;;   locgrh %r2, %r4
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r8
+;;   aghik %r2, %r4, 4096
+;;   lghi %r7, 0
+;;   clgr %r8, %r5
+;;   locgrh %r2, %r7
 ;;   stc %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lg %r4, 8(%r3)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghi %r5, 4096
+;;   llgfr %r4, %r2
+;;   lg %r5, 8(%r3)
+;;   lg %r2, 0(%r3)
+;;   agr %r2, %r4
+;;   aghi %r2, 4096
 ;;   lghi %r3, 0
-;;   clgr %r2, %r4
-;;   locgrh %r5, %r3
-;;   llc %r2, 0(%r5)
+;;   clgr %r4, %r5
+;;   locgrh %r2, %r3
+;;   llc %r2, 0(%r2)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 4a9bb7442205..ca2f67e4043c 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   stmg %r8, %r15, 64(%r15)
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -53,35 +52,35 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r7, %r2
-;;   lg %r5, 8(%r4)
-;;   lgr %r2, %r7
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   agr %r2, %r4
-;;   lghi %r4, 0
-;;   clgr %r7, %r5
-;;   locgrh %r2, %r4
-;;   stc %r3, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   llilh %r8, 65535
+;;   agr %r4, %r8
+;;   lghi %r8, 0
+;;   clgr %r5, %r2
+;;   locgrh %r4, %r8
+;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r8, %r15, 64(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r4, %r2
-;;   lg %r5, 8(%r3)
-;;   lgr %r2, %r4
-;;   ag %r2, 0(%r3)
-;;   llilh %r3, 65535
-;;   agr %r2, %r3
-;;   lghi %r3, 0
-;;   clgr %r4, %r5
-;;   locgrh %r2, %r3
-;;   llc %r2, 0(%r2)
+;;   llgfr %r5, %r2
+;;   lg %r2, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   llilh %r4, 65535
+;;   agr %r3, %r4
+;;   lghi %r4, 0
+;;   clgr %r5, %r2
+;;   locgrh %r3, %r4
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
index 7b3b2bcc60b0..927e4e223bf5 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -48,9 +48,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r4, 4096
-;;   strv %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -61,14 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r3)
-;;   aghi %r5, -4100
-;;   clgr %r2, %r5
+;;   lg %r4, 8(%r3)
+;;   aghi %r4, -4100
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r3, 4096
-;;   lrv %r2, 0(%r3,%r2)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   lrv %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 50f1dc5ade66..ea04669dd9f6 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -41,7 +41,15 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
@@ -49,16 +57,17 @@
 ;;   lgr %r5, %r2
 ;;   algfi %r5, 4294901764
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r14, 8(%r4)
-;;   clgr %r5, %r14
+;;   lg %r6, 8(%r4)
+;;   clgr %r5, %r6
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   strv %r3, 0(%r4,%r2)
+;;   lg %r5, 0(%r4)
+;;   agr %r5, %r2
+;;   llilh %r2, 65535
+;;   strv %r3, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;; block3:
 ;;   .word 0x0000 # trap=heap_oob
@@ -67,16 +76,17 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   algfi %r5, 4294901764
+;;   lgr %r4, %r2
+;;   algfi %r4, 4294901764
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r4, 8(%r3)
-;;   clgr %r5, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r4, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   llilh %r4, 65535
-;;   lrv %r2, 0(%r4,%r2)
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
+;;   llilh %r2, 65535
+;;   lrv %r2, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
index b24ed73ac507..078eb88aacba 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -48,9 +48,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r4, 4096
-;;   stc %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -61,14 +62,15 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r3)
-;;   aghi %r5, -4097
-;;   clgr %r2, %r5
+;;   lg %r4, 8(%r3)
+;;   aghi %r4, -4097
+;;   clgr %r2, %r4
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r3, 4096
-;;   llc %r2, 0(%r3,%r2)
+;;   lg %r4, 0(%r3)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   llc %r2, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
index be59a9689720..72d33a988324 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -41,7 +41,15 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
@@ -49,16 +57,17 @@
 ;;   lgr %r5, %r2
 ;;   algfi %r5, 4294901761
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r14, 8(%r4)
-;;   clgr %r5, %r14
+;;   lg %r6, 8(%r4)
+;;   clgr %r5, %r6
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   stc %r3, 0(%r4,%r2)
+;;   lg %r5, 0(%r4)
+;;   agr %r5, %r2
+;;   llilh %r2, 65535
+;;   stc %r3, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;; block3:
 ;;   .word 0x0000 # trap=heap_oob
@@ -67,16 +76,17 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   algfi %r5, 4294901761
+;;   lgr %r4, %r2
+;;   algfi %r4, 4294901761
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r4, 8(%r3)
-;;   clgr %r5, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r4, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   llilh %r4, 65535
-;;   llc %r2, 0(%r4,%r2)
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
+;;   llilh %r2, 65535
+;;   llc %r2, 0(%r2,%r5)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
index 666728fe881f..e7b3162bd066 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,7 +41,9 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r8, %r15, 64(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
@@ -53,31 +55,42 @@
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   lg %r5, 8(%r4)
-;;   lgr %r8, %r4
-;;   aghi %r5, -4
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r8)
-;;   lghi %r14, 0
-;;   clgr %r2, %r5
-;;   locgrh %r4, %r14
-;;   strv %r3, 0(%r4)
+;;   aghik %r7, %r5, -4
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
+;;   lghi %r6, 0
+;;   clgr %r2, %r7
+;;   locgrh %r5, %r6
+;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r8, %r15, 64(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   lg %r4, 8(%r3)
-;;   aghik %r5, %r4, -4
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   lghi %r3, 0
-;;   clgr %r2, %r5
-;;   locgrh %r4, %r3
-;;   lrv %r2, 0(%r4)
+;;   aghik %r6, %r4, -4
+;;   lg %r3, 0(%r3)
+;;   agrk %r5, %r3, %r2
+;;   lghi %r4, 0
+;;   clgr %r2, %r6
+;;   locgrh %r5, %r4
+;;   lrv %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
index b9c4fd6a9812..55facb05809a 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -55,17 +54,17 @@
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   lg %r5, 8(%r4)
-;;   aghik %r6, %r5, -4100
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghi %r5, 4096
-;;   lghi %r4, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r4
+;;   aghik %r8, %r5, -4100
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghik %r5, %r4, 4096
+;;   lghi %r7, 0
+;;   clgr %r2, %r8
+;;   locgrh %r5, %r7
 ;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
@@ -74,13 +73,13 @@
 ;; block0:
 ;;   lg %r4, 8(%r3)
 ;;   aghi %r4, -4100
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghi %r5, 4096
-;;   lghi %r3, 0
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
+;;   aghik %r3, %r5, 4096
+;;   lghi %r5, 0
 ;;   clgr %r2, %r4
-;;   locgrh %r5, %r3
-;;   lrv %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index ee6cac682505..59622818dc2e 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -41,9 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
-;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   stmg %r9, %r15, 72(%r15)
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
@@ -53,37 +51,48 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r7, %r2
-;;   algfi %r7, 4294901764
+;;   lgr %r5, %r2
+;;   algfi %r5, 4294901764
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r5, 8(%r4)
-;;   ag %r2, 0(%r4)
+;;   lg %r9, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
 ;;   llilh %r4, 65535
-;;   agrk %r4, %r2, %r4
-;;   lghi %r2, 0
-;;   clgr %r7, %r5
-;;   locgrh %r4, %r2
-;;   strv %r3, 0(%r4)
+;;   agr %r2, %r4
+;;   lghi %r4, 0
+;;   clgr %r5, %r9
+;;   locgrh %r2, %r4
+;;   strv %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r2
-;;   algfi %r4, 4294901764
+;;   lgr %r5, %r2
+;;   algfi %r5, 4294901764
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r5, 8(%r3)
-;;   ag %r2, 0(%r3)
+;;   lg %r9, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
 ;;   llilh %r3, 65535
-;;   agrk %r3, %r2, %r3
+;;   agrk %r4, %r2, %r3
 ;;   lghi %r2, 0
-;;   clgr %r4, %r5
-;;   locgrh %r3, %r2
-;;   lrv %r2, 0(%r3)
+;;   clgr %r5, %r9
+;;   locgrh %r4, %r2
+;;   lrv %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 907d9e6b9505..2516db71460e 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,7 +41,12 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
 ;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
 ;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
@@ -49,36 +54,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r4)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r4)
-;;   lghi %r13, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r11, %r13
-;;   stc %r3, 0(%r11)
+;;   lg %r6, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
+;;   lghi %r14, 0
+;;   clgr %r2, %r6
+;;   locgrhe %r5, %r14
+;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
-;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
-;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
-;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   stmg %r14, %r15, 112(%r15)
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r3)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r3)
-;;   lghi %r4, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r11, %r4
-;;   llc %r2, 0(%r11)
+;;   lg %r14, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r4, %r3, %r2
+;;   lghi %r5, 0
+;;   clgr %r2, %r14
+;;   locgrhe %r4, %r5
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r14, %r15, 112(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
index 9331c9404892..20876eabc832 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -55,17 +54,17 @@
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   lg %r5, 8(%r4)
-;;   aghik %r6, %r5, -4097
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghi %r5, 4096
-;;   lghi %r4, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r4
+;;   aghik %r8, %r5, -4097
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghik %r5, %r4, 4096
+;;   lghi %r7, 0
+;;   clgr %r2, %r8
+;;   locgrh %r5, %r7
 ;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
@@ -74,13 +73,13 @@
 ;; block0:
 ;;   lg %r4, 8(%r3)
 ;;   aghi %r4, -4097
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghi %r5, 4096
-;;   lghi %r3, 0
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
+;;   aghik %r3, %r5, 4096
+;;   lghi %r5, 0
 ;;   clgr %r2, %r4
-;;   locgrh %r5, %r3
-;;   llc %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 8fcb0d779b38..aa582fced9a9 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -41,9 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r7, %r15, 56(%r15)
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
-;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   stmg %r9, %r15, 72(%r15)
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
 ;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
@@ -53,37 +51,48 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r7, %r2
-;;   algfi %r7, 4294901761
+;;   lgr %r5, %r2
+;;   algfi %r5, 4294901761
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r5, 8(%r4)
-;;   ag %r2, 0(%r4)
+;;   lg %r9, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
 ;;   llilh %r4, 65535
-;;   agrk %r4, %r2, %r4
-;;   lghi %r2, 0
-;;   clgr %r7, %r5
-;;   locgrh %r4, %r2
-;;   stc %r3, 0(%r4)
+;;   agr %r2, %r4
+;;   lghi %r4, 0
+;;   clgr %r5, %r9
+;;   locgrh %r2, %r4
+;;   stc %r3, 0(%r2)
 ;;   jg label1
 ;; block1:
-;;   lmg %r7, %r15, 56(%r15)
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r9, %r15, 72(%r15)
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r4, %r2
-;;   algfi %r4, 4294901761
+;;   lgr %r5, %r2
+;;   algfi %r5, 4294901761
 ;;   jgnle .+2 # trap=heap_oob
-;;   lg %r5, 8(%r3)
-;;   ag %r2, 0(%r3)
+;;   lg %r9, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
 ;;   llilh %r3, 65535
-;;   agrk %r3, %r2, %r3
+;;   agrk %r4, %r2, %r3
 ;;   lghi %r2, 0
-;;   clgr %r4, %r5
-;;   locgrh %r3, %r2
-;;   llc %r2, 0(%r3)
+;;   clgr %r5, %r9
+;;   locgrh %r4, %r2
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
+;;   lmg %r9, %r15, 72(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
index c034f17edad9..76ab2e750760 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -47,9 +47,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r4, 4096
-;;   strv %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,13 +61,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r4, 8(%r3)
-;;   clgr %r2, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r3, 4096
-;;   lrv %r2, 0(%r3,%r2)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   lghi %r4, 4096
+;;   lrv %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 27825b12e20d..ef080b4b88d5 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -47,9 +47,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   strv %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   llilh %r5, 65535
+;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,13 +61,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r4, 8(%r3)
-;;   clgr %r2, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   llilh %r3, 65535
-;;   lrv %r2, 0(%r3,%r2)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   llilh %r4, 65535
+;;   lrv %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
index 076de1172519..8e2d6e944a92 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -47,9 +47,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r4, 4096
-;;   stc %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 4096
+;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,13 +61,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r4, 8(%r3)
-;;   clgr %r2, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r3, 4096
-;;   llc %r2, 0(%r3,%r2)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   lghi %r4, 4096
+;;   llc %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
index d4473f5877f8..cc758ab83039 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -47,9 +47,10 @@
 ;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   llilh %r4, 65535
-;;   stc %r3, 0(%r4,%r2)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   llilh %r5, 65535
+;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,13 +61,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r4, 8(%r3)
-;;   clgr %r2, %r4
+;;   lg %r5, 8(%r3)
+;;   clgr %r2, %r5
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   llilh %r3, 65535
-;;   llc %r2, 0(%r3,%r2)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   llilh %r4, 65535
+;;   llc %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index 7ffb39d00c00..be7704012f59 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,7 +41,12 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
 ;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
 ;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
@@ -49,36 +54,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r4)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r4)
-;;   lghi %r13, 0
-;;   clgr %r2, %r5
-;;   locgrh %r11, %r13
-;;   strv %r3, 0(%r11)
+;;   lg %r6, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
+;;   lghi %r14, 0
+;;   clgr %r2, %r6
+;;   locgrh %r5, %r14
+;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
-;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
-;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
-;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   stmg %r14, %r15, 112(%r15)
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r3)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r3)
-;;   lghi %r4, 0
-;;   clgr %r2, %r5
-;;   locgrh %r11, %r4
-;;   lrv %r2, 0(%r11)
+;;   lg %r14, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r4, %r3, %r2
+;;   lghi %r5, 0
+;;   clgr %r2, %r14
+;;   locgrh %r4, %r5
+;;   lrv %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r14, %r15, 112(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
index 1c2070ffc42c..f11d7bf61098 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -54,13 +54,13 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r6, 8(%r4)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghi %r5, 4096
-;;   lghi %r14, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r14
+;;   lg %r7, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghik %r5, %r4, 4096
+;;   lghi %r6, 0
+;;   clgr %r2, %r7
+;;   locgrh %r5, %r6
 ;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
@@ -69,20 +69,28 @@
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r14, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   aghi %r4, 4096
-;;   lghi %r5, 0
-;;   clgr %r2, %r14
-;;   locgrh %r4, %r5
-;;   lrv %r2, 0(%r4)
+;;   lg %r6, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r5, %r3, 4096
+;;   lghi %r4, 0
+;;   clgr %r2, %r6
+;;   locgrh %r5, %r4
+;;   lrv %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index 3d8f014252ba..ebd6f401c062 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -54,45 +53,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r7, 8(%r4)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
+;;   lg %r8, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
 ;;   llilh %r4, 65535
 ;;   agr %r5, %r4
-;;   lghi %r6, 0
-;;   clgr %r2, %r7
-;;   locgrh %r5, %r6
+;;   lghi %r7, 0
+;;   clgr %r2, %r8
+;;   locgrh %r5, %r7
 ;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
-;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
-;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
-;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
-;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
-;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
-;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
-;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
-;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r6, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
+;;   lg %r4, 8(%r3)
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
 ;;   llilh %r3, 65535
-;;   agrk %r5, %r4, %r3
-;;   lghi %r4, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r4
-;;   lrv %r2, 0(%r5)
+;;   agrk %r3, %r5, %r3
+;;   lghi %r5, 0
+;;   clgr %r2, %r4
+;;   locgrh %r3, %r5
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 175a19bf50e8..a7f62a297712 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,7 +41,12 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
 ;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
 ;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
 ;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
@@ -49,36 +54,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r4)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r4)
-;;   lghi %r13, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r11, %r13
-;;   stc %r3, 0(%r11)
+;;   lg %r6, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
+;;   lghi %r14, 0
+;;   clgr %r2, %r6
+;;   locgrhe %r5, %r14
+;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r11, %r15, 88(%r15)
-;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
-;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
-;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   stmg %r14, %r15, 112(%r15)
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r5, 8(%r3)
-;;   lgr %r11, %r2
-;;   ag %r11, 0(%r3)
-;;   lghi %r4, 0
-;;   clgr %r2, %r5
-;;   locgrhe %r11, %r4
-;;   llc %r2, 0(%r11)
+;;   lg %r14, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agrk %r4, %r3, %r2
+;;   lghi %r5, 0
+;;   clgr %r2, %r14
+;;   locgrhe %r4, %r5
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
-;;   lmg %r11, %r15, 88(%r15)
+;;   lmg %r14, %r15, 112(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
index 9a62871d7006..5fbf8ea239ef 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -54,13 +54,13 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r6, 8(%r4)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghi %r5, 4096
-;;   lghi %r14, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r14
+;;   lg %r7, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghik %r5, %r4, 4096
+;;   lghi %r6, 0
+;;   clgr %r2, %r7
+;;   locgrh %r5, %r6
 ;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
@@ -69,20 +69,28 @@
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r14, %r15, 112(%r15)
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
 ;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r14, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   aghi %r4, 4096
-;;   lghi %r5, 0
-;;   clgr %r2, %r14
-;;   locgrh %r4, %r5
-;;   llc %r2, 0(%r4)
+;;   lg %r6, 8(%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r5, %r3, 4096
+;;   lghi %r4, 0
+;;   clgr %r2, %r6
+;;   locgrh %r5, %r4
+;;   llc %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r14, %r15, 112(%r15)
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 2f861cc1f10a..dba033d3a90a 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -41,8 +41,7 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   stmg %r7, %r15, 56(%r15)
 ;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
 ;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
 ;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
@@ -54,45 +53,33 @@
 ;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r7, 8(%r4)
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
+;;   lg %r8, 8(%r4)
+;;   lg %r4, 0(%r4)
+;;   agrk %r5, %r4, %r2
 ;;   llilh %r4, 65535
 ;;   agr %r5, %r4
-;;   lghi %r6, 0
-;;   clgr %r2, %r7
-;;   locgrh %r5, %r6
+;;   lghi %r7, 0
+;;   clgr %r2, %r8
+;;   locgrh %r5, %r7
 ;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
+;;   lmg %r7, %r15, 56(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
-;;   stmg %r6, %r15, 48(%r15)
-;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
-;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
-;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
-;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
-;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
-;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
-;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
-;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
-;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
-;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lg %r6, 8(%r3)
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
+;;   lg %r4, 8(%r3)
+;;   lg %r5, 0(%r3)
+;;   agr %r5, %r2
 ;;   llilh %r3, 65535
-;;   agrk %r5, %r4, %r3
-;;   lghi %r4, 0
-;;   clgr %r2, %r6
-;;   locgrh %r5, %r4
-;;   llc %r2, 0(%r5)
+;;   agrk %r3, %r5, %r3
+;;   lghi %r5, 0
+;;   clgr %r2, %r4
+;;   locgrh %r3, %r5
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
-;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
index dff24197b473..dde65b188d7d 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -41,14 +41,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   clgfi %r4, 268431356
+;;   llgfr %r5, %r2
+;;   clgfi %r5, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   lghi %r2, 4096
-;;   strv %r3, 0(%r2,%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   lghi %r5, 4096
+;;   strv %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -59,15 +59,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   clgfi %r3, 268431356
+;;   llgfr %r5, %r2
+;;   clgfi %r5, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   lgr %r4, %r5
-;;   ag %r3, 0(%r4)
-;;   lghi %r2, 4096
-;;   lrv %r2, 0(%r2,%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   lghi %r4, 4096
+;;   lrv %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
index 072dfed1e7c6..1a0a4d7f7c95 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -41,14 +41,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   clgfi %r4, 268431359
+;;   llgfr %r5, %r2
+;;   clgfi %r5, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r4, 0(%r5)
-;;   lghi %r2, 4096
-;;   stc %r3, 0(%r2,%r4)
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r5
+;;   lghi %r5, 4096
+;;   stc %r3, 0(%r5,%r4)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -59,15 +59,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   clgfi %r3, 268431359
+;;   llgfr %r5, %r2
+;;   clgfi %r5, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   lgr %r4, %r5
-;;   ag %r3, 0(%r4)
-;;   lghi %r2, 4096
-;;   llc %r2, 0(%r2,%r3)
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r5
+;;   lghi %r4, 4096
+;;   llc %r2, 0(%r4,%r3)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
index 502fc9856a03..b2e1740f68d4 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,13 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r5, %r2
-;;   lgr %r2, %r4
-;;   lgr %r4, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r2, 0
-;;   clgfi %r5, 268435452
-;;   locgrh %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 0
+;;   clgfi %r2, 268435452
+;;   locgrh %r4, %r5
 ;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
@@ -57,14 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r5, %r2
-;;   lgr %r4, %r3
-;;   lgr %r3, %r5
-;;   ag %r3, 0(%r4)
-;;   lghi %r2, 0
-;;   clgfi %r5, 268435452
-;;   locgrh %r3, %r2
-;;   lrv %r2, 0(%r3)
+;;   llgfr %r2, %r2
+;;   lg %r3, 0(%r3)
+;;   agrk %r4, %r3, %r2
+;;   lghi %r3, 0
+;;   clgfi %r2, 268435452
+;;   locgrh %r4, %r3
+;;   lrv %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
index 8366b8251c3f..1000414b1f3f 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -39,18 +39,30 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
-;;   lghi %r5, 0
-;;   clgfi %r2, 268431356
-;;   locgrh %r4, %r5
-;;   strv %r3, 0(%r4)
+;;   llgfr %r6, %r2
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r6
+;;   aghik %r5, %r4, 4096
+;;   lghi %r2, 0
+;;   clgfi %r6, 268431356
+;;   locgrh %r5, %r2
+;;   strv %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
@@ -58,13 +70,13 @@
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   llgfr %r2, %r2
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   aghi %r4, 4096
-;;   lghi %r3, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r5, %r3, 4096
+;;   lghi %r4, 0
 ;;   clgfi %r2, 268431356
-;;   locgrh %r4, %r3
-;;   lrv %r2, 0(%r4)
+;;   locgrh %r5, %r4
+;;   lrv %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 3594b2b9e64c..58d3daabe08f 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,13 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r5, %r2
-;;   lgr %r2, %r4
-;;   lgr %r4, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r2, 0
-;;   clgfi %r5, 268435455
-;;   locgrh %r4, %r2
+;;   llgfr %r2, %r2
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   lghi %r5, 0
+;;   clgfi %r2, 268435455
+;;   locgrh %r4, %r5
 ;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
@@ -57,14 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r5, %r2
-;;   lgr %r4, %r3
-;;   lgr %r3, %r5
-;;   ag %r3, 0(%r4)
-;;   lghi %r2, 0
-;;   clgfi %r5, 268435455
-;;   locgrh %r3, %r2
-;;   llc %r2, 0(%r3)
+;;   llgfr %r2, %r2
+;;   lg %r3, 0(%r3)
+;;   agrk %r4, %r3, %r2
+;;   lghi %r3, 0
+;;   clgfi %r2, 268435455
+;;   locgrh %r4, %r3
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
index c51890d0a287..05c9ddee1697 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -39,18 +39,30 @@
 
 ;; function u0:0:
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
+;;   stmg %r6, %r15, 48(%r15)
+;;   unwind SaveReg { clobber_offset: 48, reg: p6i }
+;;   unwind SaveReg { clobber_offset: 56, reg: p7i }
+;;   unwind SaveReg { clobber_offset: 64, reg: p8i }
+;;   unwind SaveReg { clobber_offset: 72, reg: p9i }
+;;   unwind SaveReg { clobber_offset: 80, reg: p10i }
+;;   unwind SaveReg { clobber_offset: 88, reg: p11i }
+;;   unwind SaveReg { clobber_offset: 96, reg: p12i }
+;;   unwind SaveReg { clobber_offset: 104, reg: p13i }
+;;   unwind SaveReg { clobber_offset: 112, reg: p14i }
+;;   unwind SaveReg { clobber_offset: 120, reg: p15i }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   llgfr %r2, %r2
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
-;;   lghi %r5, 0
-;;   clgfi %r2, 268431359
-;;   locgrh %r4, %r5
-;;   stc %r3, 0(%r4)
+;;   llgfr %r6, %r2
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r6
+;;   aghik %r5, %r4, 4096
+;;   lghi %r2, 0
+;;   clgfi %r6, 268431359
+;;   locgrh %r5, %r2
+;;   stc %r3, 0(%r5)
 ;;   jg label1
 ;; block1:
+;;   lmg %r6, %r15, 48(%r15)
 ;;   br %r14
 ;;
 ;; function u0:1:
@@ -58,13 +70,13 @@
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
 ;;   llgfr %r2, %r2
-;;   lgr %r4, %r2
-;;   ag %r4, 0(%r3)
-;;   aghi %r4, 4096
-;;   lghi %r3, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r5, %r3, 4096
+;;   lghi %r4, 0
 ;;   clgfi %r2, 268431359
-;;   locgrh %r4, %r3
-;;   llc %r2, 0(%r4)
+;;   locgrh %r5, %r4
+;;   llc %r2, 0(%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
index 794a1572764f..c82925958375 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -41,12 +41,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r2, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r5, 4096
-;;   strv %r3, 0(%r5,%r4)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r4)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   strv %r3, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +54,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r2, %r5
-;;   ag %r3, 0(%r2)
-;;   lghi %r4, 4096
-;;   lrv %r2, 0(%r4,%r3)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   lrv %r2, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
-;;   br %r14
\ No newline at end of file
+;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
index bbeb3ae418b5..291393224add 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -41,12 +41,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r2, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r5, 4096
-;;   stc %r3, 0(%r5,%r4)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r4)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   stc %r3, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +54,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r2, %r5
-;;   ag %r3, 0(%r2)
-;;   lghi %r4, 4096
-;;   llc %r2, 0(%r4,%r3)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   llc %r2, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
-;;   br %r14
\ No newline at end of file
+;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
index e76eae59fe0b..bb896678129c 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,12 +41,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r2, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r5, 4096
-;;   strv %r3, 0(%r5,%r4)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r4)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   strv %r3, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +54,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r2, %r5
-;;   ag %r3, 0(%r2)
-;;   lghi %r4, 4096
-;;   lrv %r2, 0(%r4,%r3)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   lrv %r2, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
-;;   br %r14
\ No newline at end of file
+;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
index 94614dc1c641..461f2e2529ae 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,12 +41,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r4
-;;   llgfr %r4, %r2
-;;   lgr %r2, %r5
-;;   ag %r4, 0(%r2)
-;;   lghi %r5, 4096
-;;   stc %r3, 0(%r5,%r4)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r4)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   stc %r3, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +54,11 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r3
-;;   llgfr %r3, %r2
-;;   lgr %r2, %r5
-;;   ag %r3, 0(%r2)
-;;   lghi %r4, 4096
-;;   llc %r2, 0(%r4,%r3)
+;;   llgfr %r5, %r2
+;;   lg %r2, 0(%r3)
+;;   agrk %r5, %r2, %r5
+;;   lghi %r2, 4096
+;;   llc %r2, 0(%r2,%r5)
 ;;   jg label1
 ;; block1:
-;;   br %r14
\ No newline at end of file
+;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
index 56cce5cd1cef..4b627be146c2 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -44,9 +44,10 @@
 ;;   clgfi %r2, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r5, 4096
-;;   strv %r3, 0(%r5,%r2)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
+;;   lghi %r4, 4096
+;;   strv %r3, 0(%r4,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,9 +61,10 @@
 ;;   clgfi %r2, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r5, 4096
-;;   lrv %r2, 0(%r5,%r2)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
+;;   lghi %r3, 4096
+;;   lrv %r2, 0(%r3,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
index 99d2a21705f1..0b6de10334b4 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -44,9 +44,10 @@
 ;;   clgfi %r2, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r5, 4096
-;;   stc %r3, 0(%r5,%r2)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
+;;   lghi %r4, 4096
+;;   stc %r3, 0(%r4,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,9 +61,10 @@
 ;;   clgfi %r2, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r5, 4096
-;;   llc %r2, 0(%r5,%r2)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
+;;   lghi %r3, 4096
+;;   llc %r2, 0(%r3,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
index 05c6839d5969..b1073fcff8b2 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,12 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r4)
+;;   agrk %r4, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435452
-;;   locgrh %r5, %r4
-;;   strv %r3, 0(%r5)
+;;   locgrh %r4, %r5
+;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +55,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r3)
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435452
-;;   locgrh %r5, %r4
-;;   lrv %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
index 2851ab44629e..59ad7b0da23b 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,9 +41,9 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghi %r4, 4096
 ;;   lghi %r5, 0
 ;;   clgfi %r2, 268431356
 ;;   locgrh %r4, %r5
@@ -56,13 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghik %r3, %r5, 4096
-;;   lghi %r5, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r4, %r3, 4096
+;;   lghi %r3, 0
 ;;   clgfi %r2, 268431356
-;;   locgrh %r3, %r5
-;;   lrv %r2, 0(%r3)
+;;   locgrh %r4, %r3
+;;   lrv %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 0df2d11cbba4..ffdcb39a8924 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,12 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r4)
+;;   agrk %r4, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435455
-;;   locgrh %r5, %r4
-;;   stc %r3, 0(%r5)
+;;   locgrh %r4, %r5
+;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +55,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r3)
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435455
-;;   locgrh %r5, %r4
-;;   llc %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
index 5b8b480191b1..9380455a2fb3 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,9 +41,9 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghi %r4, 4096
 ;;   lghi %r5, 0
 ;;   clgfi %r2, 268431359
 ;;   locgrh %r4, %r5
@@ -56,13 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghik %r3, %r5, 4096
-;;   lghi %r5, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r4, %r3, 4096
+;;   lghi %r3, 0
 ;;   clgfi %r2, 268431359
-;;   locgrh %r3, %r5
-;;   llc %r2, 0(%r3)
+;;   locgrh %r4, %r3
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
index b74f04d06f9b..59af6deba717 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0x1000_offset.wat
@@ -44,9 +44,10 @@
 ;;   clgfi %r2, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r5, 4096
-;;   strv %r3, 0(%r5,%r2)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
+;;   lghi %r4, 4096
+;;   strv %r3, 0(%r4,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,9 +61,10 @@
 ;;   clgfi %r2, 268431356
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r5, 4096
-;;   lrv %r2, 0(%r5,%r2)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
+;;   lghi %r3, 4096
+;;   lrv %r2, 0(%r3,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
index 1cc7a766fae8..2a102639d434 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0x1000_offset.wat
@@ -44,9 +44,10 @@
 ;;   clgfi %r2, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r4)
-;;   lghi %r5, 4096
-;;   stc %r3, 0(%r5,%r2)
+;;   lg %r4, 0(%r4)
+;;   agrk %r2, %r4, %r2
+;;   lghi %r4, 4096
+;;   stc %r3, 0(%r4,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
@@ -60,9 +61,10 @@
 ;;   clgfi %r2, 268431359
 ;;   jgh label3 ; jg label1
 ;; block1:
-;;   ag %r2, 0(%r3)
-;;   lghi %r5, 4096
-;;   llc %r2, 0(%r5,%r2)
+;;   lg %r3, 0(%r3)
+;;   agrk %r2, %r3, %r2
+;;   lghi %r3, 4096
+;;   llc %r2, 0(%r3,%r2)
 ;;   jg label2
 ;; block2:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index 486133ed00ff..e5c103ed42a4 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -41,12 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r4)
+;;   agrk %r4, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435452
-;;   locgrh %r5, %r4
-;;   strv %r3, 0(%r5)
+;;   locgrh %r4, %r5
+;;   strv %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +55,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r3)
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435452
-;;   locgrh %r5, %r4
-;;   lrv %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   lrv %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
index a71863212d37..7ac23d4685c7 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0x1000_offset.wat
@@ -41,9 +41,9 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghi %r4, 4096
 ;;   lghi %r5, 0
 ;;   clgfi %r2, 268431356
 ;;   locgrh %r4, %r5
@@ -56,13 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghik %r3, %r5, 4096
-;;   lghi %r5, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r4, %r3, 4096
+;;   lghi %r3, 0
 ;;   clgfi %r2, 268431356
-;;   locgrh %r3, %r5
-;;   lrv %r2, 0(%r3)
+;;   locgrh %r4, %r3
+;;   lrv %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 89aec920be0d..2b8508130ec4 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -41,12 +41,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r4)
+;;   agrk %r4, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435455
-;;   locgrh %r5, %r4
-;;   stc %r3, 0(%r5)
+;;   locgrh %r4, %r5
+;;   stc %r3, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
@@ -55,12 +55,12 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   lghi %r4, 0
+;;   lg %r5, 0(%r3)
+;;   agrk %r3, %r5, %r2
+;;   lghi %r5, 0
 ;;   clgfi %r2, 268435455
-;;   locgrh %r5, %r4
-;;   llc %r2, 0(%r5)
+;;   locgrh %r3, %r5
+;;   llc %r2, 0(%r3)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
index 548feaca8244..e6b78f78abb2 100644
--- a/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
+++ b/cranelift/filetests/filetests/isa/s390x/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0x1000_offset.wat
@@ -41,9 +41,9 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r4)
-;;   aghik %r4, %r5, 4096
+;;   lg %r4, 0(%r4)
+;;   agr %r4, %r2
+;;   aghi %r4, 4096
 ;;   lghi %r5, 0
 ;;   clgfi %r2, 268431359
 ;;   locgrh %r4, %r5
@@ -56,13 +56,13 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 160, offset_downward_to_clobbers: 0 }
 ;;   unwind StackAlloc { size: 0 }
 ;; block0:
-;;   lgr %r5, %r2
-;;   ag %r5, 0(%r3)
-;;   aghik %r3, %r5, 4096
-;;   lghi %r5, 0
+;;   lg %r3, 0(%r3)
+;;   agr %r3, %r2
+;;   aghik %r4, %r3, 4096
+;;   lghi %r3, 0
 ;;   clgfi %r2, 268431359
-;;   locgrh %r3, %r5
-;;   llc %r2, 0(%r3)
+;;   locgrh %r4, %r3
+;;   llc %r2, 0(%r4)
 ;;   jg label1
 ;; block1:
 ;;   br %r14
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 23825be3f765..2d28a4356113 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -45,17 +45,18 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    %r8, %r11
-;;   addq    %r11, const(0), %r11
+;;   movl    %edi, %edi
+;;   movq    %rdi, %rax
+;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %rdi
-;;   cmpq    %rdi, %r11
+;;   movq    8(%rdx), %rcx
+;;   cmpq    %rcx, %rax
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rdx), %r8
-;;   movl    $-65536, %eax
-;;   movl    %esi, 0(%r8,%rax,1)
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
+;;   movl    $-65536, %edx
+;;   movl    %esi, 0(%rcx,%rdx,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -70,17 +71,20 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    %r8, %r11
-;;   addq    %r11, const(0), %r11
+;;   movq    %rsi, %rax
+;;   movl    %edi, %esi
+;;   movq    %rsi, %rdi
+;;   addq    %rdi, const(0), %rdi
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %rdi
-;;   cmpq    %rdi, %r11
+;;   movq    %rax, %rdx
+;;   movq    8(%rdx), %rax
+;;   cmpq    %rax, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rsi), %r8
-;;   movl    $-65536, %eax
-;;   movl    0(%r8,%rax,1), %eax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rsi,1), %rcx
+;;   movl    $-65536, %edx
+;;   movl    0(%rcx,%rdx,1), %eax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
index 187b535a986e..cf9bb34ce3f1 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -45,17 +45,18 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    %r8, %r11
-;;   addq    %r11, const(0), %r11
+;;   movl    %edi, %edi
+;;   movq    %rdi, %rax
+;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %rdi
-;;   cmpq    %rdi, %r11
+;;   movq    8(%rdx), %rcx
+;;   cmpq    %rcx, %rax
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rdx), %r8
-;;   movl    $-65536, %eax
-;;   movb    %sil, 0(%r8,%rax,1)
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
+;;   movl    $-65536, %edx
+;;   movb    %sil, 0(%rcx,%rdx,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -70,17 +71,20 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    %r8, %r11
-;;   addq    %r11, const(0), %r11
+;;   movq    %rsi, %rax
+;;   movl    %edi, %esi
+;;   movq    %rsi, %rdi
+;;   addq    %rdi, const(0), %rdi
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %rdi
-;;   cmpq    %rdi, %r11
+;;   movq    %rax, %rdx
+;;   movq    8(%rdx), %rax
+;;   cmpq    %rax, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rsi), %r8
-;;   movl    $-65536, %eax
-;;   movzbq  0(%r8,%rax,1), %rax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rsi,1), %rcx
+;;   movl    $-65536, %edx
+;;   movzbq  0(%rcx,%rdx,1), %rax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
index c8a51cb693a4..312f4d19d11f 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -46,14 +46,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
 ;;   movl    %edi, %edi
-;;   movq    8(%rdx), %r11
-;;   subq    %r11, $4, %r11
-;;   movq    %rdi, %r10
-;;   addq    %r10, 0(%rdx), %r10
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %r11, %rdi
-;;   cmovnbeq %rax, %r10, %r10
-;;   movl    %esi, 0(%r10)
+;;   movq    8(%rdx), %rax
+;;   subq    %rax, $4, %rax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %r11
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbeq %rcx, %r11, %r11
+;;   movl    %esi, 0(%r11)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -66,16 +66,16 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rsi, %rax
-;;   movl    %edi, %esi
-;;   movq    8(%rax), %r11
-;;   subq    %r11, $4, %r11
-;;   movq    %rsi, %r10
-;;   addq    %r10, 0(%rax), %r10
-;;   xorq    %rdi, %rdi, %rdi
-;;   cmpq    %r11, %rsi
-;;   cmovnbeq %rdi, %r10, %r10
-;;   movl    0(%r10), %eax
+;;   movq    %rsi, %r8
+;;   movl    %edi, %edi
+;;   movq    8(%r8), %rsi
+;;   subq    %rsi, $4, %rsi
+;;   movq    0(%r8), %rax
+;;   lea     0(%rax,%rdi,1), %r11
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %rsi, %rdi
+;;   cmovnbeq %rax, %r11, %r11
+;;   movl    0(%r11), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index dc35c58d198f..1604f80e005a 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -45,18 +45,19 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    %r11, %rax
-;;   addq    %rax, const(0), %rax
+;;   movl    %edi, %r8d
+;;   movq    %r8, %rcx
+;;   addq    %rcx, const(0), %rcx
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %rcx
-;;   addq    %r11, 0(%rdx), %r11
-;;   movl    $-65536, %edx
-;;   lea     0(%r11,%rdx,1), %rdi
+;;   movq    8(%rdx), %r9
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%r8,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rax
 ;;   xorq    %rdx, %rdx, %rdx
-;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rdi, %rdi
-;;   movl    %esi, 0(%rdi)
+;;   cmpq    %r9, %rcx
+;;   cmovnbeq %rdx, %rax, %rax
+;;   movl    %esi, 0(%rax)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -69,18 +70,19 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    %r11, %rax
-;;   addq    %rax, const(0), %rax
+;;   movl    %edi, %r8d
+;;   movq    %r8, %rcx
+;;   addq    %rcx, const(0), %rcx
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %rcx
-;;   addq    %r11, 0(%rsi), %r11
-;;   movl    $-65536, %edx
-;;   lea     0(%r11,%rdx,1), %rdi
-;;   xorq    %rdx, %rdx, %rdx
-;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rdi, %rdi
-;;   movl    0(%rdi), %eax
+;;   movq    8(%rsi), %rdx
+;;   movq    0(%rsi), %r9
+;;   lea     0(%r9,%r8,1), %r8
+;;   movl    $-65536, %r9d
+;;   lea     0(%r8,%r9,1), %rax
+;;   xorq    %r8, %r8, %r8
+;;   cmpq    %rdx, %rcx
+;;   cmovnbeq %r8, %rax, %rax
+;;   movl    0(%rax), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 17a6726b0cc9..cf560a569716 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -45,14 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    8(%rdx), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rdx), %r9
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %rdi, %r11
-;;   cmovnbq %rax, %r9, %r9
-;;   movb    %sil, 0(%r9)
+;;   movl    %edi, %edi
+;;   movq    8(%rdx), %rax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %r10
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbq %rcx, %r10, %r10
+;;   movb    %sil, 0(%r10)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -65,14 +65,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
+;;   movl    %edi, %ecx
 ;;   movq    8(%rsi), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rsi), %r9
-;;   xorq    %rsi, %rsi, %rsi
-;;   cmpq    %rdi, %r11
-;;   cmovnbq %rsi, %r9, %r9
-;;   movzbq  0(%r9), %rax
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rcx,1), %r10
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %rdi, %rcx
+;;   cmovnbq %rax, %r10, %r10
+;;   movzbq  0(%r10), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 171202116deb..a42e7eea29a3 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -45,18 +45,19 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    %r11, %rax
-;;   addq    %rax, const(0), %rax
+;;   movl    %edi, %r8d
+;;   movq    %r8, %rcx
+;;   addq    %rcx, const(0), %rcx
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %rcx
-;;   addq    %r11, 0(%rdx), %r11
-;;   movl    $-65536, %edx
-;;   lea     0(%r11,%rdx,1), %rdi
+;;   movq    8(%rdx), %r9
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%r8,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rax
 ;;   xorq    %rdx, %rdx, %rdx
-;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rdi, %rdi
-;;   movb    %sil, 0(%rdi)
+;;   cmpq    %r9, %rcx
+;;   cmovnbeq %rdx, %rax, %rax
+;;   movb    %sil, 0(%rax)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -69,18 +70,19 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    %r11, %rax
-;;   addq    %rax, const(0), %rax
+;;   movl    %edi, %r8d
+;;   movq    %r8, %rcx
+;;   addq    %rcx, const(0), %rcx
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %rcx
-;;   addq    %r11, 0(%rsi), %r11
-;;   movl    $-65536, %edx
-;;   lea     0(%r11,%rdx,1), %rdi
-;;   xorq    %rdx, %rdx, %rdx
-;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rdi, %rdi
-;;   movzbq  0(%rdi), %rax
+;;   movq    8(%rsi), %rdx
+;;   movq    0(%rsi), %r9
+;;   lea     0(%r9,%r8,1), %r8
+;;   movl    $-65536, %r9d
+;;   lea     0(%r8,%r9,1), %rax
+;;   xorq    %r8, %r8, %r8
+;;   cmpq    %rdx, %rcx
+;;   cmovnbeq %r8, %rax, %rax
+;;   movzbq  0(%rax), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 57e2d1680d46..f08a002320bc 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -45,14 +45,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    8(%rdx), %r10
-;;   cmpq    %r10, %r8
+;;   movl    %edi, %r11d
+;;   movq    8(%rdx), %rdi
+;;   cmpq    %rdi, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rdx), %r8
-;;   movl    $-65536, %edi
-;;   movl    %esi, 0(%r8,%rdi,1)
+;;   movq    0(%rdx), %rdi
+;;   lea     0(%rdi,%r11,1), %rdi
+;;   movl    $-65536, %eax
+;;   movl    %esi, 0(%rdi,%rax,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -67,14 +68,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    8(%rsi), %r10
-;;   cmpq    %r10, %r8
+;;   movl    %edi, %r11d
+;;   movq    8(%rsi), %rdi
+;;   cmpq    %rdi, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rsi), %r8
-;;   movl    $-65536, %esi
-;;   movl    0(%r8,%rsi,1), %eax
+;;   movq    0(%rsi), %rdi
+;;   lea     0(%rdi,%r11,1), %rdi
+;;   movl    $-65536, %eax
+;;   movl    0(%rdi,%rax,1), %eax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
index f2c873612824..04fcb42f1c55 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -45,14 +45,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    8(%rdx), %r10
-;;   cmpq    %r10, %r8
+;;   movl    %edi, %r11d
+;;   movq    8(%rdx), %rdi
+;;   cmpq    %rdi, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rdx), %r8
-;;   movl    $-65536, %edi
-;;   movb    %sil, 0(%r8,%rdi,1)
+;;   movq    0(%rdx), %rdi
+;;   lea     0(%rdi,%r11,1), %rdi
+;;   movl    $-65536, %eax
+;;   movb    %sil, 0(%rdi,%rax,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -67,14 +68,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r8d
-;;   movq    8(%rsi), %r10
-;;   cmpq    %r10, %r8
+;;   movl    %edi, %r11d
+;;   movq    8(%rsi), %rdi
+;;   cmpq    %rdi, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %r8, 0(%rsi), %r8
-;;   movl    $-65536, %esi
-;;   movzbq  0(%r8,%rsi,1), %rax
+;;   movq    0(%rsi), %rdi
+;;   lea     0(%rdi,%r11,1), %rdi
+;;   movl    $-65536, %eax
+;;   movzbq  0(%rdi,%rax,1), %rax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index 89b24d688248..69b4d7120eee 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -45,14 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    8(%rdx), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rdx), %r9
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %rdi, %r11
-;;   cmovnbeq %rax, %r9, %r9
-;;   movl    %esi, 0(%r9)
+;;   movl    %edi, %edi
+;;   movq    8(%rdx), %rax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %r10
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbeq %rcx, %r10, %r10
+;;   movl    %esi, 0(%r10)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -65,14 +65,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
+;;   movl    %edi, %ecx
 ;;   movq    8(%rsi), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rsi), %r9
-;;   xorq    %rsi, %rsi, %rsi
-;;   cmpq    %rdi, %r11
-;;   cmovnbeq %rsi, %r9, %r9
-;;   movl    0(%r9), %eax
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rcx,1), %r10
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %rdi, %rcx
+;;   cmovnbeq %rax, %r10, %r10
+;;   movl    0(%r10), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index 8b6846dfce16..6e2728345df3 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -45,16 +45,16 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %edi
-;;   movq    8(%rdx), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rdx), %rcx
-;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r11
-;;   xorq    %rcx, %rcx, %rcx
-;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r11, %r11
-;;   movl    %esi, 0(%r11)
+;;   movl    %edi, %eax
+;;   movq    8(%rdx), %rcx
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%rax,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
+;;   xorq    %rdx, %rdx, %rdx
+;;   cmpq    %rcx, %rax
+;;   cmovnbeq %rdx, %rdi, %rdi
+;;   movl    %esi, 0(%rdi)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -67,16 +67,16 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %edi
-;;   movq    8(%rsi), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rsi), %rcx
-;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r11
-;;   xorq    %rcx, %rcx, %rcx
-;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r11, %r11
-;;   movl    0(%r11), %eax
+;;   movl    %edi, %eax
+;;   movq    8(%rsi), %rcx
+;;   movq    0(%rsi), %rdx
+;;   lea     0(%rdx,%rax,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rsi
+;;   xorq    %rdx, %rdx, %rdx
+;;   cmpq    %rcx, %rax
+;;   cmovnbeq %rdx, %rsi, %rsi
+;;   movl    0(%rsi), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 1ac80ebbc66a..032a19a9a8bc 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -45,14 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
-;;   movq    8(%rdx), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rdx), %r9
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %rdi, %r11
-;;   cmovnbq %rax, %r9, %r9
-;;   movb    %sil, 0(%r9)
+;;   movl    %edi, %edi
+;;   movq    8(%rdx), %rax
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %r10
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbq %rcx, %r10, %r10
+;;   movb    %sil, 0(%r10)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -65,14 +65,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r11d
+;;   movl    %edi, %ecx
 ;;   movq    8(%rsi), %rdi
-;;   movq    %r11, %r9
-;;   addq    %r9, 0(%rsi), %r9
-;;   xorq    %rsi, %rsi, %rsi
-;;   cmpq    %rdi, %r11
-;;   cmovnbq %rsi, %r9, %r9
-;;   movzbq  0(%r9), %rax
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rcx,1), %r10
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %rdi, %rcx
+;;   cmovnbq %rax, %r10, %r10
+;;   movzbq  0(%r10), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 5ac256e173d3..3004cd718888 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i32_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -45,16 +45,16 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %edi
-;;   movq    8(%rdx), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rdx), %rcx
-;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r11
-;;   xorq    %rcx, %rcx, %rcx
-;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r11, %r11
-;;   movb    %sil, 0(%r11)
+;;   movl    %edi, %eax
+;;   movq    8(%rdx), %rcx
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%rax,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
+;;   xorq    %rdx, %rdx, %rdx
+;;   cmpq    %rcx, %rax
+;;   cmovnbeq %rdx, %rdi, %rdi
+;;   movb    %sil, 0(%rdi)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -67,16 +67,16 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %edi
-;;   movq    8(%rsi), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rsi), %rcx
-;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r11
-;;   xorq    %rcx, %rcx, %rcx
-;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r11, %r11
-;;   movzbq  0(%r11), %rax
+;;   movl    %edi, %eax
+;;   movq    8(%rsi), %rcx
+;;   movq    0(%rsi), %rdx
+;;   lea     0(%rdx,%rax,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rsi
+;;   xorq    %rdx, %rdx, %rdx
+;;   cmpq    %rcx, %rax
+;;   cmovnbeq %rdx, %rsi, %rsi
+;;   movzbq  0(%rsi), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
index 606ac690204d..308f142069b5 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -45,16 +45,17 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r10
-;;   addq    %r10, const(0), %r10
+;;   movq    %rdi, %r11
+;;   addq    %r11, const(0), %r11
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %r11
-;;   cmpq    %r11, %r10
+;;   movq    8(%rdx), %rax
+;;   cmpq    %rax, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %eax
-;;   movl    %esi, 0(%rdi,%rax,1)
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %rax
+;;   movl    $-65536, %ecx
+;;   movl    %esi, 0(%rax,%rcx,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -69,16 +70,17 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r10
-;;   addq    %r10, const(0), %r10
+;;   movq    %rdi, %r11
+;;   addq    %r11, const(0), %r11
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %r11
-;;   cmpq    %r11, %r10
+;;   movq    8(%rsi), %rax
+;;   cmpq    %rax, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %eax
-;;   movl    0(%rdi,%rax,1), %eax
+;;   movq    0(%rsi), %rax
+;;   lea     0(%rax,%rdi,1), %rax
+;;   movl    $-65536, %ecx
+;;   movl    0(%rax,%rcx,1), %eax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
index 1253391f756b..36c6ccde055f 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -45,16 +45,17 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r10
-;;   addq    %r10, const(0), %r10
+;;   movq    %rdi, %r11
+;;   addq    %r11, const(0), %r11
 ;;   jb #trap=heap_oob
-;;   movq    8(%rdx), %r11
-;;   cmpq    %r11, %r10
+;;   movq    8(%rdx), %rax
+;;   cmpq    %rax, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %eax
-;;   movb    %sil, 0(%rdi,%rax,1)
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %rax
+;;   movl    $-65536, %ecx
+;;   movb    %sil, 0(%rax,%rcx,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -69,16 +70,17 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r10
-;;   addq    %r10, const(0), %r10
+;;   movq    %rdi, %r11
+;;   addq    %r11, const(0), %r11
 ;;   jb #trap=heap_oob
-;;   movq    8(%rsi), %r11
-;;   cmpq    %r11, %r10
+;;   movq    8(%rsi), %rax
+;;   cmpq    %rax, %r11
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %eax
-;;   movzbq  0(%rdi,%rax,1), %rax
+;;   movq    0(%rsi), %rax
+;;   lea     0(%rax,%rdi,1), %rax
+;;   movl    $-65536, %ecx
+;;   movzbq  0(%rax,%rcx,1), %rax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
index 7208a2984645..d0a2d3ab3d01 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -45,14 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r10
-;;   subq    %r10, $4, %r10
-;;   movq    %rdi, %r9
-;;   addq    %r9, 0(%rdx), %r9
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbeq %r11, %r9, %r9
-;;   movl    %esi, 0(%r9)
+;;   movq    8(%rdx), %r11
+;;   subq    %r11, $4, %r11
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %r10
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %r11, %rdi
+;;   cmovnbeq %rax, %r10, %r10
+;;   movl    %esi, 0(%r10)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -65,14 +65,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r10
-;;   subq    %r10, $4, %r10
-;;   movq    %rdi, %r9
-;;   addq    %r9, 0(%rsi), %r9
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbeq %r11, %r9, %r9
-;;   movl    0(%r9), %eax
+;;   movq    8(%rsi), %r11
+;;   subq    %r11, $4, %r11
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %r10
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    %r11, %rdi
+;;   cmovnbeq %rsi, %r10, %r10
+;;   movl    0(%r10), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index ca713ccc731b..a70d1cb486f0 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -49,9 +49,10 @@
 ;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
 ;;   movq    8(%rdx), %rcx
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %edx
-;;   lea     0(%rdi,%rdx,1), %rdi
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%rdi,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
 ;;   xorq    %rdx, %rdx, %rdx
 ;;   cmpq    %rcx, %rax
 ;;   cmovnbeq %rdx, %rdi, %rdi
@@ -72,13 +73,14 @@
 ;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
 ;;   movq    8(%rsi), %rcx
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %edx
-;;   lea     0(%rdi,%rdx,1), %rsi
+;;   movq    0(%rsi), %rdx
+;;   lea     0(%rdx,%rdi,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
 ;;   xorq    %rdx, %rdx, %rdx
 ;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rsi, %rsi
-;;   movl    0(%rsi), %eax
+;;   cmovnbeq %rdx, %rdi, %rdi
+;;   movl    0(%rdi), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
index b2f45beb12eb..cca108fb4fd5 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -45,13 +45,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbq %r11, %r8, %r8
-;;   movb    %sil, 0(%r8)
+;;   movq    8(%rdx), %r11
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %r9
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %r11, %rdi
+;;   cmovnbq %rax, %r9, %r9
+;;   movb    %sil, 0(%r9)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -64,13 +64,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rsi), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbq %r11, %r8, %r8
-;;   movzbq  0(%r8), %rax
+;;   movq    8(%rsi), %r11
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %r9
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    %r11, %rdi
+;;   cmovnbq %rsi, %r9, %r9
+;;   movzbq  0(%r9), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index c0ae962583e7..a20b2fca5c61 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -49,9 +49,10 @@
 ;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
 ;;   movq    8(%rdx), %rcx
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %edx
-;;   lea     0(%rdi,%rdx,1), %rdi
+;;   movq    0(%rdx), %rdx
+;;   lea     0(%rdx,%rdi,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
 ;;   xorq    %rdx, %rdx, %rdx
 ;;   cmpq    %rcx, %rax
 ;;   cmovnbeq %rdx, %rdi, %rdi
@@ -72,13 +73,14 @@
 ;;   addq    %rax, const(0), %rax
 ;;   jb #trap=heap_oob
 ;;   movq    8(%rsi), %rcx
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %edx
-;;   lea     0(%rdi,%rdx,1), %rsi
+;;   movq    0(%rsi), %rdx
+;;   lea     0(%rdx,%rdi,1), %rdx
+;;   movl    $-65536, %r8d
+;;   lea     0(%rdx,%r8,1), %rdi
 ;;   xorq    %rdx, %rdx, %rdx
 ;;   cmpq    %rcx, %rax
-;;   cmovnbeq %rdx, %rsi, %rsi
-;;   movzbq  0(%rsi), %rax
+;;   cmovnbeq %rdx, %rdi, %rdi
+;;   movzbq  0(%rdi), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
index e0f869ff951d..b444dc60178a 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i32_access_0xffff0000_offset.wat
@@ -45,13 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r9
-;;   cmpq    %r9, %rdi
+;;   movq    8(%rdx), %r10
+;;   cmpq    %r10, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %r11d
-;;   movl    %esi, 0(%rdi,%r11,1)
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %rdi
+;;   movl    $-65536, %eax
+;;   movl    %esi, 0(%rdi,%rax,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -66,13 +67,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r9
-;;   cmpq    %r9, %rdi
+;;   movq    8(%rsi), %r10
+;;   cmpq    %r10, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %r11d
-;;   movl    0(%rdi,%r11,1), %eax
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %rsi
+;;   movl    $-65536, %edi
+;;   movl    0(%rsi,%rdi,1), %eax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
index 0a8abfc3ceb3..7dc38fadf04f 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_no_spectre_i8_access_0xffff0000_offset.wat
@@ -45,13 +45,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r9
-;;   cmpq    %r9, %rdi
+;;   movq    8(%rdx), %r10
+;;   cmpq    %r10, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rdx), %rdi
-;;   movl    $-65536, %r11d
-;;   movb    %sil, 0(%rdi,%r11,1)
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %rdi
+;;   movl    $-65536, %eax
+;;   movb    %sil, 0(%rdi,%rax,1)
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
@@ -66,13 +67,14 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r9
-;;   cmpq    %r9, %rdi
+;;   movq    8(%rsi), %r10
+;;   cmpq    %r10, %rdi
 ;;   jnbe    label3; j label1
 ;; block1:
-;;   addq    %rdi, 0(%rsi), %rdi
-;;   movl    $-65536, %r11d
-;;   movzbq  0(%rdi,%r11,1), %rax
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %rsi
+;;   movl    $-65536, %edi
+;;   movzbq  0(%rsi,%rdi,1), %rax
 ;;   jmp     label2
 ;; block2:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index c7c43a0d3f9e..1e36556796db 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -45,13 +45,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbeq %r11, %r8, %r8
-;;   movl    %esi, 0(%r8)
+;;   movq    8(%rdx), %r11
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %r9
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %r11, %rdi
+;;   cmovnbeq %rax, %r9, %r9
+;;   movl    %esi, 0(%r9)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -64,13 +64,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rsi), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbeq %r11, %r8, %r8
-;;   movl    0(%r8), %eax
+;;   movq    8(%rsi), %r11
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %r9
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    %r11, %rdi
+;;   cmovnbeq %rsi, %r9, %r9
+;;   movl    0(%r9), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
index 3320b9fa3c4c..454ece1fa7f6 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0xffff0000_offset.wat
@@ -46,14 +46,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
 ;;   movq    8(%rdx), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rdx), %rcx
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
 ;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r10
+;;   lea     0(%rcx,%rdx,1), %r11
 ;;   xorq    %rcx, %rcx, %rcx
 ;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r10, %r10
-;;   movl    %esi, 0(%r10)
+;;   cmovnbeq %rcx, %r11, %r11
+;;   movl    %esi, 0(%r11)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -66,15 +66,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %rcx
-;;   movq    %rdi, %rax
-;;   addq    %rax, 0(%rsi), %rax
-;;   movl    $-65536, %esi
-;;   lea     0(%rax,%rsi,1), %r10
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %rcx, %rdi
-;;   cmovnbeq %rax, %r10, %r10
-;;   movl    0(%r10), %eax
+;;   movq    8(%rsi), %rax
+;;   movq    0(%rsi), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
+;;   movl    $-65536, %edx
+;;   lea     0(%rcx,%rdx,1), %r11
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbeq %rcx, %r11, %r11
+;;   movl    0(%r11), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 449f63560bf7..bb5d00619029 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -45,13 +45,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rdx), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbq %r11, %r8, %r8
-;;   movb    %sil, 0(%r8)
+;;   movq    8(%rdx), %r11
+;;   movq    0(%rdx), %rax
+;;   lea     0(%rax,%rdi,1), %r9
+;;   xorq    %rax, %rax, %rax
+;;   cmpq    %r11, %rdi
+;;   cmovnbq %rax, %r9, %r9
+;;   movb    %sil, 0(%r9)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -64,13 +64,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %r10
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rsi), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    %r10, %rdi
-;;   cmovnbq %r11, %r8, %r8
-;;   movzbq  0(%r8), %rax
+;;   movq    8(%rsi), %r11
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%rdi,1), %r9
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    %r11, %rdi
+;;   cmovnbq %rsi, %r9, %r9
+;;   movzbq  0(%r9), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
index 214b6e06f87f..16ecad9dfd11 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_dynamic_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0xffff0000_offset.wat
@@ -46,14 +46,14 @@
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
 ;;   movq    8(%rdx), %rax
-;;   movq    %rdi, %rcx
-;;   addq    %rcx, 0(%rdx), %rcx
+;;   movq    0(%rdx), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
 ;;   movl    $-65536, %edx
-;;   lea     0(%rcx,%rdx,1), %r10
+;;   lea     0(%rcx,%rdx,1), %r11
 ;;   xorq    %rcx, %rcx, %rcx
 ;;   cmpq    %rax, %rdi
-;;   cmovnbeq %rcx, %r10, %r10
-;;   movb    %sil, 0(%r10)
+;;   cmovnbeq %rcx, %r11, %r11
+;;   movb    %sil, 0(%r11)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -66,15 +66,15 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    8(%rsi), %rcx
-;;   movq    %rdi, %rax
-;;   addq    %rax, 0(%rsi), %rax
-;;   movl    $-65536, %esi
-;;   lea     0(%rax,%rsi,1), %r10
-;;   xorq    %rax, %rax, %rax
-;;   cmpq    %rcx, %rdi
-;;   cmovnbeq %rax, %r10, %r10
-;;   movzbq  0(%r10), %rax
+;;   movq    8(%rsi), %rax
+;;   movq    0(%rsi), %rcx
+;;   lea     0(%rcx,%rdi,1), %rcx
+;;   movl    $-65536, %edx
+;;   lea     0(%rcx,%rdx,1), %r11
+;;   xorq    %rcx, %rcx, %rcx
+;;   cmpq    %rax, %rdi
+;;   cmovnbeq %rcx, %r11, %r11
+;;   movzbq  0(%r11), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
index 8066a12155e7..ac8ed3a38403 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -43,13 +43,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r10d
-;;   movq    %r10, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    $268435452, %r10
-;;   cmovnbeq %r11, %r8, %r8
-;;   movl    %esi, 0(%r8)
+;;   movl    %edi, %r11d
+;;   movq    0(%rdx), %rdi
+;;   lea     0(%rdi,%r11,1), %r9
+;;   xorq    %rdi, %rdi, %rdi
+;;   cmpq    $268435452, %r11
+;;   cmovnbeq %rdi, %r9, %r9
+;;   movl    %esi, 0(%r9)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -62,13 +62,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r10d
-;;   movq    %r10, %r8
-;;   addq    %r8, 0(%rsi), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    $268435452, %r10
-;;   cmovnbeq %r11, %r8, %r8
-;;   movl    0(%r8), %eax
+;;   movl    %edi, %r11d
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%r11,1), %r9
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    $268435452, %r11
+;;   cmovnbeq %rsi, %r9, %r9
+;;   movl    0(%r9), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 93d48967ab97..cdfe8b68d46a 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i32_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -43,13 +43,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r10d
-;;   movq    %r10, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    $268435455, %r10
-;;   cmovnbeq %r11, %r8, %r8
-;;   movb    %sil, 0(%r8)
+;;   movl    %edi, %r11d
+;;   movq    0(%rdx), %rdi
+;;   lea     0(%rdi,%r11,1), %r9
+;;   xorq    %rdi, %rdi, %rdi
+;;   cmpq    $268435455, %r11
+;;   cmovnbeq %rdi, %r9, %r9
+;;   movb    %sil, 0(%r9)
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
@@ -62,13 +62,13 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movl    %edi, %r10d
-;;   movq    %r10, %r8
-;;   addq    %r8, 0(%rsi), %r8
-;;   xorq    %r11, %r11, %r11
-;;   cmpq    $268435455, %r10
-;;   cmovnbeq %r11, %r8, %r8
-;;   movzbq  0(%r8), %rax
+;;   movl    %edi, %r11d
+;;   movq    0(%rsi), %rsi
+;;   lea     0(%rsi,%r11,1), %r9
+;;   xorq    %rsi, %rsi, %rsi
+;;   cmpq    $268435455, %r11
+;;   cmovnbeq %rsi, %r9, %r9
+;;   movzbq  0(%r9), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
index c0113a35f739..3d96a293536a 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i32_access_0_offset.wat
@@ -43,11 +43,11 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rdx), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435452, %rdi
-;;   cmovnbeq %r9, %r8, %r8
+;;   cmovnbeq %r10, %r8, %r8
 ;;   movl    %esi, 0(%r8)
 ;;   jmp     label1
 ;; block1:
@@ -61,12 +61,12 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %rdx
-;;   addq    %rdx, 0(%rsi), %rdx
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rsi), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435452, %rdi
-;;   cmovnbeq %r9, %rdx, %rdx
-;;   movl    0(%rdx), %eax
+;;   cmovnbeq %r10, %r8, %r8
+;;   movl    0(%r8), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
index 6bf8e9cc8a0d..c9cc5db35a71 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0_guard_yes_spectre_i8_access_0_offset.wat
@@ -43,11 +43,11 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rdx), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435455, %rdi
-;;   cmovnbeq %r9, %r8, %r8
+;;   cmovnbeq %r10, %r8, %r8
 ;;   movb    %sil, 0(%r8)
 ;;   jmp     label1
 ;; block1:
@@ -61,12 +61,12 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %rdx
-;;   addq    %rdx, 0(%rsi), %rdx
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rsi), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435455, %rdi
-;;   cmovnbeq %r9, %rdx, %rdx
-;;   movzbq  0(%rdx), %rax
+;;   cmovnbeq %r10, %r8, %r8
+;;   movzbq  0(%r8), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
index c58eeb9a84b1..4e2897db6dcd 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i32_access_0_offset.wat
@@ -43,11 +43,11 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rdx), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435452, %rdi
-;;   cmovnbeq %r9, %r8, %r8
+;;   cmovnbeq %r10, %r8, %r8
 ;;   movl    %esi, 0(%r8)
 ;;   jmp     label1
 ;; block1:
@@ -61,12 +61,12 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %rdx
-;;   addq    %rdx, 0(%rsi), %rdx
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rsi), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435452, %rdi
-;;   cmovnbeq %r9, %rdx, %rdx
-;;   movl    0(%rdx), %eax
+;;   cmovnbeq %r10, %r8, %r8
+;;   movl    0(%r8), %eax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
index 421d738d86b7..ba3e20f79317 100644
--- a/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
+++ b/cranelift/filetests/filetests/isa/x64/wasm/load_store_static_kind_i64_index_0xffffffff_guard_yes_spectre_i8_access_0_offset.wat
@@ -43,11 +43,11 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %r8
-;;   addq    %r8, 0(%rdx), %r8
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rdx), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435455, %rdi
-;;   cmovnbeq %r9, %r8, %r8
+;;   cmovnbeq %r10, %r8, %r8
 ;;   movb    %sil, 0(%r8)
 ;;   jmp     label1
 ;; block1:
@@ -61,12 +61,12 @@
 ;;   movq    %rsp, %rbp
 ;;   unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 0 }
 ;; block0:
-;;   movq    %rdi, %rdx
-;;   addq    %rdx, 0(%rsi), %rdx
-;;   xorq    %r9, %r9, %r9
+;;   movq    0(%rsi), %r10
+;;   lea     0(%r10,%rdi,1), %r8
+;;   xorq    %r10, %r10, %r10
 ;;   cmpq    $268435455, %rdi
-;;   cmovnbeq %r9, %rdx, %rdx
-;;   movzbq  0(%rdx), %rax
+;;   cmovnbeq %r10, %r8, %r8
+;;   movzbq  0(%r8), %rax
 ;;   jmp     label1
 ;; block1:
 ;;   movq    %rbp, %rsp
diff --git a/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif
index 4404386af90a..dace18d7054f 100644
--- a/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif
+++ b/cranelift/filetests/filetests/legalizer/back-to-back-legalize.clif
@@ -11,8 +11,10 @@ block0:
 
 ; check: v1 = iconst.i32 1
 ; check: v4 = iconst.i32 1
-; check: v2 = iadd v1, v4
-; check: v5 = iconst.i32 1
-; check: v3 = iadd v2, v5
+; check: v5 = iadd v1, v4
+; check: v2 -> v5
+; check: v6 = iconst.i32 1
+; check: v7 = iadd v2, v6
+; check: v3 -> v7
 ; check: return v3
 
diff --git a/cranelift/filetests/filetests/legalizer/bxor_imm.clif b/cranelift/filetests/filetests/legalizer/bxor_imm.clif
index 12a149eed9c0..12a803732c54 100644
--- a/cranelift/filetests/filetests/legalizer/bxor_imm.clif
+++ b/cranelift/filetests/filetests/legalizer/bxor_imm.clif
@@ -9,4 +9,6 @@ block0(v0: i64, v1: i64):
 }
 
 ; check: v3 = iconst.i64 0x0001_0000_0000
-; check: v2 = bxor v0, v3
+; check: v4 = bxor v0, v3
+; check: v2 -> v4
+; check: return v2
diff --git a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
index 6bc1b88e0659..69eb80d2d5ef 100644
--- a/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
+++ b/cranelift/filetests/filetests/legalizer/canonicalize-nans.clif
@@ -13,7 +13,8 @@ block0(v0: f32):
 ;     v2 = ceil v0
 ;     v3 = fcmp ne v2, v2
 ;     v4 = f32const +NaN
-;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     v5 = select v3, v4, v2  ; v4 = +NaN
+;     v1 -> v5
 ;     return v1
 ; }
 
@@ -28,7 +29,8 @@ block0(v0: f64):
 ;     v2 = floor v0
 ;     v3 = fcmp ne v2, v2
 ;     v4 = f64const +NaN
-;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     v5 = select v3, v4, v2  ; v4 = +NaN
+;     v1 -> v5
 ;     return v1
 ; }
 
@@ -45,7 +47,8 @@ block0(v0: f32x4):
 ;     v4 = f32const +NaN
 ;     v5 = splat.f32x4 v4  ; v4 = +NaN
 ;     v6 = bitcast.f32x4 v3
-;     v1 = bitselect v6, v5, v2
+;     v7 = bitselect v6, v5, v2
+;     v1 -> v7
 ;     return v1
 ; }
 
@@ -62,7 +65,8 @@ block0(v0: f64x2):
 ;     v4 = splat.f64x2 v3  ; v3 = +NaN
 ;     v5 = fcmp ne v2, v2
 ;     v6 = bitcast.f64x2 v5
-;     v1 = bitselect v6, v4, v2
+;     v7 = bitselect v6, v4, v2
+;     v1 -> v7
 ;     return v1
 ; }
 
@@ -77,7 +81,8 @@ block0(v0: f32):
 ;     v2 = trunc v0
 ;     v3 = fcmp ne v2, v2
 ;     v4 = f32const +NaN
-;     v1 = select v3, v4, v2  ; v4 = +NaN
+;     v5 = select v3, v4, v2  ; v4 = +NaN
+;     v1 -> v5
 ;     return v1
 ; }
 
@@ -92,7 +97,8 @@ block0(v0: f32, v1: f32):
 ;     v3 = fadd v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f32const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -107,7 +113,8 @@ block0(v0: f32, v1: f32):
 ;     v3 = fsub v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f32const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -122,7 +129,8 @@ block0(v0: f64, v1: f64):
 ;     v3 = fdiv v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f64const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -137,7 +145,8 @@ block0(v0: f64, v1: f64):
 ;     v3 = fmul v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f64const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -152,7 +161,8 @@ block0(v0: f64, v1: f64):
 ;     v3 = fmax v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f64const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -167,7 +177,8 @@ block0(v0: f64, v1: f64):
 ;     v3 = fmin v0, v1
 ;     v4 = fcmp ne v3, v3
 ;     v5 = f64const +NaN
-;     v2 = select v4, v5, v3  ; v5 = +NaN
+;     v6 = select v4, v5, v3  ; v5 = +NaN
+;     v2 -> v6
 ;     return v2
 ; }
 
@@ -182,7 +193,8 @@ block0(v0: f64, v1: f64, v2: f64):
 ;     v4 = fma v0, v1, v2
 ;     v5 = fcmp ne v4, v4
 ;     v6 = f64const +NaN
-;     v3 = select v5, v6, v4  ; v6 = +NaN
+;     v7 = select v5, v6, v4  ; v6 = +NaN
+;     v3 -> v7
 ;     return v3
 ; }
 
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat
index 524f7eb83657..6efaa39f97bd 100644
--- a/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat
+++ b/cranelift/filetests/filetests/wasm/duplicate-loads-dynamic-memory.wat
@@ -49,15 +49,17 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v20 -> v1
-;;                                     v21 -> v1
 ;;                                     v22 -> v1
-;;                                     v23 -> v1
-;; @0057                               v5 = load.i64 notrap aligned v1+8
-;; @0057                               v7 = load.i64 notrap aligned v1
+;;                                     v24 -> v1
+;;                                     v26 -> v1
+;;                                     v21 = load.i64 notrap aligned v1+8
+;;                                     v5 -> v21
+;;                                     v23 = load.i64 notrap aligned v1
+;;                                     v7 -> v23
 ;; @0057                               v4 = uextend.i64 v0
-;; @0057                               v6 = icmp ugt v4, v5
+;; @0057                               v6 = icmp ugt v4, v21
 ;; @0057                               v9 = iconst.i64 0
-;; @0057                               v8 = iadd v7, v4
+;; @0057                               v8 = iadd v23, v4
 ;; @0057                               v10 = select_spectre_guard v6, v9, v8  ; v9 = 0
 ;; @0057                               v11 = load.i32 little heap v10
 ;;                                     v2 -> v11
@@ -74,15 +76,17 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v24 -> v1
-;;                                     v25 -> v1
 ;;                                     v26 -> v1
-;;                                     v27 -> v1
-;; @0064                               v5 = load.i64 notrap aligned v1+8
-;; @0064                               v7 = load.i64 notrap aligned v1
+;;                                     v28 -> v1
+;;                                     v30 -> v1
+;;                                     v25 = load.i64 notrap aligned v1+8
+;;                                     v5 -> v25
+;;                                     v27 = load.i64 notrap aligned v1
+;;                                     v7 -> v27
 ;; @0064                               v4 = uextend.i64 v0
-;; @0064                               v6 = icmp ugt v4, v5
+;; @0064                               v6 = icmp ugt v4, v25
 ;; @0064                               v11 = iconst.i64 0
-;; @0064                               v8 = iadd v7, v4
+;; @0064                               v8 = iadd v27, v4
 ;; @0064                               v9 = iconst.i64 1234
 ;; @0064                               v10 = iadd v8, v9  ; v9 = 1234
 ;; @0064                               v12 = select_spectre_guard v6, v11, v10  ; v11 = 0
diff --git a/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat b/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat
index c63173922333..e103b3b33084 100644
--- a/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat
+++ b/cranelift/filetests/filetests/wasm/duplicate-loads-static-memory.wat
@@ -44,10 +44,11 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v12 -> v1
-;;                                     v13 -> v1
-;; @0057                               v5 = load.i64 notrap aligned readonly v1
+;;                                     v14 -> v1
+;;                                     v13 = load.i64 notrap aligned readonly v1
+;;                                     v5 -> v13
 ;; @0057                               v4 = uextend.i64 v0
-;; @0057                               v6 = iadd v5, v4
+;; @0057                               v6 = iadd v13, v4
 ;; @0057                               v7 = load.i32 little heap v6
 ;;                                     v2 -> v7
 ;; @005f                               jump block1
@@ -62,10 +63,11 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v16 -> v1
-;;                                     v17 -> v1
-;; @0064                               v5 = load.i64 notrap aligned readonly v1
+;;                                     v18 -> v1
+;;                                     v17 = load.i64 notrap aligned readonly v1
+;;                                     v5 -> v17
 ;; @0064                               v4 = uextend.i64 v0
-;; @0064                               v6 = iadd v5, v4
+;; @0064                               v6 = iadd v17, v4
 ;; @0064                               v7 = iconst.i64 1234
 ;; @0064                               v8 = iadd v6, v7  ; v7 = 1234
 ;; @0064                               v9 = load.i32 little heap v8
diff --git a/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat b/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat
index f7f5f72bce6a..a0879e2ced8e 100644
--- a/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat
+++ b/cranelift/filetests/filetests/wasm/dynamic-memory-no-spectre-access-same-index-different-offsets.wat
@@ -62,22 +62,24 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v29 -> v1
-;;                                     v30 -> v1
 ;;                                     v31 -> v1
-;;                                     v32 -> v1
 ;;                                     v33 -> v1
-;;                                     v34 -> v1
-;; @0047                               v6 = load.i64 notrap aligned v1+8
+;;                                     v35 -> v1
+;;                                     v37 -> v1
+;;                                     v39 -> v1
+;;                                     v30 = load.i64 notrap aligned v1+8
+;;                                     v6 -> v30
 ;; @0047                               v5 = uextend.i64 v0
-;; @0047                               v7 = icmp ugt v5, v6
+;; @0047                               v7 = icmp ugt v5, v30
 ;; @0047                               brif v7, block2, block3
 ;;
 ;;                                 block2 cold:
 ;; @0047                               trap heap_oob
 ;;
 ;;                                 block3:
-;; @0047                               v8 = load.i64 notrap aligned v1
-;; @0047                               v9 = iadd v8, v5
+;;                                     v32 = load.i64 notrap aligned v1
+;;                                     v8 -> v32
+;; @0047                               v9 = iadd v32, v5
 ;; @0047                               v10 = load.i32 little heap v9
 ;;                                     v2 -> v10
 ;; @004c                               brif.i8 v7, block4, block5
@@ -92,15 +94,16 @@
 ;;                                     v3 -> v18
 ;; @0051                               v20 = iconst.i64 0x0010_0003
 ;; @0051                               v21 = uadd_overflow_trap.i64 v5, v20, heap_oob  ; v20 = 0x0010_0003
-;; @0051                               v23 = icmp ugt v21, v6
+;; @0051                               v23 = icmp ugt v21, v30
 ;; @0051                               brif v23, block6, block7
 ;;
 ;;                                 block6 cold:
 ;; @0051                               trap heap_oob
 ;;
 ;;                                 block7:
-;; @0051                               v24 = load.i64 notrap aligned v1
-;; @0051                               v25 = iadd v24, v5
+;;                                     v40 = load.i64 notrap aligned v1
+;;                                     v24 -> v40
+;; @0051                               v25 = iadd v40, v5
 ;; @0051                               v26 = iconst.i64 0x000f_ffff
 ;; @0051                               v27 = iadd v25, v26  ; v26 = 0x000f_ffff
 ;; @0051                               v28 = load.i32 little heap v27
@@ -118,22 +121,24 @@
 ;;
 ;;                                 block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i64):
 ;;                                     v26 -> v4
-;;                                     v27 -> v4
 ;;                                     v28 -> v4
-;;                                     v29 -> v4
 ;;                                     v30 -> v4
-;;                                     v31 -> v4
-;; @005d                               v6 = load.i64 notrap aligned v4+8
+;;                                     v32 -> v4
+;;                                     v34 -> v4
+;;                                     v36 -> v4
+;;                                     v27 = load.i64 notrap aligned v4+8
+;;                                     v6 -> v27
 ;; @005d                               v5 = uextend.i64 v0
-;; @005d                               v7 = icmp ugt v5, v6
+;; @005d                               v7 = icmp ugt v5, v27
 ;; @005d                               brif v7, block2, block3
 ;;
 ;;                                 block2 cold:
 ;; @005d                               trap heap_oob
 ;;
 ;;                                 block3:
-;; @005d                               v8 = load.i64 notrap aligned v4
-;; @005d                               v9 = iadd v8, v5
+;;                                     v29 = load.i64 notrap aligned v4
+;;                                     v8 -> v29
+;; @005d                               v9 = iadd v29, v5
 ;; @005d                               store.i32 little heap v1, v9
 ;; @0064                               brif.i8 v7, block4, block5
 ;;
@@ -146,15 +151,16 @@
 ;; @0064                               store.i32 little heap v2, v16
 ;; @006b                               v18 = iconst.i64 0x0010_0003
 ;; @006b                               v19 = uadd_overflow_trap.i64 v5, v18, heap_oob  ; v18 = 0x0010_0003
-;; @006b                               v21 = icmp ugt v19, v6
+;; @006b                               v21 = icmp ugt v19, v27
 ;; @006b                               brif v21, block6, block7
 ;;
 ;;                                 block6 cold:
 ;; @006b                               trap heap_oob
 ;;
 ;;                                 block7:
-;; @006b                               v22 = load.i64 notrap aligned v4
-;; @006b                               v23 = iadd v22, v5
+;;                                     v37 = load.i64 notrap aligned v4
+;;                                     v22 -> v37
+;; @006b                               v23 = iadd v37, v5
 ;; @006b                               v24 = iconst.i64 0x000f_ffff
 ;; @006b                               v25 = iadd v23, v24  ; v24 = 0x000f_ffff
 ;; @006b                               store.i32 little heap v3, v25
diff --git a/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat b/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat
index 55e4b6b67f7f..226b54067ea6 100644
--- a/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat
+++ b/cranelift/filetests/filetests/wasm/dynamic-memory-yes-spectre-access-same-index-different-offsets.wat
@@ -62,17 +62,19 @@
 ;;
 ;;                                 block0(v0: i32, v1: i64):
 ;;                                     v35 -> v1
-;;                                     v36 -> v1
 ;;                                     v37 -> v1
-;;                                     v38 -> v1
 ;;                                     v39 -> v1
-;;                                     v40 -> v1
-;; @0047                               v6 = load.i64 notrap aligned v1+8
-;; @0047                               v8 = load.i64 notrap aligned v1
+;;                                     v41 -> v1
+;;                                     v43 -> v1
+;;                                     v45 -> v1
+;;                                     v36 = load.i64 notrap aligned v1+8
+;;                                     v6 -> v36
+;;                                     v38 = load.i64 notrap aligned v1
+;;                                     v8 -> v38
 ;; @0047                               v5 = uextend.i64 v0
-;; @0047                               v7 = icmp ugt v5, v6
+;; @0047                               v7 = icmp ugt v5, v36
 ;; @0047                               v10 = iconst.i64 0
-;; @0047                               v9 = iadd v8, v5
+;; @0047                               v9 = iadd v38, v5
 ;; @0047                               v11 = select_spectre_guard v7, v10, v9  ; v10 = 0
 ;; @0047                               v12 = load.i32 little heap v11
 ;;                                     v2 -> v12
@@ -83,7 +85,7 @@
 ;;                                     v3 -> v22
 ;; @0051                               v24 = iconst.i64 0x0010_0003
 ;; @0051                               v25 = uadd_overflow_trap v5, v24, heap_oob  ; v24 = 0x0010_0003
-;; @0051                               v27 = icmp ugt v25, v6
+;; @0051                               v27 = icmp ugt v25, v36
 ;; @0051                               v30 = iconst.i64 0x000f_ffff
 ;; @0051                               v31 = iadd v9, v30  ; v30 = 0x000f_ffff
 ;; @0051                               v33 = select_spectre_guard v27, v10, v31  ; v10 = 0
@@ -102,17 +104,19 @@
 ;;
 ;;                                 block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i64):
 ;;                                     v32 -> v4
-;;                                     v33 -> v4
 ;;                                     v34 -> v4
-;;                                     v35 -> v4
 ;;                                     v36 -> v4
-;;                                     v37 -> v4
-;; @005d                               v6 = load.i64 notrap aligned v4+8
-;; @005d                               v8 = load.i64 notrap aligned v4
+;;                                     v38 -> v4
+;;                                     v40 -> v4
+;;                                     v42 -> v4
+;;                                     v33 = load.i64 notrap aligned v4+8
+;;                                     v6 -> v33
+;;                                     v35 = load.i64 notrap aligned v4
+;;                                     v8 -> v35
 ;; @005d                               v5 = uextend.i64 v0
-;; @005d                               v7 = icmp ugt v5, v6
+;; @005d                               v7 = icmp ugt v5, v33
 ;; @005d                               v10 = iconst.i64 0
-;; @005d                               v9 = iadd v8, v5
+;; @005d                               v9 = iadd v35, v5
 ;; @005d                               v11 = select_spectre_guard v7, v10, v9  ; v10 = 0
 ;; @005d                               store little heap v1, v11
 ;; @0064                               v17 = iconst.i64 4
@@ -121,7 +125,7 @@
 ;; @0064                               store little heap v2, v20
 ;; @006b                               v22 = iconst.i64 0x0010_0003
 ;; @006b                               v23 = uadd_overflow_trap v5, v22, heap_oob  ; v22 = 0x0010_0003
-;; @006b                               v25 = icmp ugt v23, v6
+;; @006b                               v25 = icmp ugt v23, v33
 ;; @006b                               v28 = iconst.i64 0x000f_ffff
 ;; @006b                               v29 = iadd v9, v28  ; v28 = 0x000f_ffff
 ;; @006b                               v31 = select_spectre_guard v25, v10, v29  ; v10 = 0