From 6ff8b5f4c9b79c4f35bdc26f62177284f40ad8d9 Mon Sep 17 00:00:00 2001 From: Leonard Chan Date: Thu, 31 Jul 2025 11:52:39 -0700 Subject: [PATCH] [WIP] Relative VTables for Rust This is a WIP patch for implementing https://github.com/rust-lang/compiler-team/issues/903. It adds a new unstable flag `-Zexperimental-relative-rust-abi-vtables` that makes vtables PIC-friendly. This is only supported for LLVM codegen and not supported for other backends. Early feedback on this is welcome. I'm not sure if how I implemented it is the best way of doing so since much of the actual vtable emission is heavily done during LLVM codegen. That is, the vtable to MIR looks like a normal table of pointers and byte arrays and I really only make the vtables relative on the codegen level. Locally, I can build the stage 1 compiler and runtimes with relative vtables, but I couldn't figure out how to tell the build system to only build stage 1 binaries with this flag, so I work around this by unconditionally enabling relative vtables in rustc. The end goal I think we'd like is either something akin to multilibs in clang where the compiler chooses which runtimes to use based off compilation flags, or binding this ABI to the target and have it be part of the default ABI for that target (just like how relative vtables are the default for Fuchsia in C++ with Clang). I think the later is what target modifiers do (https://github.com/rust-lang/rust/issues/136966). Action Items: - I'm still experimenting with building Fuchsia with this to assert it works e2e and I still need to do some measurements to see if this is still worth pursuing. - More work will still be needed to ensure the correct relative intrinsics are emitted with CFI and LTO. Rn I'm experimenting on a normal build. --- compiler/rustc_abi/src/lib.rs | 29 +++- compiler/rustc_codegen_llvm/src/builder.rs | 4 + compiler/rustc_codegen_llvm/src/common.rs | 47 ++++++- compiler/rustc_codegen_llvm/src/consts.rs | 128 +++++++++++++++--- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 9 ++ compiler/rustc_codegen_llvm/src/type_.rs | 7 + compiler/rustc_codegen_ssa/src/base.rs | 23 +++- compiler/rustc_codegen_ssa/src/meth.rs | 87 ++++++++++-- compiler/rustc_codegen_ssa/src/mir/block.rs | 18 ++- compiler/rustc_codegen_ssa/src/size_of_val.rs | 14 +- .../rustc_codegen_ssa/src/traits/builder.rs | 1 + .../rustc_codegen_ssa/src/traits/consts.rs | 1 + .../rustc_codegen_ssa/src/traits/type_.rs | 1 + .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 12 ++ compiler/rustc_session/src/config.rs | 10 ++ compiler/rustc_session/src/options.rs | 3 + compiler/rustc_ty_utils/src/abi.rs | 29 +++- src/doc/rustc/src/codegen-options/index.md | 4 + .../relative-vtables/simple-vtable.rs | 79 +++++++++++ 19 files changed, 458 insertions(+), 48 deletions(-) create mode 100644 tests/codegen-llvm/relative-vtables/simple-vtable.rs diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index de44c8755a078..bcfcdd8db5b9f 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -43,7 +43,7 @@ use std::fmt; #[cfg(feature = "nightly")] use std::iter::Step; use std::num::{NonZeroUsize, ParseIntError}; -use std::ops::{Add, AddAssign, Deref, Mul, RangeFull, RangeInclusive, Sub}; +use std::ops::{Add, AddAssign, Deref, Div, Mul, RangeFull, RangeInclusive, Sub}; use std::str::FromStr; use bitflags::bitflags; @@ -818,6 +818,14 @@ impl Size { if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } } + #[inline] + pub fn checked_div(self, count: u64, cx: &C) -> Option { + let dl = cx.data_layout(); + + let bytes = self.bytes().checked_div(count)?; + if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } + } + /// Truncates `value` to `self` bits and then sign-extends it to 128 bits /// (i.e., if it is negative, fill with 1's on the left). #[inline] @@ -905,6 +913,25 @@ impl Mul for Size { } } +impl Div for u64 { + type Output = Size; + #[inline] + fn div(self, size: Size) -> Size { + size / self + } +} + +impl Div for Size { + type Output = Size; + #[inline] + fn div(self, count: u64) -> Size { + match self.bytes().checked_div(count) { + Some(bytes) => Size::from_bytes(bytes), + None => panic!("Size::div: {} / {} doesn't fit in u64", self.bytes(), count), + } + } +} + impl AddAssign for Size { #[inline] fn add_assign(&mut self, other: Size) { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index c082a82306848..a7b10c1b1732b 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -623,6 +623,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn load_relative(&mut self, ptr: &'ll Value, byte_offset: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildLoadRelative(self.llbuilder, ptr, byte_offset) } + } + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index b0cf9925019d2..f2db38b94c75a 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -289,8 +289,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.const_bitcast(llval, llty) }; } else { - let init = - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); let alloc = alloc.inner(); let value = match alloc.mutability { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), @@ -322,7 +326,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }), ))) .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); self.static_addr_of_impl(init, alloc.inner().align, None) } GlobalAlloc::Static(def_id) => { @@ -356,7 +365,37 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value { - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false) + const_alloc_to_llvm(self, alloc.inner(), /*static*/ false, /*vtable_base*/ None) + } + + fn construct_vtable( + &self, + vtable_allocation: ConstAllocation<'_>, + num_entries: u64, + ) -> Self::Value { + // When constructing relative vtables, we need to create the global first before creating + // the initializer so the initializer has references to the global we will bind it to. + // Regular vtables aren't self-referential so we can just create the initializer on its + // own. + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = self.type_array(self.type_i32(), num_entries); + let vtable = self.static_addr_of_mut_from_type( + llty, + self.data_layout().i32_align, + Some("vtable"), + ); + let init = const_alloc_to_llvm( + self, + vtable_allocation.inner(), + /*static*/ false, + Some(vtable), + ); + self.static_addr_of_impl_for_gv(init, vtable) + } else { + let vtable_const = self.const_data_from_alloc(vtable_allocation); + let align = self.data_layout().pointer_align().abi; + self.static_addr_of(vtable_const, align, Some("vtable")) + } } fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 9844c9444b3d0..ab4c3549dd4ae 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,6 +1,6 @@ use std::ops::Range; -use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; +use rustc_abi::{Align, Endian, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; @@ -29,6 +29,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, alloc: &Allocation, is_static: bool, + vtable_base: Option<&'ll Value>, ) -> &'ll Value { // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be @@ -44,6 +45,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let dl = cx.data_layout(); let pointer_size = dl.pointer_size(); let pointer_size_bytes = pointer_size.bytes() as usize; + let use_relative_layout = cx.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables + && vtable_base.is_some(); // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. @@ -52,7 +55,11 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &'a CodegenCx<'ll, 'b>, alloc: &'a Allocation, range: Range, + use_relative_layout: bool, ) { + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + let pointer_size_bytes = pointer_size.bytes() as usize; let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); let chunk_to_llval = move |chunk| match chunk { @@ -75,7 +82,43 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; if allow_uninit_chunks { - llvals.extend(chunks.map(chunk_to_llval)); + if use_relative_layout { + // Rather than being stored as a struct of pointers or byte-arrays, a relative + // vtable is a pure i32 array, so its components must be chunks of i32s. Here we + // explicitly group any sequence of bytes into i32s. + // + // Normally we can only do this if an 8-byte constant can fit into 4 bytes. + for chunk in chunks { + match chunk { + InitChunk::Init(range) => { + let range = + (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = + alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + for bytes in bytes.chunks_exact(pointer_size_bytes) { + assert!( + bytes[4..pointer_size_bytes].iter().all(|&x| x == 0), + "Cannot fit constant into 4-bytes: {:?}", + bytes + ); + let bytes: [u8; 4] = bytes[0..4].try_into().unwrap(); + let val: u32 = match dl.endian { + Endian::Big => u32::from_be_bytes(bytes), + Endian::Little => u32::from_le_bytes(bytes), + }; + llvals.push(cx.const_u32(val)); + } + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + let val = cx.const_undef(cx.type_array(cx.type_i8(), len / 2)); + llvals.push(val); + } + }; + } + } else { + llvals.extend(chunks.map(chunk_to_llval)); + } } else { // If this allocation contains any uninit bytes, codegen as if it was initialized // (using some arbitrary value for uninit bytes). @@ -93,7 +136,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + append_chunks_of_init_and_uninit_bytes( + &mut llvals, + cx, + alloc, + next_offset..offset, + use_relative_layout, + ); } let ptr_offset = read_target_uint( dl.endian, @@ -109,14 +158,34 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), - Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(pointer_size), - }, - cx.type_ptr_ext(address_space), - )); + let s = { + let scalar = cx.scalar_to_backend( + InterpScalar::from_pointer( + Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_ptr_ext(address_space), + ); + + if use_relative_layout { + unsafe { + let fptr = llvm::LLVMDSOLocalEquivalent(scalar); + let sub = llvm::LLVMConstSub( + llvm::LLVMConstPtrToInt(fptr, cx.type_i64()), + llvm::LLVMConstPtrToInt(vtable_base.unwrap(), cx.type_i64()), + ); + llvm::LLVMConstTrunc(sub, cx.type_i32()) + } + } else { + scalar + } + }; + + llvals.push(s); next_offset = offset + pointer_size_bytes; } if alloc.len() >= next_offset { @@ -124,7 +193,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range, use_relative_layout); } // Avoid wrapping in a struct if there is only a single value. This ensures @@ -132,7 +201,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // is a valid C string. LLVM only considers bare arrays for this optimization, // not arrays wrapped in a struct. LLVM handles this at: // https://github.com/rust-lang/llvm-project/blob/acaea3d2bb8f351b740db7ebce7d7a40b9e21488/llvm/lib/Target/TargetLoweringObjectFile.cpp#L249-L280 - if let &[data] = &*llvals { data } else { cx.const_struct(&llvals, true) } + if let &[data] = &*llvals { + data + } else if use_relative_layout { + cx.const_array(cx.type_i32(), &llvals) + } else { + cx.const_struct(&llvals, true) + } } fn codegen_static_initializer<'ll, 'tcx>( @@ -140,7 +215,7 @@ fn codegen_static_initializer<'ll, 'tcx>( def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { let alloc = cx.tcx.eval_static_initializer(def_id)?; - Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true), alloc)) + Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true, /*vtable_base*/ None), alloc)) } fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { @@ -233,21 +308,31 @@ impl<'ll> CodegenCx<'ll, '_> { cv: &'ll Value, align: Align, kind: Option<&str>, + ) -> &'ll Value { + let gv = self.static_addr_of_mut_from_type(self.val_ty(cv), align, kind); + llvm::set_initializer(gv, cv); + gv + } + + pub(crate) fn static_addr_of_mut_from_type( + &self, + ty: &'ll Type, + align: Align, + kind: Option<&str>, ) -> &'ll Value { let gv = match kind { Some(kind) if !self.tcx.sess.fewer_names() => { let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + let gv = self.define_global(&name, ty).unwrap_or_else(|| { bug!("symbol `{}` is already defined", name); }); gv } - _ => self.define_global("", self.val_ty(cv)).unwrap_or_else(|| { + _ => self.define_global("", ty).unwrap_or_else(|| { bug!("anonymous global symbol is already defined"); }), }; llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); - llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); gv @@ -280,6 +365,15 @@ impl<'ll> CodegenCx<'ll, '_> { gv } + pub(crate) fn static_addr_of_impl_for_gv(&self, cv: &'ll Value, gv: &'ll Value) -> &'ll Value { + assert!(!self.const_globals.borrow().contains_key(&cv)); + let mut binding = self.const_globals.borrow_mut(); + binding.insert(cv, gv); + llvm::set_initializer(gv, cv); + llvm::set_global_constant(gv, true); + gv + } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 53f0f9ff9d01b..788d19aa8d475 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1077,6 +1077,8 @@ unsafe extern "C" { pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMConstSub<'a>(LHS: &'a Value, RHS: &'a Value) -> &'a Value; + pub(crate) fn LLVMConstTrunc<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; // Operations on global variables, functions, and aliases (globals) pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1107,6 +1109,13 @@ unsafe extern "C" { pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); pub(crate) safe fn LLVMSetExternallyInitialized(GlobalVar: &Value, IsExtInit: Bool); + pub(crate) fn LLVMDSOLocalEquivalent(GlobalVar: &Value) -> &Value; + pub(crate) fn LLVMBuildLoadRelative<'a>( + Builder: &Builder<'a>, + Ptr: &'a Value, + ByteOffset: &'a Value, + ) -> &'a Value; + // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( C: &Context, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 81bb70c958790..63574fb822ae9 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -292,6 +292,13 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { fn_abi.ptr_to_llvm_type(self) } + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + self.type_i32() + } else { + fn_abi.ptr_to_llvm_type(self) + } + } fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index f58be0a3b9a3a..43b9dc9653266 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -203,7 +203,28 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(entry_idx) = vptr_entry_idx { let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes(); - load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let val = load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset / 2, + source, + true, + /*load_relative*/ true, + ); + bx.zext(val, bx.type_isize()) + } else { + load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset, + source, + true, + /*load_relative*/ false, + ) + } } else { old_info } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 2fa466b500179..79914766243e8 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -25,12 +25,32 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the function pointer from the object. debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); - - let llty = bx.fn_ptr_backend_type(fn_abi); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.vtable_component_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + nonnull, + /*load_relative*/ true, + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + nonnull, + /*load_relative*/ false, + ) + } } pub(crate) fn get_optional_fn>( @@ -61,12 +81,33 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the data pointer from the object. debug!("get_int({:?}, {:?})", llvtable, self); - - let llty = bx.type_isize(); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.type_i32(); + let val = load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + false, + /*load_relative*/ false, + ); + bx.zext(val, bx.type_isize()) + } else { + let llty = bx.type_isize(); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + false, + /*load_relative*/ false, + ) + } } } @@ -114,9 +155,17 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); - let vtable_const = cx.const_data_from_alloc(vtable_allocation); - let align = cx.data_layout().pointer_align().abi; - let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); + let num_entries = { + if let Some(trait_ref) = trait_ref { + let trait_ref = trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_and_anonymize_regions(trait_ref); + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + } + } + .len(); + let vtable = cx.construct_vtable(vtable_allocation, num_entries as u64); cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable); cx.create_vtable_debuginfo(ty, trait_ref, vtable); @@ -132,6 +181,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( vtable_byte_offset: u64, ty: Ty<'tcx>, nonnull: bool, + load_relative: bool, ) -> Bx::Value { let ptr_align = bx.data_layout().pointer_align().abi; @@ -141,6 +191,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) { let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref).as_bytes()).unwrap(); + // FIXME: Add correct intrinsic for RV here. let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); return func; } else if nonnull { @@ -148,11 +199,23 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } - let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); - let ptr = bx.load(llty, gep, ptr_align); + let ptr = if load_relative { + bx.load_relative(llvtable, bx.const_i32(vtable_byte_offset.try_into().unwrap())) + } else { + let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); + bx.load(llty, gep, ptr_align) + }; + // VTable loads are invariant. bx.set_invariant_load(ptr); - if nonnull { + // FIXME: The verifier complains with + // + // nonnull applies only to load instructions, use attributes for calls or invokes + // @llvm.load.relative.i32 (ptr nonnull %13, i32 12), !dbg !4323, !invariant.load !27, !noalias !27, !nonnull !27 + // + // For now, do not mark the load relative intrinsic with nonnull, but I think it should be fine + // to do so since it's effectively a load. + if nonnull && !load_relative { bx.nonnull_metadata(ptr); } ptr diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index e2241a77d186c..5ef719230efc4 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -612,7 +612,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args1 = [place.val.llval]; &args1[..] }; - let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() { + let (maybe_null, drop_fn, fn_abi, drop_instance, vtable) = match ty.kind() { // FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`? ty::Dynamic(_, _) => { @@ -645,6 +645,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { .get_optional_fn(bx, vtable, ty, fn_abi), fn_abi, virtual_drop, + Some(vtable), ) } _ => ( @@ -652,6 +653,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty()), drop_fn, + None, ), }; @@ -659,10 +661,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // generated for no-op drops. if maybe_null { let is_not_null = bx.append_sibling_block("is_not_null"); - let llty = bx.fn_ptr_backend_type(fn_abi); - let null = bx.const_null(llty); let non_null = - bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null); + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + bx.icmp( + base::bin_op_to_icmp_predicate(mir::BinOp::Ne, /*signed*/ false), + drop_fn, + vtable.unwrap(), + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + let null = bx.const_null(llty); + bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null) + }; bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target)); bx.switch_to_block(is_not_null); self.set_debug_loc(bx, *source_info); diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index e1bd8014d7a2f..007c5eec1309e 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -33,11 +33,15 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) .get_usize(bx, vtable, t); - // Size is always <= isize::MAX. - let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; - bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); - // Alignment is always nonzero. - bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + // FIXME: The range metadatat can only be applied to load instructions, but it probably + // can also be changed to apply to the load_relative intrinsic. + if !bx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables { + // Size is always <= isize::MAX. + let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; + bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); + // Alignment is always nonzero. + bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + } (size, align) } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 60296e36e0c1a..d87cd280e0048 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -237,6 +237,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn alloca(&mut self, size: Size, align: Align) -> Self::Value; fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; + fn load_relative(&mut self, ptr: Self::Value, byte_offset: Self::Value) -> Self::Value; fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; fn atomic_load( &mut self, diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index d83a04d814be3..1e249ffac1141 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -38,6 +38,7 @@ pub trait ConstCodegenMethods: BackendTypes { fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value; + fn construct_vtable(&self, alloc: ConstAllocation<'_>, num_entries: u64) -> Self::Value; fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 32c24965e1bf6..bb1f814338e23 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -116,6 +116,7 @@ pub trait LayoutTypeCodegenMethods<'tcx>: BackendTypes { fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn reg_backend_type(&self, ty: &Reg) -> Self::Type; /// The backend type used for a rust type when it's in an SSA register. /// diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index ad459986826a5..1c872c16a4141 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -123,6 +123,10 @@ extern "C" void LLVMRustSetLastError(const char *Err) { LastError = strdup(Err); } +extern "C" LLVMValueRef LLVMDSOLocalEquivalent(LLVMValueRef GlobalVal) { + return wrap(DSOLocalEquivalent::get(unwrap(GlobalVal))); +} + extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, const char *Target) { #if LLVM_VERSION_GE(21, 0) @@ -593,6 +597,14 @@ extern "C" void LLVMRustSetAllowReassoc(LLVMValueRef V) { } } +extern "C" LLVMValueRef LLVMBuildLoadRelative(LLVMBuilderRef B, LLVMValueRef Ptr, + LLVMValueRef ByteOffset) { + Type *Int32Ty = Type::getInt32Ty(unwrap(B)->getContext()); + Value *call = unwrap(B)->CreateIntrinsic( + Intrinsic::load_relative, {Int32Ty}, {unwrap(Ptr), unwrap(ByteOffset)}); + return wrap(call); +} + extern "C" uint64_t LLVMRustGetArrayNumElements(LLVMTypeRef Ty) { return unwrap(Ty)->getArrayNumElements(); } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 60e301a5fd870..2bc71a20b2993 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2857,6 +2857,16 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M early_dcx.early_fatal("can't dump dependency graph without `-Z query-dep-graph`"); } + // The rust build system doesn't have a good way of saying "cross compile these + // libraries for other targets with this flag", so unconditionally build these + // archs with relative vtables enabled by default. + if target_triple.tuple().ends_with("fuchsia") + || target_triple.tuple().contains("-cros-") + || target_triple.tuple().contains("android") + { + unstable_opts.experimental_relative_rust_abi_vtables = true; + } + let logical_env = parse_logical_env(early_dcx, matches); let sysroot = Sysroot::new(matches.opt_str("sysroot").map(PathBuf::from)); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index b89aec7d22a91..e47e35c86c580 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2313,6 +2313,9 @@ options! { "enforce the type length limit when monomorphizing instances in codegen"), experimental_default_bounds: bool = (false, parse_bool, [TRACKED], "enable default bounds for experimental group of auto traits"), + // Change this to true to unconditionally build all runtimes with relative vtables. + experimental_relative_rust_abi_vtables: bool = (false, parse_bool, [TRACKED], + "use the relative layout for vtables"), export_executable_symbols: bool = (false, parse_bool, [TRACKED], "export symbols from executables, as if they were dynamic libraries"), external_clangrt: bool = (false, parse_bool, [UNTRACKED], diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 0f09e548f0e2b..5ae651e3b3b7a 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -311,8 +311,21 @@ fn arg_attrs_for_rust_scalar<'tcx>( None }; if let Some(kind) = kind { - attrs.pointee_align = - Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + let is_vtable = if let ty::Ref(_, ty, _) = *layout.ty.kind() { + ty.is_trait() + && matches!(kind, PointerKind::SharedRef { frozen: true }) + && !is_return + } else { + false + }; + + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables && is_vtable + { + attrs.pointee_align = Some(cx.tcx().data_layout.i32_align); + } else { + attrs.pointee_align = + Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + } // `Box` are not necessarily dereferenceable for the entire duration of the function as // they can be deallocated at any time. Same for non-frozen shared references (see @@ -324,8 +337,16 @@ fn arg_attrs_for_rust_scalar<'tcx>( PointerKind::Box { .. } | PointerKind::SharedRef { frozen: false } | PointerKind::MutableRef { unpin: false } => Size::ZERO, - PointerKind::SharedRef { frozen: true } - | PointerKind::MutableRef { unpin: true } => pointee.size, + PointerKind::SharedRef { frozen: true } => { + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables + && is_vtable + { + pointee.size / 2 + } else { + pointee.size + } + } + PointerKind::MutableRef { unpin: true } => pointee.size, }; // The aliasing rules for `Box` are still not decided, but currently we emit diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index 0e340de4daa27..019700f4d3293 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -162,6 +162,10 @@ This option allows you to put extra data in each output filename. It takes a string to add as a suffix to the filename. See the [`--emit` flag][option-emit] for more information. +## experimental-relative-rust-abi-vtables + +This option uses the relative layout for vtables. + ## force-frame-pointers This flag forces the use of frame pointers. It takes one of the following diff --git a/tests/codegen-llvm/relative-vtables/simple-vtable.rs b/tests/codegen-llvm/relative-vtables/simple-vtable.rs new file mode 100644 index 0000000000000..ab35ef642749b --- /dev/null +++ b/tests/codegen-llvm/relative-vtables/simple-vtable.rs @@ -0,0 +1,79 @@ +//@ compile-flags: -Zexperimental-relative-rust-abi-vtables=y + +#![crate_type = "lib"] + +// CHECK: @vtable.0 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 0, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_FOO:@".*Struct2\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_BAR:@".*Struct2\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK: @vtable.1 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[VT1_DROP_IN_PLACE:\@".*drop_in_place[^\"]*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 24, +// CHECK-SAME: i32 8, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_FOO:@".*Struct\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_BAR:@".*Struct\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK-DAG: define {{.*}}void [[STRUCT2_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT2_BAR]](ptr +// CHECK-DAG: define {{.*}}void [[STRUCT_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT_BAR]](ptr + +trait MyTrait { + fn foo(&self); + fn bar(&self) -> u32; +} + +struct Struct { + s: String, +} + +struct Struct2 { + u: u32, +} + +impl MyTrait for Struct { + fn foo(&self) { + println!("Struct foo {}", self.s); + } + fn bar(&self) -> u32 { + 42 + } +} + +impl MyTrait for Struct2 { + fn foo(&self) { + println!("Struct2 foo {}", self.bar()); + } + fn bar(&self) -> u32 { + self.u + } +} + +/// This is only here to manifest the vtables. +pub fn create_struct(b: bool) -> Box { + if b { Box::new(Struct { s: "abc".to_string() }) } else { Box::new(Struct2 { u: 1 }) } +} + +// CHECK-DAG: define void @_ZN13simple_vtable10invoke_foo{{[a-zA-Z0-9]*}}( +// CHECK-SAME: ptr noundef nonnull align 1 [[DATA:%.*]], +// CHECK-SAME: ptr noalias noundef readonly align 4 dereferenceable(20) [[VTABLE:%.*]]) {{.*}}{ +// CHECK-NEXT: start: +// CHECK-NEXT: [[FUNC:%.*]] = tail call ptr @llvm.load.relative.i32(ptr nonnull [[VTABLE]], i32 12), !invariant.load +// CHECK-NEXT: tail call void [[FUNC]](ptr noundef nonnull align 1 [[DATA]]) +// CHECK-NEXT: ret void +// CHECK-NEXT: } +pub fn invoke_foo(x: &dyn MyTrait) { + x.foo(); +} + +// CHECK-DAG: define void @_ZN13simple_vtable11invoke_drop{{[a-zA-Z0-9]*}}(i1 noundef zeroext %b) {{.*}}{ +// CHECK: [[BOX:%.*]] = tail call { ptr, ptr } @_ZN13simple_vtable13create_struct{{.*}}(i1 noundef zeroext %b) +pub fn invoke_drop(b: bool) { + let bx = create_struct(b); + invoke_foo(&*bx); +}