From 3d3e05876bcbf3a184bda9a5f91a2f707f90633e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ma=C4=87kowski?= Date: Thu, 15 Jan 2026 13:47:28 +0000 Subject: [PATCH] Add support for stage-2 page tables --- CHANGELOG.md | 11 +++ README.md | 4 +- src/descriptor.rs | 223 +++++++++++++++++++++++++++++++++------------ src/idmap.rs | 228 ++++++++++++++++++++++++++++------------------ src/lib.rs | 86 ++++++++++++----- src/linearmap.rs | 139 ++++++++++++++++------------ src/paging.rs | 205 +++++++++++++++++++++++------------------ src/target.rs | 20 ++-- 8 files changed, 581 insertions(+), 335 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a94e93..1a78dda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +### Breaking changes + +- `Attributes` is now called `Stage1Attributes`. +- `TranslationRegime` enum now includes the `Stage2` variant. + +### New features + +- Added support for stage-2 page tables. + ## 0.11.0 ### Breaking changes diff --git a/README.md b/README.md index edf2118..a1cce75 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ This crate provides a library to manipulate page tables conforming to the AArch64 Virtual Memory System Architecture. -Currently it only supports: +Currently it supports: -- stage 1 page tables +- stage 1 and stage 2 page tables - 4 KiB pages - EL3, NS-EL2, NS-EL2&0 and NS-EL1&0 translation regimes - 64-bit descriptors diff --git a/src/descriptor.rs b/src/descriptor.rs index f9df401..85565d6 100644 --- a/src/descriptor.rs +++ b/src/descriptor.rs @@ -12,7 +12,8 @@ use crate::paging::PageTableWithLevel; use bitflags::bitflags; use core::fmt::{self, Debug, Display, Formatter}; -use core::ops::{Add, Sub}; +use core::marker::PhantomData; +use core::ops::{Add, BitAnd, BitOr, BitXor, Not, Sub}; use core::sync::atomic::{AtomicUsize, Ordering}; /// An aarch64 virtual address, the input type of a stage 1 page table. @@ -98,10 +99,39 @@ impl Sub for PhysicalAddress { } } +/// Trait abstracting the attributes used in page table descriptors. +/// +/// This allows the same page table structure to be used for different translation regimes (e.g. +/// Stage 1 vs Stage 2) which use different attribute bit definitions. +pub trait PagingAttributes: + bitflags::Flags + + Copy + + Clone + + Debug + + PartialEq + + Default + + Send + + Sync + + PartialOrd + + BitOr + + BitAnd + + BitXor + + Sub + + Not +{ + /// The bit indicating that a mapping is valid. + const VALID: Self; + /// The bit indicating that a descriptor is a table or page (leaf at level 3) rather than a block. + const TABLE_OR_PAGE: Self; + + /// Returns true if it is architecturally safe to update from `old` to `new` without break-before-make. + fn is_bbm_safe(old: Self, new: Self) -> bool; +} + bitflags! { - /// Attribute bits for a mapping in a page table. + /// Attribute bits for a mapping in a Stage 1 page table. #[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub struct Attributes: usize { + pub struct Stage1Attributes: usize { const VALID = 1 << 0; const TABLE_OR_PAGE = 1 << 1; @@ -145,7 +175,29 @@ bitflags! { } } -impl Attributes { +impl PagingAttributes for Stage1Attributes { + const VALID: Self = Self::VALID; + const TABLE_OR_PAGE: Self = Self::TABLE_OR_PAGE; + + fn is_bbm_safe(old: Self, new: Self) -> bool { + // Masks of bits that may be set resp. cleared on a live, valid mapping without BBM + let clear_allowed_mask = Self::VALID + | Self::READ_ONLY + | Self::ACCESSED + | Self::DBM + | Self::PXN + | Self::UXN + | Self::SWFLAG_0 + | Self::SWFLAG_1 + | Self::SWFLAG_2 + | Self::SWFLAG_3; + let set_allowed_mask = clear_allowed_mask | Self::NON_GLOBAL; + + (!old & new & !set_allowed_mask).is_empty() && (old & !new & !clear_allowed_mask).is_empty() + } +} + +impl Stage1Attributes { /// Mask for the bits determining the shareability of the mapping. pub const SHAREABILITY_MASK: Self = Self::INNER_SHAREABLE; @@ -153,6 +205,69 @@ impl Attributes { pub const ATTRIBUTE_INDEX_MASK: Self = Self::ATTRIBUTE_INDEX_7; } +bitflags! { + /// Attribute bits for a mapping in a Stage 2 page table. + #[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct Stage2Attributes: usize { + const VALID = 1 << 0; + const TABLE_OR_PAGE = 1 << 1; + + const MEMATTR_DEVICE_nGnRnE = 0 << 2; + const MEMATTR_DEVICE_nGnRE = 1 << 2; + const MEMATTR_DEVICE_nGRE = 2 << 2; + const MEMATTR_DEVICE_GRE = 3 << 2; + /// Inner Non-cacheable + const MEMATTR_NORMAL_INNER_NC = 1 << 2; + /// Inner Write-Through Cacheable + const MEMATTR_NORMAL_INNER_WT = 2 << 2; + /// Inner Write-Back Cacheable + const MEMATTR_NORMAL_INNER_WB = 3 << 2; + /// Outer Non-cacheable + const MEMATTR_NORMAL_OUTER_NC = 1 << 4; + /// Outer Write-Through Cacheable + const MEMATTR_NORMAL_OUTER_WT = 2 << 4; + /// Outer Write-Back Cacheable + const MEMATTR_NORMAL_OUTER_WB = 3 << 4; + + // S2AP[1:0] at [7:6] + const S2AP_ACCESS_NONE = 0 << 6; + const S2AP_ACCESS_RO = 1 << 6; + const S2AP_ACCESS_WO = 2 << 6; + const S2AP_ACCESS_RW = 3 << 6; + + const SH_NONE = 0 << 8; + const SH_OUTER = 2 << 8; + const SH_INNER = 3 << 8; + + const ACCESS_FLAG = 1 << 10; + + const XN = 1 << 54; + + const SWFLAG_0 = 1 << 55; + const SWFLAG_1 = 1 << 56; + const SWFLAG_2 = 1 << 57; + const SWFLAG_3 = 1 << 58; + } +} + +impl PagingAttributes for Stage2Attributes { + const VALID: Self = Self::VALID; + const TABLE_OR_PAGE: Self = Self::TABLE_OR_PAGE; + + fn is_bbm_safe(old: Self, new: Self) -> bool { + let allowed_mask = Self::VALID + | Self::S2AP_ACCESS_RW // also covers NONE, RO, WO changes + | Self::ACCESS_FLAG + | Self::XN + | Self::SWFLAG_0 + | Self::SWFLAG_1 + | Self::SWFLAG_2 + | Self::SWFLAG_3; + + ((old ^ new) & !allowed_mask).is_empty() + } +} + pub(crate) type DescriptorBits = usize; /// An entry in a page table. @@ -163,14 +278,21 @@ pub(crate) type DescriptorBits = usize; /// - A block mapping, if it is not in the lowest level page table. /// - A pointer to a lower level pagetable, if it is not in the lowest level page table. #[repr(C)] -pub struct Descriptor(pub(crate) AtomicUsize); +pub struct Descriptor( + pub(crate) AtomicUsize, + PhantomData, +); -impl Descriptor { +impl Descriptor { /// An empty (i.e. 0) descriptor. - pub const EMPTY: Self = Self(AtomicUsize::new(0)); + pub const EMPTY: Self = Self::new(0); const PHYSICAL_ADDRESS_BITMASK: usize = !(PAGE_SIZE - 1) & !(0xffff << 48); + pub(crate) const fn new(value: usize) -> Self { + Descriptor(AtomicUsize::new(value), PhantomData) + } + /// Returns the contents of a descriptor which may be potentially live /// Use acquire semantics so that the load is not reordered with subsequent loads pub(crate) fn bits(&self) -> DescriptorBits { @@ -189,39 +311,38 @@ impl Descriptor { PhysicalAddress(bits & Self::PHYSICAL_ADDRESS_BITMASK) } - fn flags_from_bits(bits: DescriptorBits) -> Attributes { - Attributes::from_bits_retain(bits & !Self::PHYSICAL_ADDRESS_BITMASK) + fn flags_from_bits(bits: DescriptorBits) -> A { + A::from_bits_retain(bits & !Self::PHYSICAL_ADDRESS_BITMASK) } /// Returns the flags of this page table entry, or `None` if its state does not /// contain a valid set of flags. - pub fn flags(&self) -> Attributes { + pub fn flags(&self) -> A { Self::flags_from_bits(self.bits()) } - /// Returns `true` if [`Attributes::VALID`] is set on this entry, e.g. if the entry is mapped. + /// Returns `true` if [`PagingAttributes::VALID`] is set on this entry, e.g. if the entry is mapped. pub fn is_valid(&self) -> bool { - (self.bits() & Attributes::VALID.bits()) != 0 + (self.bits() & A::VALID.bits()) != 0 } /// Returns `true` if this is a valid entry pointing to a next level translation table or a page. pub fn is_table_or_page(&self) -> bool { - self.flags() - .contains(Attributes::TABLE_OR_PAGE | Attributes::VALID) + self.flags().contains(A::TABLE_OR_PAGE | A::VALID) } - pub(crate) fn set(&mut self, pa: PhysicalAddress, flags: Attributes) { + pub(crate) fn set(&mut self, pa: PhysicalAddress, flags: A) { self.0.store( (pa.0 & Self::PHYSICAL_ADDRESS_BITMASK) | flags.bits(), Ordering::Release, ); } - pub(crate) fn subtable( + pub(crate) fn subtable>( &self, translation: &T, level: usize, - ) -> Option> { + ) -> Option> { if level < LEAF_LEVEL && self.is_table_or_page() { let output_address = self.output_address(); let table = translation.physical_to_virtual(output_address); @@ -231,7 +352,7 @@ impl Descriptor { } } -impl Debug for Descriptor { +impl Debug for Descriptor { fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { write!(f, "{:#016x}", self.bits())?; if self.is_valid() { @@ -241,47 +362,47 @@ impl Debug for Descriptor { } } -enum DescriptorEnum<'a> { +enum DescriptorEnum<'a, A: PagingAttributes> { /// A descriptor that is part of a set of page tables that are currently in use by one of the /// CPUs - Active(&'a mut Descriptor), + Active(&'a mut Descriptor), /// A descriptor that is part of a set of page tables that are currently inactive. This means /// TLB maintenance may be elided until the next time the page tables are made active. - Inactive(&'a mut Descriptor), + Inactive(&'a mut Descriptor), /// A descriptor that does not actually represent an entry in a page table. It permits updaters /// taking an UpdatableDescriptor to be called for a dry run to observe their effect without /// the need to pass an actual descriptor. - ActiveClone(DescriptorBits), + ActiveClone(DescriptorBits, PhantomData), } -pub struct UpdatableDescriptor<'a> { - descriptor: DescriptorEnum<'a>, +pub struct UpdatableDescriptor<'a, A: PagingAttributes = Stage1Attributes> { + descriptor: DescriptorEnum<'a, A>, level: usize, updated: bool, } -impl<'a> UpdatableDescriptor<'a> { +impl<'a, A: PagingAttributes> UpdatableDescriptor<'a, A> { /// Creates a new wrapper around a real descriptor that may or may not be live - pub(crate) fn new(desc: &'a mut Descriptor, level: usize, live: bool) -> Self { + pub(crate) fn new(desc: &'a mut Descriptor, level: usize, live: bool) -> Self { Self { descriptor: if live { DescriptorEnum::Active(desc) } else { DescriptorEnum::Inactive(desc) }, - level: level, + level, updated: false, } } /// Creates a new wrapper around an ActiveClone descriptor, which is used to observe the /// effect of user provided updater functions without applying them to actual descriptors - pub(crate) fn clone_from(d: &Descriptor, level: usize) -> Self { + pub(crate) fn clone_from(d: &Descriptor, level: usize) -> Self { Self { - descriptor: DescriptorEnum::ActiveClone(d.bits()), - level: level, + descriptor: DescriptorEnum::ActiveClone(d.bits(), PhantomData), + level, updated: false, } } @@ -299,29 +420,29 @@ impl<'a> UpdatableDescriptor<'a> { /// Returns whether this descriptor represents a table mapping. In this case, the output address /// refers to a next level table. pub fn is_table(&self) -> bool { - self.level < 3 && self.flags().contains(Attributes::TABLE_OR_PAGE) + self.level < 3 && self.flags().contains(A::TABLE_OR_PAGE) } /// Returns the bit representation of the underlying descriptor pub fn bits(&self) -> DescriptorBits { match &self.descriptor { DescriptorEnum::Active(d) | DescriptorEnum::Inactive(d) => d.bits(), - DescriptorEnum::ActiveClone(d) => *d, + DescriptorEnum::ActiveClone(d, _) => *d, } } /// Assigns the underlying descriptor according to `pa` and `flags, provided that doing so is /// permitted under BBM rules - pub fn set(&mut self, pa: PhysicalAddress, flags: Attributes) -> Result<(), ()> { + pub fn set(&mut self, pa: PhysicalAddress, flags: A) -> Result<(), ()> { if !self.bbm_permits_update(pa, flags) { return Err(()); } - let val = (pa.0 & Descriptor::PHYSICAL_ADDRESS_BITMASK) | flags.bits(); + let val = (pa.0 & Descriptor::::PHYSICAL_ADDRESS_BITMASK) | flags.bits(); match &mut self.descriptor { DescriptorEnum::Active(d) | DescriptorEnum::Inactive(d) => { self.updated |= val != d.0.swap(val, Ordering::Release) } - DescriptorEnum::ActiveClone(d) => { + DescriptorEnum::ActiveClone(d, _) => { self.updated |= *d != val; *d = val } @@ -334,12 +455,12 @@ impl<'a> UpdatableDescriptor<'a> { /// Depending on the flags this could be the address of a subtable, a mapping, or (if it is not /// a valid mapping) entirely arbitrary. pub fn output_address(&self) -> PhysicalAddress { - Descriptor::output_address_from_bits(self.bits()) + Descriptor::::output_address_from_bits(self.bits()) } /// Returns the flags of this descriptor - pub fn flags(&self) -> Attributes { - Descriptor::flags_from_bits(self.bits()) + pub fn flags(&self) -> A { + Descriptor::::flags_from_bits(self.bits()) } /// Returns whether this descriptor should be considered live and valid, in which case BBM @@ -348,13 +469,13 @@ impl<'a> UpdatableDescriptor<'a> { fn is_live_and_valid(&self) -> bool { match &self.descriptor { DescriptorEnum::Inactive(_) => false, - _ => self.flags().contains(Attributes::VALID), + _ => self.flags().contains(A::VALID), } } /// Returns whether BBM permits setting the flags on this descriptor to `flags` - fn bbm_permits_update(&self, pa: PhysicalAddress, flags: Attributes) -> bool { - if !self.is_live_and_valid() || !flags.contains(Attributes::VALID) { + fn bbm_permits_update(&self, pa: PhysicalAddress, flags: A) -> bool { + if !self.is_live_and_valid() || !flags.contains(A::VALID) { return true; } @@ -363,29 +484,15 @@ impl<'a> UpdatableDescriptor<'a> { return false; } - // Masks of bits that may be set resp. cleared on a live, valid mapping without BBM - let clear_allowed_mask = Attributes::VALID - | Attributes::READ_ONLY - | Attributes::ACCESSED - | Attributes::DBM - | Attributes::PXN - | Attributes::UXN - | Attributes::SWFLAG_0 - | Attributes::SWFLAG_1 - | Attributes::SWFLAG_2 - | Attributes::SWFLAG_3; - let set_allowed_mask = clear_allowed_mask | Attributes::NON_GLOBAL; - - (!self.flags() & flags & !set_allowed_mask).is_empty() - && (self.flags() & !flags & !clear_allowed_mask).is_empty() + A::is_bbm_safe(self.flags(), flags) } /// Modifies the descriptor by setting or clearing its flags. - pub fn modify_flags(&mut self, set: Attributes, clear: Attributes) -> Result<(), ()> { + pub fn modify_flags(&mut self, set: A, clear: A) -> Result<(), ()> { let oldval = self.flags(); let flags = (oldval | set) & !clear; - if (oldval ^ flags).contains(Attributes::TABLE_OR_PAGE) { + if (oldval ^ flags).contains(A::TABLE_OR_PAGE) { // Cannot convert between table and block/page descriptors, regardless of whether or // not BBM permits this and whether the entry is live, given that doing so would // corrupt our data strucutures. diff --git a/src/idmap.rs b/src/idmap.rs index 02ff9e5..2e6a5ce 100644 --- a/src/idmap.rs +++ b/src/idmap.rs @@ -8,25 +8,37 @@ use crate::{ MapError, Mapping, - descriptor::{Attributes, Descriptor, PhysicalAddress, UpdatableDescriptor, VirtualAddress}, + descriptor::{ + Descriptor, PagingAttributes, PhysicalAddress, Stage1Attributes, UpdatableDescriptor, + VirtualAddress, + }, paging::{ Constraints, MemoryRegion, PageTable, Translation, TranslationRegime, VaRange, deallocate, }, }; +use core::marker::PhantomData; use core::ptr::NonNull; /// Identity mapping, where every virtual address is either unmapped or mapped to the identical IPA. #[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub struct IdTranslation; +pub struct IdTranslation { + _phantom: PhantomData, +} + +impl IdTranslation { + pub fn new() -> Self { + Self { + _phantom: PhantomData, + } + } -impl IdTranslation { fn virtual_to_physical(va: VirtualAddress) -> PhysicalAddress { PhysicalAddress(va.0) } } -impl Translation for IdTranslation { - fn allocate_table(&mut self) -> (NonNull, PhysicalAddress) { +impl Translation for IdTranslation { + fn allocate_table(&mut self) -> (NonNull>, PhysicalAddress) { let table = PageTable::new(); // Physical address is the same as the virtual address because we are using identity mapping @@ -34,7 +46,7 @@ impl Translation for IdTranslation { (table, PhysicalAddress(table.as_ptr() as usize)) } - unsafe fn deallocate_table(&mut self, page_table: NonNull) { + unsafe fn deallocate_table(&mut self, page_table: NonNull>) { // SAFETY: Our caller promises that the memory was allocated by `allocate_table` on this // `IdTranslation` and not yet deallocated. `allocate_table` used the global allocator and // appropriate layout by calling `PageTable::new()`. @@ -43,8 +55,8 @@ impl Translation for IdTranslation { } } - fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull { - NonNull::new(pa.0 as *mut PageTable).expect("Got physical address 0 for pagetable") + fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull> { + NonNull::new(pa.0 as *mut PageTable).expect("Got physical address 0 for pagetable") } } @@ -65,20 +77,20 @@ impl Translation for IdTranslation { /// ```no_run /// use aarch64_paging::{ /// idmap::IdMap, -/// descriptor::Attributes, +/// descriptor::Stage1Attributes, /// paging::{MemoryRegion, TranslationRegime}, /// }; /// /// const ASID: usize = 1; /// const ROOT_LEVEL: usize = 1; -/// const NORMAL_CACHEABLE: Attributes = Attributes::ATTRIBUTE_INDEX_1.union(Attributes::INNER_SHAREABLE); +/// const NORMAL_CACHEABLE: Stage1Attributes = Stage1Attributes::ATTRIBUTE_INDEX_1.union(Stage1Attributes::INNER_SHAREABLE); /// /// // Create a new EL1 page table with identity mapping. /// let mut idmap = IdMap::new(ASID, ROOT_LEVEL, TranslationRegime::El1And0); /// // Map a 2 MiB region of memory as read-write. /// idmap.map_range( /// &MemoryRegion::new(0x80200000, 0x80400000), -/// NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::VALID | Attributes::ACCESSED, +/// NORMAL_CACHEABLE | Stage1Attributes::NON_GLOBAL | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, /// ).unwrap(); /// // SAFETY: Everything the program uses is within the 2 MiB region mapped above. /// let ttbr = unsafe { @@ -97,8 +109,8 @@ impl Translation for IdTranslation { /// // Now change the mapping to read-only and executable. /// idmap.map_range( /// &MemoryRegion::new(0x80200000, 0x80400000), -/// NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::READ_ONLY | Attributes::VALID -/// | Attributes::ACCESSED, +/// NORMAL_CACHEABLE | Stage1Attributes::NON_GLOBAL | Stage1Attributes::READ_ONLY | Stage1Attributes::VALID +/// | Stage1Attributes::ACCESSED, /// ).unwrap(); /// // SAFETY: Everything the program will used is mapped in by this page table. /// unsafe { @@ -106,16 +118,16 @@ impl Translation for IdTranslation { /// } /// ``` #[derive(Debug)] -pub struct IdMap { - mapping: Mapping, +pub struct IdMap { + mapping: Mapping, A>, } -impl IdMap { +impl IdMap { /// Creates a new identity-mapping page table with the given ASID and root level. pub fn new(asid: usize, rootlevel: usize, translation_regime: TranslationRegime) -> Self { Self { mapping: Mapping::new( - IdTranslation, + IdTranslation::::new(), asid, rootlevel, translation_regime, @@ -173,7 +185,7 @@ impl IdMap { /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page /// table is active. This function writes block and page entries, but only maps them if `flags` - /// contains `Attributes::VALID`, otherwise the entries remain invalid. + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -186,7 +198,7 @@ impl IdMap { /// /// Returns [`MapError::BreakBeforeMakeViolation`] if the range intersects with live mappings, /// and modifying those would violate architectural break-before-make (BBM) requirements. - pub fn map_range(&mut self, range: &MemoryRegion, flags: Attributes) -> Result<(), MapError> { + pub fn map_range(&mut self, range: &MemoryRegion, flags: A) -> Result<(), MapError> { self.map_range_with_constraints(range, flags, Constraints::empty()) } @@ -197,7 +209,7 @@ impl IdMap { /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page /// table is active. This function writes block and page entries, but only maps them if `flags` - /// contains `Attributes::VALID`, otherwise the entries remain invalid. + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -213,10 +225,10 @@ impl IdMap { pub fn map_range_with_constraints( &mut self, range: &MemoryRegion, - flags: Attributes, + flags: A, constraints: Constraints, ) -> Result<(), MapError> { - let pa = IdTranslation::virtual_to_physical(range.start()); + let pa = IdTranslation::::virtual_to_physical(range.start()); self.mapping.map_range(range, pa, flags, constraints) } @@ -259,7 +271,7 @@ impl IdMap { /// and modifying those would violate architectural break-before-make (BBM) requirements. pub fn modify_range(&mut self, range: &MemoryRegion, f: &F) -> Result<(), MapError> where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor<'_, A>) -> Result<(), ()> + ?Sized, { self.mapping.modify_range(range, f) } @@ -289,7 +301,7 @@ impl IdMap { /// largest virtual address covered by the page table given its root level. pub fn walk_range(&self, range: &MemoryRegion, f: &mut F) -> Result<(), MapError> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, { self.mapping.walk_range(range, f) } @@ -341,14 +353,13 @@ mod tests { use super::*; use crate::{ MapError, VirtualAddress, - descriptor::Attributes, paging::{BITS_PER_LEVEL, MemoryRegion, PAGE_SIZE}, }; const MAX_ADDRESS_FOR_ROOT_LEVEL_1: usize = 1 << 39; - const DEVICE_NGNRE: Attributes = Attributes::ATTRIBUTE_INDEX_0; - const NORMAL_CACHEABLE: Attributes = - Attributes::ATTRIBUTE_INDEX_1.union(Attributes::INNER_SHAREABLE); + const DEVICE_NGNRE: Stage1Attributes = Stage1Attributes::ATTRIBUTE_INDEX_0; + const NORMAL_CACHEABLE: Stage1Attributes = + Stage1Attributes::ATTRIBUTE_INDEX_1.union(Stage1Attributes::INNER_SHAREABLE); #[test] fn map_valid() { @@ -360,7 +371,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, 1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -377,7 +388,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, PAGE_SIZE * 2), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -397,7 +408,7 @@ mod tests { MAX_ADDRESS_FOR_ROOT_LEVEL_1 - 1, MAX_ADDRESS_FOR_ROOT_LEVEL_1 ), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -414,7 +425,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(PAGE_SIZE * 1023, PAGE_SIZE * 1025), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -431,7 +442,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, MAX_ADDRESS_FOR_ROOT_LEVEL_1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -448,7 +459,7 @@ mod tests { idmap .map_range_with_constraints( &MemoryRegion::new(BLOCK_SIZE, 2 * BLOCK_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, Constraints::NO_BLOCK_MAPPINGS, ) .unwrap(); @@ -460,7 +471,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(BLOCK_SIZE, BLOCK_SIZE + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Ok(()) ); @@ -472,7 +483,7 @@ mod tests { idmap .map_range( &MemoryRegion::new(BLOCK_SIZE, 2 * BLOCK_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .ok(); // SAFETY: This doesn't actually activate the page table in tests, it just treats it as @@ -484,7 +495,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(BLOCK_SIZE - PAGE_SIZE, 2 * BLOCK_SIZE + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Ok(()) ); @@ -494,7 +505,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(BLOCK_SIZE, BLOCK_SIZE + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Ok(()) ); @@ -503,7 +514,10 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(BLOCK_SIZE, BLOCK_SIZE + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED | Attributes::READ_ONLY, + NORMAL_CACHEABLE + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED + | Stage1Attributes::READ_ONLY, ), Err(MapError::BreakBeforeMakeViolation(MemoryRegion::new( BLOCK_SIZE, @@ -516,7 +530,10 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, BLOCK_SIZE + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED | Attributes::READ_ONLY, + NORMAL_CACHEABLE + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED + | Stage1Attributes::READ_ONLY, ), Err(MapError::BreakBeforeMakeViolation(MemoryRegion::new( BLOCK_SIZE, @@ -526,7 +543,10 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, BLOCK_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED | Attributes::READ_ONLY, + NORMAL_CACHEABLE + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED + | Stage1Attributes::READ_ONLY, ), Ok(()) ); @@ -535,7 +555,10 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, BLOCK_SIZE), - DEVICE_NGNRE | Attributes::VALID | Attributes::ACCESSED | Attributes::NON_GLOBAL, + DEVICE_NGNRE + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED + | Stage1Attributes::NON_GLOBAL, ), Err(MapError::BreakBeforeMakeViolation(MemoryRegion::new( 0, PAGE_SIZE @@ -562,7 +585,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, 2 * PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Ok(()) ); @@ -572,9 +595,9 @@ mod tests { idmap.map_range( &MemoryRegion::new(0, PAGE_SIZE), NORMAL_CACHEABLE - | Attributes::VALID - | Attributes::ACCESSED - | Attributes::NON_GLOBAL, + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED + | Stage1Attributes::NON_GLOBAL, ), Ok(()) ); @@ -583,7 +606,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Err(MapError::BreakBeforeMakeViolation(MemoryRegion::new( 0, PAGE_SIZE @@ -599,7 +622,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ), Ok(()) ); @@ -616,7 +639,7 @@ mod tests { MAX_ADDRESS_FOR_ROOT_LEVEL_1, MAX_ADDRESS_FOR_ROOT_LEVEL_1 + 1, ), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Err(MapError::AddressRange(VirtualAddress( MAX_ADDRESS_FOR_ROOT_LEVEL_1 + PAGE_SIZE @@ -627,7 +650,7 @@ mod tests { assert_eq!( idmap.map_range( &MemoryRegion::new(0, MAX_ADDRESS_FOR_ROOT_LEVEL_1 + 1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Err(MapError::AddressRange(VirtualAddress( MAX_ADDRESS_FOR_ROOT_LEVEL_1 + PAGE_SIZE @@ -643,10 +666,10 @@ mod tests { .map_range( &MemoryRegion::new(0, PAGE_SIZE << BITS_PER_LEVEL), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); // SAFETY: This doesn't actually activate the page table in tests, it just treats it as @@ -656,15 +679,18 @@ mod tests { .map_range( &MemoryRegion::new(0, PAGE_SIZE), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); let r = idmap.map_range( &MemoryRegion::new(PAGE_SIZE, 2 * PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ); unsafe { idmap.deactivate(ttbr) }; r.unwrap(); @@ -676,10 +702,10 @@ mod tests { .map_range( &MemoryRegion::new(0, PAGE_SIZE * 2), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); // SAFETY: This doesn't actually activate the page table in tests, it just treats it as @@ -694,7 +720,10 @@ mod tests { assert!( idmap .modify_range(&MemoryRegion::new(PAGE_SIZE * 2, 1), &|_range, entry| { - entry.modify_flags(Attributes::SWFLAG_0, Attributes::from_bits(0usize).unwrap()) + entry.modify_flags( + Stage1Attributes::SWFLAG_0, + Stage1Attributes::from_bits(0usize).unwrap(), + ) },) .is_err() ); @@ -711,7 +740,10 @@ mod tests { idmap .modify_range(&MemoryRegion::new(1, PAGE_SIZE), &|_range, entry| { if !entry.is_table() { - entry.modify_flags(Attributes::SWFLAG_0, Attributes::NON_GLOBAL)?; + entry.modify_flags( + Stage1Attributes::SWFLAG_0, + Stage1Attributes::NON_GLOBAL, + )?; } Ok(()) }) @@ -721,8 +753,8 @@ mod tests { .modify_range(&MemoryRegion::new(1, PAGE_SIZE), &|_range, entry| { if !entry.is_table() { entry.modify_flags( - Attributes::SWFLAG_0, - Attributes::from_bits(0usize).unwrap(), + Stage1Attributes::SWFLAG_0, + Stage1Attributes::from_bits(0usize).unwrap(), )?; } Ok(()) @@ -731,7 +763,7 @@ mod tests { idmap .modify_range(&MemoryRegion::new(1, PAGE_SIZE), &|range, entry| { if !entry.is_table() { - assert!(entry.flags().contains(Attributes::SWFLAG_0)); + assert!(entry.flags().contains(Stage1Attributes::SWFLAG_0)); assert_eq!(range.end() - range.start(), PAGE_SIZE); } Ok(()) @@ -752,22 +784,22 @@ mod tests { idmap .map_range( &MemoryRegion::new(0, BLOCK_RANGE), - NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::SWFLAG_0, + NORMAL_CACHEABLE | Stage1Attributes::NON_GLOBAL | Stage1Attributes::SWFLAG_0, ) .unwrap(); idmap .map_range( &MemoryRegion::new(0, PAGE_SIZE), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); idmap .modify_range(&MemoryRegion::new(0, BLOCK_RANGE), &|range, entry| { if entry.level() == 3 { - let has_swflag = entry.flags().contains(Attributes::SWFLAG_0); + let has_swflag = entry.flags().contains(Stage1Attributes::SWFLAG_0); let is_first_page = range.start().0 == 0usize; assert!(has_swflag != is_first_page); } @@ -792,14 +824,14 @@ mod tests { idmap .map_range( &MemoryRegion::new(0, PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); // Unmap the whole table's worth of address space. idmap .map_range( &MemoryRegion::new(0, PAGE_SIZE * 512 * 512), - Attributes::empty(), + Stage1Attributes::empty(), ) .unwrap(); // All entries in the top-level table should be 0. @@ -833,14 +865,14 @@ mod tests { idmap .map_range( &MemoryRegion::new(ROOT_GRANULARITY, ROOT_GRANULARITY + PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); // Unmap the second entry of the root table. idmap .map_range( &MemoryRegion::new(ROOT_GRANULARITY, ROOT_GRANULARITY * 2), - Attributes::empty(), + Stage1Attributes::empty(), ) .unwrap(); // All entries in the top-level table should be 0. @@ -873,12 +905,15 @@ mod tests { idmap .map_range( &MemoryRegion::new(0, PAGE_SIZE * 2), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); // Unmap the pages again. idmap - .map_range(&MemoryRegion::new(0, PAGE_SIZE * 2), Attributes::empty()) + .map_range( + &MemoryRegion::new(0, PAGE_SIZE * 2), + Stage1Attributes::empty(), + ) .unwrap(); // Compact to remove the subtables. idmap.compact_subtables(); @@ -913,12 +948,15 @@ mod tests { idmap .map_range( &MemoryRegion::new(0, BLOCK_SIZE * 2), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); // Unmap the blocks again. idmap - .map_range(&MemoryRegion::new(0, BLOCK_SIZE * 2), Attributes::empty()) + .map_range( + &MemoryRegion::new(0, BLOCK_SIZE * 2), + Stage1Attributes::empty(), + ) .unwrap(); // Compact to remove the subtables. idmap.compact_subtables(); @@ -948,7 +986,7 @@ mod tests { idmap .map_range( &MemoryRegion::new(0, PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); idmap @@ -957,7 +995,7 @@ mod tests { &mut |_, descriptor, _| { assert!(!descriptor.is_valid()); assert_eq!(descriptor.bits(), 0); - assert_eq!(descriptor.flags(), Attributes::empty()); + assert_eq!(descriptor.flags(), Stage1Attributes::empty()); assert_eq!(descriptor.output_address(), PhysicalAddress(0)); Ok(()) }, @@ -977,14 +1015,14 @@ mod tests { idmap .map_range( &MemoryRegion::new(PAGE_SIZE, PAGE_SIZE * 3), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); // Use `modify_range` to unmap the pages. idmap .modify_range( &MemoryRegion::new(PAGE_SIZE, PAGE_SIZE * 3), - &|_, descriptor| descriptor.set(PhysicalAddress(0), Attributes::empty()), + &|_, descriptor| descriptor.set(PhysicalAddress(0), Stage1Attributes::empty()), ) .unwrap(); // Compact to remove the subtables. @@ -1009,9 +1047,21 @@ mod tests { #[test] fn table_sizes() { - assert_eq!(IdMap::new(1, 0, TranslationRegime::El1And0).size(), 1 << 48); - assert_eq!(IdMap::new(1, 1, TranslationRegime::El1And0).size(), 1 << 39); - assert_eq!(IdMap::new(1, 2, TranslationRegime::El1And0).size(), 1 << 30); - assert_eq!(IdMap::new(1, 3, TranslationRegime::El1And0).size(), 1 << 21); + assert_eq!( + IdMap::::new(1, 0, TranslationRegime::El1And0).size(), + 1 << 48 + ); + assert_eq!( + IdMap::::new(1, 1, TranslationRegime::El1And0).size(), + 1 << 39 + ); + assert_eq!( + IdMap::::new(1, 2, TranslationRegime::El1And0).size(), + 1 << 30 + ); + assert_eq!( + IdMap::::new(1, 3, TranslationRegime::El1And0).size(), + 1 << 21 + ); } } diff --git a/src/lib.rs b/src/lib.rs index cba0cf7..b8877e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,20 +19,20 @@ //! # #[cfg(feature = "alloc")] { //! use aarch64_paging::{ //! idmap::IdMap, -//! descriptor::Attributes, +//! descriptor::Stage1Attributes, //! paging::{MemoryRegion, TranslationRegime}, //! }; //! //! const ASID: usize = 1; //! const ROOT_LEVEL: usize = 1; -//! const NORMAL_CACHEABLE: Attributes = Attributes::ATTRIBUTE_INDEX_1.union(Attributes::INNER_SHAREABLE); +//! const NORMAL_CACHEABLE: Stage1Attributes = Stage1Attributes::ATTRIBUTE_INDEX_1.union(Stage1Attributes::INNER_SHAREABLE); //! //! // Create a new EL1 page table with identity mapping. //! let mut idmap = IdMap::new(ASID, ROOT_LEVEL, TranslationRegime::El1And0); //! // Map a 2 MiB region of memory as read-write. //! idmap.map_range( //! &MemoryRegion::new(0x80200000, 0x80400000), -//! NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::VALID | Attributes::ACCESSED, +//! NORMAL_CACHEABLE | Stage1Attributes::NON_GLOBAL | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, //! ).unwrap(); //! // SAFETY: Everything the program uses is within the 2 MiB region mapped above. //! unsafe { @@ -63,7 +63,8 @@ extern crate alloc; use core::arch::asm; use core::sync::atomic::{AtomicUsize, Ordering}; use descriptor::{ - Attributes, Descriptor, DescriptorBits, PhysicalAddress, UpdatableDescriptor, VirtualAddress, + Descriptor, DescriptorBits, PagingAttributes, PhysicalAddress, Stage1Attributes, + UpdatableDescriptor, VirtualAddress, }; use paging::{Constraints, MemoryRegion, RootTable, Translation, TranslationRegime, VaRange}; use thiserror::Error; @@ -85,8 +86,8 @@ pub enum MapError { #[error("Error updating page table entry {0:?}")] PteUpdateFault(DescriptorBits), /// The requested flags are not supported for this mapping - #[error("Flags {0:?} unsupported for mapping.")] - InvalidFlags(Attributes), + #[error("Flags {0:#x} unsupported for mapping.")] + InvalidFlags(usize), /// Updating the range violates break-before-make rules and the mapping is live #[error("Cannot remap region {0} while translation is live.")] BreakBeforeMakeViolation(MemoryRegion), @@ -100,8 +101,8 @@ pub enum MapError { /// switch back to a previous static page table, and then `activate` again after making the desired /// changes. #[derive(Debug)] -pub struct Mapping { - root: RootTable, +pub struct Mapping, A: PagingAttributes = Stage1Attributes> { + root: RootTable, asid: usize, active_count: AtomicUsize, } @@ -117,7 +118,7 @@ fn wait_for_tlb_maintenance() { } } -impl Mapping { +impl, A: PagingAttributes> Mapping { /// Creates a new page table with the given ASID, root level and translation mapping. pub fn new( translation: T, @@ -230,6 +231,14 @@ impl Mapping { previous_ttbr = out(reg) previous_ttbr, options(preserves_flags), ), + (TranslationRegime::Stage2, VaRange::Lower) => asm!( + "mrs {previous_ttbr}, vttbr_el2", + "msr vttbr_el2, {ttbrval}", + "isb", + ttbrval = in(reg) self.root_address().0 | (self.asid << 48), + previous_ttbr = out(reg) previous_ttbr, + options(preserves_flags), + ), _ => { panic!("Invalid combination of exception level and VA range."); } @@ -302,6 +311,17 @@ impl Mapping { (TranslationRegime::El3, VaRange::Lower) => { panic!("EL3 page table can't safety be deactivated."); } + (TranslationRegime::Stage2, VaRange::Lower) => asm!( + // For Stage 2, we invalidate using the current VTTBR (which has our VMID), + // then restore the previous VTTBR. + "tlbi vmalls12e1", + "dsb nsh", + "isb", + "msr vttbr_el2, {ttbrval}", + "isb", + ttbrval = in(reg) previous_ttbr, + options(preserves_flags), + ), _ => { panic!("Invalid combination of exception level and VA range."); } @@ -314,11 +334,11 @@ impl Mapping { /// without violating architectural break-before-make (BBM) requirements. fn check_range_bbm(&self, range: &MemoryRegion, updater: &F) -> Result<(), MapError> where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, { self.root.visit_range( range, - &mut |mr: &MemoryRegion, d: &Descriptor, level: usize| { + &mut |mr: &MemoryRegion, d: &Descriptor, level: usize| { let err = MapError::BreakBeforeMakeViolation(mr.clone()); let mut desc = UpdatableDescriptor::clone_from(d, level); @@ -348,9 +368,13 @@ impl Mapping { // deactivated, at which point TLB invalidation would have occurred, and so no TLB // maintenance is needed. self.root - .visit_range(range, &mut |mr: &MemoryRegion, _: &Descriptor, _: usize| { - Ok(self.root.translation_regime().invalidate_va(mr.start())) - }) + .visit_range( + range, + &mut |mr: &MemoryRegion, _: &Descriptor, _: usize| { + self.root.translation_regime().invalidate_va(mr.start()); + Ok(()) + }, + ) .unwrap(); wait_for_tlb_maintenance(); @@ -360,14 +384,14 @@ impl Mapping { /// Maps the given range of virtual addresses to the corresponding range of physical addresses /// starting at `pa`, with the given flags, taking the given constraints into account. /// - /// To unmap a range, pass `flags` which don't contain the `Attributes::VALID` bit. In this case - /// the `pa` is ignored. + /// To unmap a range, pass `flags` which don't contain the [`PagingAttributes::VALID`] bit. + /// In this case the `pa` is ignored. /// /// This should generally only be called while the page table is not active. In particular, any /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page /// table is active. This function writes block and page entries, but only maps them if `flags` - /// contains `Attributes::VALID`, otherwise the entries remain invalid. + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -384,15 +408,15 @@ impl Mapping { &mut self, range: &MemoryRegion, pa: PhysicalAddress, - flags: Attributes, + flags: A, constraints: Constraints, ) -> Result<(), MapError> { if self.active() { - let c = |mr: &MemoryRegion, d: &mut UpdatableDescriptor| { + let c = |mr: &MemoryRegion, d: &mut UpdatableDescriptor| { let mask = !(paging::granularity_at_level(d.level()) - 1); let pa = (mr.start() - range.start() + pa.0) & mask; let flags = if d.level() == 3 { - flags | Attributes::TABLE_OR_PAGE + flags | A::TABLE_OR_PAGE } else { flags }; @@ -443,7 +467,7 @@ impl Mapping { /// and modifying those would violate architectural break-before-make (BBM) requirements. pub fn modify_range(&mut self, range: &MemoryRegion, f: &F) -> Result<(), MapError> where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, { if self.active() { self.check_range_bbm(range, f)?; @@ -475,7 +499,7 @@ impl Mapping { /// largest virtual address covered by the page table given its root level. pub fn walk_range(&self, range: &MemoryRegion, f: &mut F) -> Result<(), MapError> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, { self.root.walk_range(range, f) } @@ -531,7 +555,7 @@ impl Mapping { } } -impl Drop for Mapping { +impl, A: PagingAttributes> Drop for Mapping { fn drop(&mut self) { if self.active() { panic!("Dropping active page table mapping!"); @@ -550,13 +574,25 @@ mod tests { #[test] #[should_panic] fn no_el2_asid() { - Mapping::new(IdTranslation, 1, 1, TranslationRegime::El2, VaRange::Lower); + Mapping::::new( + IdTranslation::new(), + 1, + 1, + TranslationRegime::El2, + VaRange::Lower, + ); } #[cfg(feature = "alloc")] #[test] #[should_panic] fn no_el3_asid() { - Mapping::new(IdTranslation, 1, 1, TranslationRegime::El3, VaRange::Lower); + Mapping::::new( + IdTranslation::new(), + 1, + 1, + TranslationRegime::El3, + VaRange::Lower, + ); } } diff --git a/src/linearmap.rs b/src/linearmap.rs index e1095ea..84ad7d9 100644 --- a/src/linearmap.rs +++ b/src/linearmap.rs @@ -8,23 +8,28 @@ use crate::{ MapError, Mapping, - descriptor::{Attributes, Descriptor, PhysicalAddress, UpdatableDescriptor, VirtualAddress}, + descriptor::{ + Descriptor, PagingAttributes, PhysicalAddress, Stage1Attributes, UpdatableDescriptor, + VirtualAddress, + }, paging::{ Constraints, MemoryRegion, PAGE_SIZE, PageTable, Translation, TranslationRegime, VaRange, deallocate, is_aligned, }, }; +use core::marker::PhantomData; use core::ptr::NonNull; /// Linear mapping, where every virtual address is either unmapped or mapped to an IPA with a fixed /// offset. #[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub struct LinearTranslation { +pub struct LinearTranslation { /// The offset from a virtual address to the corresponding (intermediate) physical address. offset: isize, + _phantom: PhantomData, } -impl LinearTranslation { +impl LinearTranslation { /// Constructs a new linear translation, which will map a virtual address `va` to the /// (intermediate) physical address `va + offset`. /// @@ -36,7 +41,10 @@ impl LinearTranslation { offset, PAGE_SIZE, ); } - Self { offset } + Self { + offset, + _phantom: PhantomData, + } } fn virtual_to_physical(&self, va: VirtualAddress) -> Result { @@ -48,8 +56,8 @@ impl LinearTranslation { } } -impl Translation for LinearTranslation { - fn allocate_table(&mut self) -> (NonNull, PhysicalAddress) { +impl Translation for LinearTranslation { + fn allocate_table(&mut self) -> (NonNull>, PhysicalAddress) { let table = PageTable::new(); // Assume that the same linear mapping is used everywhere. let va = VirtualAddress(table.as_ptr() as usize); @@ -60,7 +68,7 @@ impl Translation for LinearTranslation { (table, pa) } - unsafe fn deallocate_table(&mut self, page_table: NonNull) { + unsafe fn deallocate_table(&mut self, page_table: NonNull>) { // SAFETY: Our caller promises that the memory was allocated by `allocate_table` on this // `LinearTranslation` and not yet deallocated. `allocate_table` used the global allocator // and appropriate layout by calling `PageTable::new()`. @@ -69,13 +77,13 @@ impl Translation for LinearTranslation { } } - fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull { + fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull> { let signed_pa = pa.0 as isize; if signed_pa < 0 { panic!("Invalid physical address {} for pagetable", pa); } if let Some(va) = signed_pa.checked_sub(self.offset) { - if let Some(ptr) = NonNull::new(va as *mut PageTable) { + if let Some(ptr) = NonNull::new(va as *mut PageTable) { ptr } else { panic!( @@ -100,11 +108,11 @@ fn checked_add_to_unsigned(a: isize, b: isize) -> Option { /// This assumes that the same linear mapping is used both for the page table being managed, and for /// code that is managing it. #[derive(Debug)] -pub struct LinearMap { - mapping: Mapping, +pub struct LinearMap { + mapping: Mapping, A>, } -impl LinearMap { +impl LinearMap { /// Creates a new identity-mapping page table with the given ASID, root level and offset, for /// use in the given TTBR. /// @@ -179,7 +187,7 @@ impl LinearMap { /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page /// table is active. This function writes block and page entries, but only maps them if `flags` - /// contains `Attributes::VALID`, otherwise the entries remain invalid. + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -195,7 +203,7 @@ impl LinearMap { /// /// Returns [`MapError::BreakBeforeMakeViolation`] if the range intersects with live mappings, /// and modifying those would violate architectural break-before-make (BBM) requirements. - pub fn map_range(&mut self, range: &MemoryRegion, flags: Attributes) -> Result<(), MapError> { + pub fn map_range(&mut self, range: &MemoryRegion, flags: A) -> Result<(), MapError> { self.map_range_with_constraints(range, flags, Constraints::empty()) } @@ -206,7 +214,7 @@ impl LinearMap { /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page /// table is active. This function writes block and page entries, but only maps them if `flags` - /// contains `Attributes::VALID`, otherwise the entries remain invalid. + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -225,7 +233,7 @@ impl LinearMap { pub fn map_range_with_constraints( &mut self, range: &MemoryRegion, - flags: Attributes, + flags: A, constraints: Constraints, ) -> Result<(), MapError> { let pa = self @@ -274,7 +282,7 @@ impl LinearMap { /// and modifying those would violate architectural break-before-make (BBM) requirements. pub fn modify_range(&mut self, range: &MemoryRegion, f: &F) -> Result<(), MapError> where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, { self.mapping.modify_range(range, f) } @@ -304,7 +312,7 @@ impl LinearMap { /// largest virtual address covered by the page table given its root level. pub fn walk_range(&self, range: &MemoryRegion, f: &mut F) -> Result<(), MapError> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, { self.mapping.walk_range(range, f) } @@ -356,15 +364,14 @@ mod tests { use super::*; use crate::{ MapError, - descriptor::Attributes, paging::{BITS_PER_LEVEL, MemoryRegion, PAGE_SIZE}, }; const MAX_ADDRESS_FOR_ROOT_LEVEL_1: usize = 1 << 39; const GIB_512_S: isize = 512 * 1024 * 1024 * 1024; const GIB_512: usize = 512 * 1024 * 1024 * 1024; - const NORMAL_CACHEABLE: Attributes = - Attributes::ATTRIBUTE_INDEX_1.union(Attributes::INNER_SHAREABLE); + const NORMAL_CACHEABLE: Stage1Attributes = + Stage1Attributes::ATTRIBUTE_INDEX_1.union(Stage1Attributes::INNER_SHAREABLE); #[test] fn map_valid() { @@ -373,7 +380,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(0, 1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -383,7 +390,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(0, PAGE_SIZE * 2), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -396,7 +403,7 @@ mod tests { MAX_ADDRESS_FOR_ROOT_LEVEL_1 - 1, MAX_ADDRESS_FOR_ROOT_LEVEL_1 ), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -414,7 +421,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(0, MAX_ADDRESS_FOR_ROOT_LEVEL_1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -433,7 +440,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(PAGE_SIZE, PAGE_SIZE + 1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -449,7 +456,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(PAGE_SIZE, PAGE_SIZE * 3), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -468,7 +475,7 @@ mod tests { MAX_ADDRESS_FOR_ROOT_LEVEL_1 - 1, MAX_ADDRESS_FOR_ROOT_LEVEL_1 ), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -486,7 +493,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(LEVEL_2_BLOCK_SIZE, MAX_ADDRESS_FOR_ROOT_LEVEL_1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Ok(()) ); @@ -503,7 +510,7 @@ mod tests { MAX_ADDRESS_FOR_ROOT_LEVEL_1, MAX_ADDRESS_FOR_ROOT_LEVEL_1 + 1, ), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Err(MapError::AddressRange(VirtualAddress( MAX_ADDRESS_FOR_ROOT_LEVEL_1 + PAGE_SIZE @@ -514,7 +521,7 @@ mod tests { assert_eq!( pagetable.map_range( &MemoryRegion::new(0, MAX_ADDRESS_FOR_ROOT_LEVEL_1 + 1), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED ), Err(MapError::AddressRange(VirtualAddress( MAX_ADDRESS_FOR_ROOT_LEVEL_1 + PAGE_SIZE @@ -535,7 +542,7 @@ mod tests { #[test] fn physical_address_in_range_ttbr0() { - let translation = LinearTranslation::new(4096); + let translation = LinearTranslation::::new(4096); assert_eq!( translation.physical_to_virtual(PhysicalAddress(8192)), NonNull::new(4096 as *mut PageTable).unwrap(), @@ -549,14 +556,14 @@ mod tests { #[test] #[should_panic] fn physical_address_to_zero_ttbr0() { - let translation = LinearTranslation::new(4096); + let translation: LinearTranslation = LinearTranslation::new(4096); translation.physical_to_virtual(PhysicalAddress(4096)); } #[test] #[should_panic] fn physical_address_out_of_range_ttbr0() { - let translation = LinearTranslation::new(4096); + let translation: LinearTranslation = LinearTranslation::new(4096); translation.physical_to_virtual(PhysicalAddress(-4096_isize as usize)); } @@ -580,7 +587,7 @@ mod tests { fn physical_address_to_zero_ttbr1() { // Map the 512 GiB region at the top of virtual address space to the bottom of physical // address space. - let translation = LinearTranslation::new(GIB_512_S); + let translation: LinearTranslation = LinearTranslation::new(GIB_512_S); translation.physical_to_virtual(PhysicalAddress(GIB_512)); } @@ -589,13 +596,13 @@ mod tests { fn physical_address_out_of_range_ttbr1() { // Map the 512 GiB region at the top of virtual address space to the bottom of physical // address space. - let translation = LinearTranslation::new(GIB_512_S); + let translation: LinearTranslation = LinearTranslation::new(GIB_512_S); translation.physical_to_virtual(PhysicalAddress(-4096_isize as usize)); } #[test] fn virtual_address_out_of_range() { - let translation = LinearTranslation::new(-4096); + let translation: LinearTranslation = LinearTranslation::new(-4096); let va = VirtualAddress(1024); assert_eq!( translation.virtual_to_physical(va), @@ -607,7 +614,7 @@ mod tests { fn virtual_address_range_ttbr1() { // Map the 512 GiB region at the top of virtual address space to the bottom of physical // address space. - let translation = LinearTranslation::new(GIB_512_S); + let translation: LinearTranslation = LinearTranslation::new(GIB_512_S); // The first page in the region covered by TTBR1. assert_eq!( @@ -629,7 +636,7 @@ mod tests { pagetable .map_range( &MemoryRegion::new(0, 1 << 30), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); assert_eq!( @@ -643,7 +650,7 @@ mod tests { pagetable .map_range( &MemoryRegion::new(0, 1 << 30), - NORMAL_CACHEABLE | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE | Stage1Attributes::VALID | Stage1Attributes::ACCESSED, ) .unwrap(); assert_eq!( @@ -652,7 +659,7 @@ mod tests { ); } - fn make_map() -> LinearMap { + fn make_map() -> LinearMap { let mut lmap = LinearMap::new(1, 1, 4096, TranslationRegime::El1And0, VaRange::Lower); // Mapping VA range 0x0 - 0x2000 to PA range 0x1000 - 0x3000 lmap.map_range(&MemoryRegion::new(0, PAGE_SIZE * 2), NORMAL_CACHEABLE) @@ -665,7 +672,10 @@ mod tests { let mut lmap = make_map(); assert!( lmap.modify_range(&MemoryRegion::new(PAGE_SIZE * 2, 1), &|_range, entry| { - entry.modify_flags(Attributes::SWFLAG_0, Attributes::from_bits(0usize).unwrap()) + entry.modify_flags( + Stage1Attributes::SWFLAG_0, + Stage1Attributes::from_bits(0usize).unwrap(), + ) },) .is_err() ); @@ -676,14 +686,17 @@ mod tests { let mut lmap = make_map(); lmap.modify_range(&MemoryRegion::new(1, PAGE_SIZE), &|_range, entry| { if !entry.is_table() { - entry.modify_flags(Attributes::SWFLAG_0, Attributes::from_bits(0usize).unwrap())?; + entry.modify_flags( + Stage1Attributes::SWFLAG_0, + Stage1Attributes::from_bits(0usize).unwrap(), + )?; } Ok(()) }) .unwrap(); lmap.modify_range(&MemoryRegion::new(1, PAGE_SIZE), &|range, entry| { if !entry.is_table() { - assert!(entry.flags().contains(Attributes::SWFLAG_0)); + assert!(entry.flags().contains(Stage1Attributes::SWFLAG_0)); assert_eq!(range.end() - range.start(), PAGE_SIZE); } Ok(()) @@ -698,17 +711,20 @@ mod tests { let mut lmap = LinearMap::new(1, 1, 0x1000, TranslationRegime::El1And0, VaRange::Lower); lmap.map_range( &MemoryRegion::new(0, BLOCK_RANGE), - NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::SWFLAG_0, + NORMAL_CACHEABLE | Stage1Attributes::NON_GLOBAL | Stage1Attributes::SWFLAG_0, ) .unwrap(); lmap.map_range( &MemoryRegion::new(0, PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); lmap.modify_range(&MemoryRegion::new(0, BLOCK_RANGE), &|range, entry| { if entry.level() == 3 { - let has_swflag = entry.flags().contains(Attributes::SWFLAG_0); + let has_swflag = entry.flags().contains(Stage1Attributes::SWFLAG_0); let is_first_page = range.start().0 == 0usize; assert!(has_swflag != is_first_page); } @@ -731,10 +747,10 @@ mod tests { lmap.map_range( &MemoryRegion::new(0, BLOCK_SIZE), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); // SAFETY: This doesn't actually activate the page table in tests, it just treats it as @@ -743,24 +759,27 @@ mod tests { lmap.map_range( &MemoryRegion::new(0, PAGE_SIZE), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); lmap.map_range( &MemoryRegion::new(PAGE_SIZE, 2 * PAGE_SIZE), NORMAL_CACHEABLE - | Attributes::NON_GLOBAL - | Attributes::READ_ONLY - | Attributes::VALID - | Attributes::ACCESSED, + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::READ_ONLY + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ) .unwrap(); let r = lmap.map_range( &MemoryRegion::new(PAGE_SIZE, 2 * PAGE_SIZE), - NORMAL_CACHEABLE | Attributes::NON_GLOBAL | Attributes::VALID | Attributes::ACCESSED, + NORMAL_CACHEABLE + | Stage1Attributes::NON_GLOBAL + | Stage1Attributes::VALID + | Stage1Attributes::ACCESSED, ); unsafe { lmap.deactivate(ttbr) }; r.unwrap(); diff --git a/src/paging.rs b/src/paging.rs index ee8b11b..469a534 100644 --- a/src/paging.rs +++ b/src/paging.rs @@ -7,7 +7,8 @@ use crate::MapError; use crate::descriptor::{ - Attributes, Descriptor, PhysicalAddress, UpdatableDescriptor, VirtualAddress, + Descriptor, PagingAttributes, PhysicalAddress, Stage1Attributes, UpdatableDescriptor, + VirtualAddress, }; #[cfg(feature = "alloc")] @@ -19,7 +20,6 @@ use core::fmt::{self, Debug, Display, Formatter}; use core::marker::PhantomData; use core::ops::Range; use core::ptr::NonNull; -use core::sync::atomic::AtomicUsize; const PAGE_SHIFT: usize = 12; @@ -57,6 +57,8 @@ pub enum TranslationRegime { El2And0, /// Non-secure EL1&0, stage 1. El1And0, + /// Non-secure Stage 2. + Stage2, } impl TranslationRegime { @@ -93,6 +95,11 @@ impl TranslationRegime { va = in(reg) va, options(preserves_flags, nostack), ), + TranslationRegime::Stage2 => asm!( + "tlbi ipas2e1is, {va}", + va = in(reg) va, + options(preserves_flags, nostack), + ), }; }; } @@ -111,10 +118,10 @@ pub(crate) fn granularity_at_level(level: usize) -> usize { /// An implementation of this trait needs to be provided to the mapping routines, so that the /// physical addresses used in the page tables can be converted into virtual addresses that can be /// used to access their contents from the code. -pub trait Translation { +pub trait Translation { /// Allocates a zeroed page, which is already mapped, to be used for a new subtable of some /// pagetable. Returns both a pointer to the page and its physical address. - fn allocate_table(&mut self) -> (NonNull, PhysicalAddress); + fn allocate_table(&mut self) -> (NonNull>, PhysicalAddress); /// Deallocates the page which was previous allocated by [`allocate_table`](Self::allocate_table). /// @@ -122,10 +129,10 @@ pub trait Translation { /// /// The memory must have been allocated by `allocate_table` on the same `Translation`, and not /// yet deallocated. - unsafe fn deallocate_table(&mut self, page_table: NonNull); + unsafe fn deallocate_table(&mut self, page_table: NonNull>); /// Given the physical address of a subtable, returns the virtual address at which it is mapped. - fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull; + fn physical_to_virtual(&self, pa: PhysicalAddress) -> NonNull>; } impl MemoryRegion { @@ -204,15 +211,15 @@ bitflags! { } /// A complete hierarchy of page tables including all levels. -pub struct RootTable { - table: PageTableWithLevel, +pub struct RootTable, A: PagingAttributes = Stage1Attributes> { + table: PageTableWithLevel, translation: T, pa: PhysicalAddress, translation_regime: TranslationRegime, va_range: VaRange, } -impl RootTable { +impl, A: PagingAttributes> RootTable { /// Creates a new page table starting at the given root level. /// /// The level must be between 0 and 3; level -1 (for 52-bit addresses with LPA2) is not @@ -253,9 +260,9 @@ impl RootTable { /// Recursively maps a range into the pagetable hierarchy starting at the root level, mapping /// the pages to the corresponding physical address range starting at `pa`. Block and page - /// entries will be written to, but will only be mapped if `flags` contains `Attributes::VALID`. + /// entries will be written to, but will only be mapped if `flags` contains [`PagingAttributes::VALID`]. /// - /// To unmap a range, pass `flags` which don't contain the `Attributes::VALID` bit. In this case + /// To unmap a range, pass `flags` which don't contain the [`PagingAttributes::VALID`] bit. In this case /// the `pa` is ignored. /// /// Returns an error if the virtual address range is out of the range covered by the pagetable, @@ -264,11 +271,11 @@ impl RootTable { &mut self, range: &MemoryRegion, pa: PhysicalAddress, - flags: Attributes, + flags: A, constraints: Constraints, ) -> Result<(), MapError> { - if flags.contains(Attributes::TABLE_OR_PAGE) { - return Err(MapError::InvalidFlags(Attributes::TABLE_OR_PAGE)); + if flags.contains(A::TABLE_OR_PAGE) { + return Err(MapError::InvalidFlags(flags.bits())); } self.verify_region(range)?; self.table @@ -322,7 +329,8 @@ impl RootTable { /// This should generally only be called while the page table is not active. In particular, any /// change that may require break-before-make per the architecture must be made while the page /// table is inactive. Mapping a previously unmapped memory range may be done while the page - /// table is active. + /// table is active. This function writes block and page entries, but only maps them if `flags` + /// contains [`PagingAttributes::VALID`], otherwise the entries remain invalid. /// /// # Errors /// @@ -342,7 +350,7 @@ impl RootTable { live: bool, ) -> Result where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, { self.verify_region(range)?; self.table.modify_range( @@ -379,7 +387,7 @@ impl RootTable { /// largest virtual address covered by the page table given its root level. pub fn walk_range(&self, range: &MemoryRegion, f: &mut F) -> Result<(), MapError> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), ()>, { self.visit_range(range, &mut |mr, desc, level| { f(mr, desc, level).map_err(|_| MapError::PteUpdateFault(desc.bits())) @@ -399,7 +407,7 @@ impl RootTable { // Private version of `walk_range` using a closure that returns MapError on error pub(crate) fn visit_range(&self, range: &MemoryRegion, f: &mut F) -> Result<(), MapError> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), MapError>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), MapError>, { self.verify_region(range)?; self.table.visit_range(&self.translation, range, f) @@ -439,7 +447,7 @@ impl RootTable { } } -impl Debug for RootTable { +impl, A: PagingAttributes> Debug for RootTable { fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> { writeln!( f, @@ -451,7 +459,7 @@ impl Debug for RootTable { } } -impl Drop for RootTable { +impl, A: PagingAttributes> Drop for RootTable { fn drop(&mut self) { // SAFETY: We created the table in `RootTable::new` by calling `PageTableWithLevel::new` // with `self.translation`. Subtables were similarly created by @@ -489,20 +497,20 @@ impl Iterator for ChunkedIterator<'_> { /// Smart pointer which owns a [`PageTable`] and knows what level it is at. This allows it to /// implement methods to walk the page table hierachy which require knowing the starting level. #[derive(Debug)] -pub(crate) struct PageTableWithLevel { - table: NonNull, +pub(crate) struct PageTableWithLevel, A: PagingAttributes = Stage1Attributes> { + table: NonNull>, level: usize, _translation: PhantomData, } // SAFETY: The underlying PageTable is process-wide and can be safely accessed from any thread // with appropriate synchronization. This type manages ownership for the raw pointer. -unsafe impl Send for PageTableWithLevel {} +unsafe impl + Send, A: PagingAttributes> Send for PageTableWithLevel {} // SAFETY: &Self only allows reading from the page table, which is safe to do from any thread. -unsafe impl Sync for PageTableWithLevel {} +unsafe impl + Sync, A: PagingAttributes> Sync for PageTableWithLevel {} -impl PageTableWithLevel { +impl, A: PagingAttributes> PageTableWithLevel { /// Allocates a new, zeroed, appropriately-aligned page table with the given translation, /// returning both a pointer to it and its physical address. fn new(translation: &mut T, level: usize) -> (Self, PhysicalAddress) { @@ -516,7 +524,7 @@ impl PageTableWithLevel { ) } - pub(crate) fn from_pointer(table: NonNull, level: usize) -> Self { + pub(crate) fn from_pointer(table: NonNull>, level: usize) -> Self { Self { table, level, @@ -525,7 +533,7 @@ impl PageTableWithLevel { } /// Returns a reference to the descriptor corresponding to a given virtual address. - fn get_entry(&self, va: VirtualAddress) -> &Descriptor { + fn get_entry(&self, va: VirtualAddress) -> &Descriptor { let shift = PAGE_SHIFT + (LEAF_LEVEL - self.level) * BITS_PER_LEVEL; let index = (va.0 >> shift) % (1 << BITS_PER_LEVEL); // SAFETY: We know that the pointer is properly aligned, dereferenced and initialised, and @@ -536,7 +544,7 @@ impl PageTableWithLevel { } /// Returns a mutable reference to the descriptor corresponding to a given virtual address. - fn get_entry_mut(&mut self, va: VirtualAddress) -> &mut Descriptor { + fn get_entry_mut(&mut self, va: VirtualAddress) -> &mut Descriptor { let shift = PAGE_SHIFT + (LEAF_LEVEL - self.level) * BITS_PER_LEVEL; let index = (va.0 >> shift) % (1 << BITS_PER_LEVEL); // SAFETY: We know that the pointer is properly aligned, dereferenced and initialised, and @@ -551,16 +559,14 @@ impl PageTableWithLevel { fn split_entry( translation: &mut T, chunk: &MemoryRegion, - entry: &mut Descriptor, + entry: &mut Descriptor, level: usize, ) -> Self { let granularity = granularity_at_level(level); let (mut subtable, subtable_pa) = Self::new(translation, level + 1); let old_flags = entry.flags(); let old_pa = entry.output_address(); - if !old_flags.contains(Attributes::TABLE_OR_PAGE) - && (!old_flags.is_empty() || old_pa.0 != 0) - { + if !old_flags.contains(A::TABLE_OR_PAGE) && (!old_flags.is_empty() || old_pa.0 != 0) { // `old` was a block entry, so we need to split it. // Recreate the entire block in the newly added table. let a = align_down(chunk.0.start.0, granularity); @@ -576,15 +582,15 @@ impl PageTableWithLevel { // If `old` was not a block entry, a newly zeroed page will be added to the hierarchy, // which might be live in this case. We rely on the release semantics of the set() below to // ensure that all observers that see the new entry will also see the zeroed contents. - entry.set(subtable_pa, Attributes::TABLE_OR_PAGE | Attributes::VALID); + entry.set(subtable_pa, A::TABLE_OR_PAGE | A::VALID); subtable } /// Maps the the given virtual address range in this pagetable to the corresponding physical /// address range starting at the given `pa`, recursing into any subtables as necessary. To map - /// block and page entries, `Attributes::VALID` must be set in `flags`. + /// block and page entries, [`PagingAttributes::VALID`] must be set in `flags`. /// - /// If `flags` doesn't contain `Attributes::VALID` then the `pa` is ignored. + /// If `flags` doesn't contain [`PagingAttributes::VALID`] then the `pa` is ignored. /// /// Assumes that the entire range is within the range covered by this pagetable. /// @@ -596,7 +602,7 @@ impl PageTableWithLevel { translation: &mut T, range: &MemoryRegion, mut pa: PhysicalAddress, - flags: Attributes, + flags: A, constraints: Constraints, ) { let level = self.level; @@ -606,9 +612,9 @@ impl PageTableWithLevel { let entry = self.get_entry_mut(chunk.0.start); if level == LEAF_LEVEL { - if flags.contains(Attributes::VALID) { + if flags.contains(A::VALID) { // Put down a page mapping. - entry.set(pa, flags | Attributes::TABLE_OR_PAGE); + entry.set(pa, flags | A::TABLE_OR_PAGE); } else { // Put down an invalid entry. entry.set(PhysicalAddress(0), flags); @@ -627,14 +633,14 @@ impl PageTableWithLevel { // Rather than leak the entire subhierarchy, only put down // a block mapping if the region is not already covered by // a table mapping. - if flags.contains(Attributes::VALID) { + if flags.contains(A::VALID) { entry.set(pa, flags); } else { entry.set(PhysicalAddress(0), flags); } } else if chunk.is_block(level) && let Some(mut subtable) = entry.subtable(translation, level) - && !flags.contains(Attributes::VALID) + && !flags.contains(A::VALID) { // There is a subtable but we can remove it. To avoid break-before-make violations // this is only allowed if the new mapping is not valid, i.e. we are unmapping the @@ -704,7 +710,7 @@ impl PageTableWithLevel { if first_entry == 0 { writeln!(f, "0")?; } else { - writeln!(f, "{:?}", Descriptor(AtomicUsize::new(first_entry)))?; + writeln!(f, "{:?}", Descriptor::::new(first_entry))?; } } } @@ -750,7 +756,7 @@ impl PageTableWithLevel { live: bool, ) -> Result where - F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, + F: Fn(&MemoryRegion, &mut UpdatableDescriptor) -> Result<(), ()> + ?Sized, { let mut modified = false; let level = self.level; @@ -786,7 +792,7 @@ impl PageTableWithLevel { /// If the function returns an error, the walk is terminated and the error value is passed on fn visit_range(&self, translation: &T, range: &MemoryRegion, f: &mut F) -> Result<(), E> where - F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), E>, + F: FnMut(&MemoryRegion, &Descriptor, usize) -> Result<(), E>, { let level = self.level; for chunk in range.split(level) { @@ -814,7 +820,7 @@ impl PageTableWithLevel { if let Some(mut subtable) = entry.subtable(translation, self.level) && subtable.compact_subtables(translation) { - entry.set(PhysicalAddress(0), Attributes::empty()); + entry.set(PhysicalAddress(0), A::default()); // SAFETY: The subtable was created with the same translation by // `PageTableWithLevel::new`, and is no longer referenced by this table. We don't @@ -835,7 +841,7 @@ impl PageTableWithLevel { /// - `Some(LEAF_LEVEL)` if it is mapped as a single page /// - `Some(level)` if it is mapped as a block at `level` #[cfg(all(test, feature = "alloc"))] - fn mapping_level(&self, translation: &T, va: VirtualAddress) -> Option { + pub(crate) fn mapping_level(&self, translation: &T, va: VirtualAddress) -> Option { let entry = self.get_entry(va); if let Some(subtable) = entry.subtable(translation, self.level) { subtable.mapping_level(translation, va) @@ -851,11 +857,11 @@ impl PageTableWithLevel { /// A single level of a page table. #[repr(C, align(4096))] -pub struct PageTable { - entries: [Descriptor; 1 << BITS_PER_LEVEL], +pub struct PageTable { + entries: [Descriptor; 1 << BITS_PER_LEVEL], } -impl PageTable { +impl PageTable { /// An empty (i.e. zeroed) page table. This may be useful for initialising statics. pub const EMPTY: Self = Self { entries: [Descriptor::EMPTY; 1 << BITS_PER_LEVEL], @@ -874,11 +880,11 @@ impl PageTable { /// Returns `Ok(())` on success, or `Err(())` if the size of the byte slice is not equal to the /// size of a page table. pub fn write_to(&self, page: &mut [u8]) -> Result<(), ()> { - if page.len() != self.entries.len() * size_of::() { + if page.len() != self.entries.len() * size_of::>() { return Err(()); } for (chunk, desc) in page - .chunks_exact_mut(size_of::()) + .chunks_exact_mut(size_of::>()) .zip(self.entries.iter()) { chunk.copy_from_slice(&desc.bits().to_le_bytes()); @@ -887,7 +893,7 @@ impl PageTable { } } -impl Default for PageTable { +impl Default for PageTable { fn default() -> Self { Self::EMPTY } @@ -947,7 +953,6 @@ mod tests { use crate::target::TargetAllocator; #[cfg(feature = "alloc")] use alloc::{format, string::ToString, vec, vec::Vec}; - use core::sync::atomic::AtomicUsize; #[cfg(feature = "alloc")] #[test] @@ -1011,60 +1016,69 @@ mod tests { #[test] fn invalid_descriptor() { - let desc = Descriptor(AtomicUsize::new(0usize)); + let desc = Descriptor::::new(0usize); assert!(!desc.is_valid()); - assert!(!desc.flags().contains(Attributes::VALID)); + assert!(!desc.flags().contains(Stage1Attributes::VALID)); } #[test] fn set_descriptor() { const PHYSICAL_ADDRESS: usize = 0x12340000; - let mut desc = Descriptor(AtomicUsize::new(0usize)); + let mut desc = Descriptor::::new(0usize); assert!(!desc.is_valid()); desc.set( PhysicalAddress(PHYSICAL_ADDRESS), - Attributes::TABLE_OR_PAGE | Attributes::USER | Attributes::SWFLAG_1 | Attributes::VALID, + Stage1Attributes::TABLE_OR_PAGE + | Stage1Attributes::USER + | Stage1Attributes::SWFLAG_1 + | Stage1Attributes::VALID, ); assert!(desc.is_valid()); assert_eq!( desc.flags(), - Attributes::TABLE_OR_PAGE | Attributes::USER | Attributes::SWFLAG_1 | Attributes::VALID + Stage1Attributes::TABLE_OR_PAGE + | Stage1Attributes::USER + | Stage1Attributes::SWFLAG_1 + | Stage1Attributes::VALID ); assert_eq!(desc.output_address(), PhysicalAddress(PHYSICAL_ADDRESS)); } #[test] fn modify_descriptor_flags() { - let mut desc = Descriptor(AtomicUsize::new(0usize)); + let mut desc = Descriptor::::new(0usize); assert!(!desc.is_valid()); desc.set( PhysicalAddress(0x12340000), - Attributes::TABLE_OR_PAGE | Attributes::USER | Attributes::SWFLAG_1, + Stage1Attributes::TABLE_OR_PAGE | Stage1Attributes::USER | Stage1Attributes::SWFLAG_1, ); UpdatableDescriptor::new(&mut desc, 3, true) .modify_flags( - Attributes::DBM | Attributes::SWFLAG_3, - Attributes::VALID | Attributes::SWFLAG_1, + Stage1Attributes::DBM | Stage1Attributes::SWFLAG_3, + Stage1Attributes::VALID | Stage1Attributes::SWFLAG_1, ) .unwrap(); assert!(!desc.is_valid()); assert_eq!( desc.flags(), - Attributes::TABLE_OR_PAGE | Attributes::USER | Attributes::SWFLAG_3 | Attributes::DBM + Stage1Attributes::TABLE_OR_PAGE + | Stage1Attributes::USER + | Stage1Attributes::SWFLAG_3 + | Stage1Attributes::DBM ); } #[test] #[should_panic] fn modify_descriptor_table_or_page_flag() { - let mut desc = Descriptor(AtomicUsize::new(0usize)); + let mut desc = Descriptor::::new(0usize); assert!(!desc.is_valid()); desc.set( PhysicalAddress(0x12340000), - Attributes::TABLE_OR_PAGE | Attributes::USER | Attributes::SWFLAG_1, + Stage1Attributes::TABLE_OR_PAGE | Stage1Attributes::USER | Stage1Attributes::SWFLAG_1, ); UpdatableDescriptor::new(&mut desc, 3, false) - .modify_flags(Attributes::VALID, Attributes::TABLE_OR_PAGE) + .modify_flags(Stage1Attributes::VALID, Stage1Attributes::TABLE_OR_PAGE) .unwrap(); } @@ -1086,27 +1100,37 @@ mod tests { #[test] #[should_panic] fn no_el2_ttbr1() { - RootTable::::new(IdTranslation, 1, TranslationRegime::El2, VaRange::Upper); + RootTable::::new( + IdTranslation::new(), + 1, + TranslationRegime::El2, + VaRange::Upper, + ); } #[cfg(feature = "alloc")] #[test] #[should_panic] fn no_el3_ttbr1() { - RootTable::::new(IdTranslation, 1, TranslationRegime::El3, VaRange::Upper); + RootTable::::new( + IdTranslation::new(), + 1, + TranslationRegime::El3, + VaRange::Upper, + ); } #[test] fn table_or_page() { // Invalid. - assert!(!Descriptor(AtomicUsize::new(0b00)).is_table_or_page()); - assert!(!Descriptor(AtomicUsize::new(0b10)).is_table_or_page()); + assert!(!Descriptor::::new(0b00).is_table_or_page()); + assert!(!Descriptor::::new(0b10).is_table_or_page()); // Block mapping. - assert!(!Descriptor(AtomicUsize::new(0b01)).is_table_or_page()); + assert!(!Descriptor::::new(0b01).is_table_or_page()); // Table or page. - assert!(Descriptor(AtomicUsize::new(0b11)).is_table_or_page()); + assert!(Descriptor::::new(0b11).is_table_or_page()); } #[test] @@ -1115,14 +1139,14 @@ mod tests { const UNKNOWN: usize = 1 << 50 | 1 << 52; // Invalid. - assert!(!Descriptor(AtomicUsize::new(UNKNOWN | 0b00)).is_table_or_page()); - assert!(!Descriptor(AtomicUsize::new(UNKNOWN | 0b10)).is_table_or_page()); + assert!(!Descriptor::::new(UNKNOWN | 0b00).is_table_or_page()); + assert!(!Descriptor::::new(UNKNOWN | 0b10).is_table_or_page()); // Block mapping. - assert!(!Descriptor(AtomicUsize::new(UNKNOWN | 0b01)).is_table_or_page()); + assert!(!Descriptor::::new(UNKNOWN | 0b01).is_table_or_page()); // Table or page. - assert!(Descriptor(AtomicUsize::new(UNKNOWN | 0b11)).is_table_or_page()); + assert!(Descriptor::::new(UNKNOWN | 0b11).is_table_or_page()); } #[cfg(feature = "alloc")] @@ -1155,7 +1179,7 @@ mod tests { .map_range( &MemoryRegion::new(PAGE_SIZE * 3, PAGE_SIZE * 6), PhysicalAddress(PAGE_SIZE * 3), - Attributes::VALID | Attributes::NON_GLOBAL, + Stage1Attributes::VALID | Stage1Attributes::NON_GLOBAL, Constraints::empty(), ) .unwrap(); @@ -1163,7 +1187,7 @@ mod tests { .map_range( &MemoryRegion::new(PAGE_SIZE * 6, PAGE_SIZE * 7), PhysicalAddress(PAGE_SIZE * 6), - Attributes::VALID | Attributes::READ_ONLY, + Stage1Attributes::VALID | Stage1Attributes::READ_ONLY, Constraints::empty(), ) .unwrap(); @@ -1171,20 +1195,19 @@ mod tests { .map_range( &MemoryRegion::new(PAGE_SIZE * 8, PAGE_SIZE * 9), PhysicalAddress(PAGE_SIZE * 8), - Attributes::VALID | Attributes::READ_ONLY, + Stage1Attributes::VALID | Stage1Attributes::READ_ONLY, Constraints::empty(), ) .unwrap(); assert_eq!( format!("{table:?}"), "RootTable { pa: 0x0000000000000000, translation_regime: El1And0, va_range: Lower, level: 1, table: -0 : 0x00000000001003 (0x0000000000001000, Attributes(VALID | TABLE_OR_PAGE)) - 0 : 0x00000000002003 (0x0000000000002000, Attributes(VALID | TABLE_OR_PAGE)) - 0 -2 : 0 - 3 -5 : 0x00000000003803 (0x0000000000003000, Attributes(VALID | TABLE_OR_PAGE | NON_GLOBAL)) - 6 : 0x00000000006083 (0x0000000000006000, Attributes(VALID | TABLE_OR_PAGE | READ_ONLY)) +0 : 0x00000000001003 (0x0000000000001000, Stage1Attributes(VALID | TABLE_OR_PAGE)) + 0 : 0x00000000002003 (0x0000000000002000, Stage1Attributes(VALID | TABLE_OR_PAGE)) + 0 -2 : 0\n 3 -5 : 0x00000000003803 (0x0000000000003000, Stage1Attributes(VALID | TABLE_OR_PAGE | NON_GLOBAL)) + 6 : 0x00000000006083 (0x0000000000006000, Stage1Attributes(VALID | TABLE_OR_PAGE | READ_ONLY)) 7 : 0 - 8 : 0x00000000008083 (0x0000000000008000, Attributes(VALID | TABLE_OR_PAGE | READ_ONLY)) + 8 : 0x00000000008083 (0x0000000000008000, Stage1Attributes(VALID | TABLE_OR_PAGE | READ_ONLY)) 9 -511: 0 1 -511: 0 1 -511: 0 @@ -1206,7 +1229,7 @@ mod tests { .map_range( &MemoryRegion::new(BLOCK_SIZE * 3, BLOCK_SIZE * 6), PhysicalAddress(BLOCK_SIZE * 3), - Attributes::VALID | Attributes::NON_GLOBAL, + Stage1Attributes::VALID | Stage1Attributes::NON_GLOBAL, Constraints::empty(), ) .unwrap(); @@ -1214,7 +1237,7 @@ mod tests { .map_range( &MemoryRegion::new(BLOCK_SIZE * 6, BLOCK_SIZE * 7), PhysicalAddress(BLOCK_SIZE * 6), - Attributes::VALID | Attributes::READ_ONLY, + Stage1Attributes::VALID | Stage1Attributes::READ_ONLY, Constraints::empty(), ) .unwrap(); @@ -1222,19 +1245,19 @@ mod tests { .map_range( &MemoryRegion::new(BLOCK_SIZE * 8, BLOCK_SIZE * 9), PhysicalAddress(BLOCK_SIZE * 8), - Attributes::VALID | Attributes::READ_ONLY, + Stage1Attributes::VALID | Stage1Attributes::READ_ONLY, Constraints::empty(), ) .unwrap(); assert_eq!( format!("{table:?}"), "RootTable { pa: 0x0000000000000000, translation_regime: El1And0, va_range: Lower, level: 1, table: -0 : 0x00000000001003 (0x0000000000001000, Attributes(VALID | TABLE_OR_PAGE)) +0 : 0x00000000001003 (0x0000000000001000, Stage1Attributes(VALID | TABLE_OR_PAGE)) 0 -2 : 0 - 3 -5 : 0x00000000600801 (0x0000000000600000, Attributes(VALID | NON_GLOBAL)) - 6 : 0x00000000c00081 (0x0000000000c00000, Attributes(VALID | READ_ONLY)) + 3 -5 : 0x00000000600801 (0x0000000000600000, Stage1Attributes(VALID | NON_GLOBAL)) + 6 : 0x00000000c00081 (0x0000000000c00000, Stage1Attributes(VALID | READ_ONLY)) 7 : 0 - 8 : 0x00000001000081 (0x0000000001000000, Attributes(VALID | READ_ONLY)) + 8 : 0x00000001000081 (0x0000000001000000, Stage1Attributes(VALID | READ_ONLY)) 9 -511: 0 1 -511: 0 }" diff --git a/src/target.rs b/src/target.rs index fdf87a7..ab130dd 100644 --- a/src/target.rs +++ b/src/target.rs @@ -18,7 +18,7 @@ use core::{mem::size_of, ptr::NonNull}; /// /// ``` /// use aarch64_paging::{ -/// descriptor::{Attributes, PhysicalAddress}, +/// descriptor::{PhysicalAddress, Stage1Attributes}, /// paging::{Constraints, MemoryRegion, RootTable, TranslationRegime, VaRange}, /// target::TargetAllocator, /// }; @@ -34,10 +34,10 @@ use core::{mem::size_of, ptr::NonNull}; /// map.map_range( /// &MemoryRegion::new(0x0, 0x1000), /// PhysicalAddress(0x4_2000), -/// Attributes::VALID -/// | Attributes::ATTRIBUTE_INDEX_0 -/// | Attributes::INNER_SHAREABLE -/// | Attributes::UXN, +/// Stage1Attributes::VALID +/// | Stage1Attributes::ATTRIBUTE_INDEX_0 +/// | Stage1Attributes::INNER_SHAREABLE +/// | Stage1Attributes::UXN, /// Constraints::empty(), /// ) /// .unwrap(); @@ -137,7 +137,7 @@ impl Translation for TargetAllocator { #[cfg(test)] mod tests { use super::*; - use crate::descriptor::Attributes; + use crate::descriptor::Stage1Attributes; use crate::paging::{Constraints, MemoryRegion, RootTable, TranslationRegime, VaRange}; const ROOT_LEVEL: usize = 1; @@ -153,10 +153,10 @@ mod tests { map.map_range( &MemoryRegion::new(0x0, 0x1000), PhysicalAddress(0x4_2000), - Attributes::VALID - | Attributes::ATTRIBUTE_INDEX_0 - | Attributes::INNER_SHAREABLE - | Attributes::UXN, + Stage1Attributes::VALID + | Stage1Attributes::ATTRIBUTE_INDEX_0 + | Stage1Attributes::INNER_SHAREABLE + | Stage1Attributes::UXN, Constraints::empty(), ) .unwrap();