diff --git a/litebox_common_linux/src/lib.rs b/litebox_common_linux/src/lib.rs index 379b27e1d..78498bc08 100644 --- a/litebox_common_linux/src/lib.rs +++ b/litebox_common_linux/src/lib.rs @@ -21,6 +21,7 @@ use crate::signal::SigSet; pub mod errno; pub mod loader; pub mod mm; +pub mod physical_pointers; pub mod signal; pub mod vmap; diff --git a/litebox_common_linux/src/physical_pointers.rs b/litebox_common_linux/src/physical_pointers.rs new file mode 100644 index 000000000..c1afc25db --- /dev/null +++ b/litebox_common_linux/src/physical_pointers.rs @@ -0,0 +1,500 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Physical Pointer Abstraction with On-demand Mapping +//! +//! This module adds supports for accessing physical addresses (e.g., VTL0 +//! or normal-world physical memory) from LiteBox with on-demand mapping. +//! In the context of LVBS and OP-TEE, accessing physical memory is +//! necessary because VTL0 and VTL1 as well as normal world and secure +//! world exchange data using physical addresses. +//! +//! The safe read/write APIs in this module follow the same safety model as +//! safe wrappers around DMA buffers or shared physical memory. The +//! physical memory is external to Rust's ordinary ownership model and may +//! be changed by hardware or another privilege level. These APIs remain +//! safe because they do not create Rust references into that external +//! memory; they only perform bounded copies between a temporary mapping +//! and memory owned by LiteBox. +//! +//! The safe APIs should validate whether a given physical address is okay +//! to access. For example, accessing LiteBox's own memory through this +//! physical pointer abstraction is prohibited to avoid confused-deputy +//! attacks and to ensure Rust memory safety. In the case of LVBS, LiteBox +//! obtains the physical memory information from VTL0 like the total +//! physical memory range assigned to VTL1/LiteBox. Thus, this module can +//! confirm a given physical address does not belong to VTL1's physical +//! memory. + +use crate::vmap::{ + GlobalVmapManager, PhysPageAddr, PhysPageMapInfo, PhysPageMapPermissions, PhysPointerError, + VmapManager, +}; +use core::marker::PhantomData; +use zerocopy::{FromBytes, IntoBytes}; + +/// The concrete [`PhysPageMapInfo`] produced by the `VmapManager` behind a [`GlobalVmapManager`]. +type MapInfoOf = + <>::Manager as VmapManager>::MapInfo; + +/// Allocate a zeroed `Box` on the heap. +/// +/// # Panics +/// +/// Panics if `T` is a zero-sized type, since `alloc_zeroed` with a zero-sized +/// layout is undefined behavior. +fn box_new_zeroed() -> alloc::boxed::Box { + assert!( + core::mem::size_of::() > 0, + "box_new_zeroed does not support zero-sized types" + ); + let layout = core::alloc::Layout::new::(); + // Safety: layout has a non-zero size and correct alignment for T. + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }.cast::(); + if ptr.is_null() { + alloc::alloc::handle_alloc_error(layout); + } + // Safety: ptr is a valid, zeroed, properly aligned heap allocation for T. + // T: FromBytes guarantees all-zero is a valid bit pattern. + unsafe { alloc::boxed::Box::from_raw(ptr) } +} + +#[inline] +fn align_down(address: usize, align: usize) -> usize { + address & !(align - 1) +} + +/// Represent a physical pointer to an object with on-demand mapping. +/// +/// Safe methods on this type copy to or from a temporary mapping. They never expose +/// references or slices into the mapped physical memory. +/// +/// Read methods require `T: FromBytes` because external memory may contain any bit pattern. +/// Write methods require `T: IntoBytes` because values are written by copying their byte +/// representation. +/// +/// - `pages`: An array of page-aligned physical addresses. We expect physical addresses in this array are +/// virtually contiguous. +/// - `offset`: The offset within `pages[0]` where the object starts. It should be smaller than `ALIGN`. +/// - `count`: The number of objects of type `T` that can be accessed from this pointer. +/// - `T`: The type of the object being pointed to. `pages` with respect to `offset` should cover enough +/// memory for an object of type `T`. +#[repr(C)] +pub struct PhysMutPtr> { + pages: alloc::boxed::Box<[PhysPageAddr]>, + offset: usize, + count: usize, + _type: PhantomData, + _vmap: PhantomData, +} + +impl PhysMutPtr +where + V: GlobalVmapManager, +{ + /// Create a new `PhysMutPtr` from the given physical page array and offset. + /// + /// All addresses in `pages` should be valid and aligned to `ALIGN`, and `offset` should be + /// smaller than `ALIGN`. Also, `pages` should contain enough pages to cover at least one + /// object of type `T` starting from `offset`. If these conditions are not met, this function + /// returns `Err(PhysPointerError)`. + pub fn new(pages: &[PhysPageAddr], offset: usize) -> Result { + if core::mem::size_of::() == 0 { + return Err(PhysPointerError::UnsupportedZeroSizedType); + } + if offset >= ALIGN { + return Err(PhysPointerError::InvalidBaseOffset(offset, ALIGN)); + } + let size = if pages.is_empty() { + 0 + } else { + pages + .len() + .checked_mul(ALIGN) + .ok_or(PhysPointerError::Overflow)? + - offset + }; + if size < core::mem::size_of::() { + return Err(PhysPointerError::InsufficientPhysicalPages( + size, + core::mem::size_of::(), + )); + } + V::manager().validate_unowned(pages)?; + Ok(Self { + pages: pages.into(), + offset, + count: size / core::mem::size_of::(), + _type: PhantomData, + _vmap: PhantomData, + }) + } + + /// Create a new `PhysMutPtr` from the given contiguous physical address and length. + /// + /// This is a shortcut for + /// `PhysMutPtr::new([align_down(pa), align_down(pa) + ALIGN, ..., align_up(pa + bytes) - ALIGN], pa % ALIGN)`. + /// This function assumes that `pa`, ..., `pa+bytes` are both physically and virtually contiguous. If not, + /// later accesses through `PhysMutPtr` may read/write data in a wrong order. + pub fn with_contiguous_pages(pa: usize, bytes: usize) -> Result { + if bytes < core::mem::size_of::() { + return Err(PhysPointerError::InsufficientPhysicalPages( + bytes, + core::mem::size_of::(), + )); + } + let start_page = align_down(pa, ALIGN); + let end_page = pa + .checked_add(bytes) + .and_then(|end| end.checked_next_multiple_of(ALIGN)) + .ok_or(PhysPointerError::Overflow)?; + let span = end_page + .checked_sub(start_page) + .ok_or(PhysPointerError::Overflow)?; + let mut pages = alloc::vec::Vec::with_capacity(span / ALIGN); + let mut current_page = start_page; + while current_page < end_page { + pages.push( + PhysPageAddr::::new(current_page) + .ok_or(PhysPointerError::InvalidPhysicalAddress(current_page))?, + ); + current_page = current_page + .checked_add(ALIGN) + .ok_or(PhysPointerError::Overflow)?; + } + Self::new(&pages, pa - start_page) + } + + /// Create a new `PhysMutPtr` from the given physical address for a single object. + /// + /// This is a shortcut for `PhysMutPtr::with_contiguous_pages(pa, size_of::())`. + /// + /// Note: This module doesn't provide `as_usize` because LiteBox should not dereference physical addresses directly. + pub fn with_usize(pa: usize) -> Result { + Self::with_contiguous_pages(pa, core::mem::size_of::()) + } + + /// Read the value at the given offset from the physical pointer. + /// + /// Returns an owned copy of the value read from physical memory. + pub fn read_at_offset(&self, count: usize) -> Result, PhysPointerError> + where + T: FromBytes, + { + if count >= self.count { + return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); + } + let guard = self.map_and_get_ptr_guard( + count, + core::mem::size_of::(), + PhysPageMapPermissions::READ, + )?; + let mut boxed = box_new_zeroed::(); + // SAFETY: `boxed` is a freshly allocated `T` and is thus valid for writes + // of `size_of::()` bytes, which is the guard's mapped size. + unsafe { guard.copy_out(core::ptr::from_mut::(boxed.as_mut()).cast::())? }; + Ok(boxed) + } + + /// Read a slice of values at the given offset from the physical pointer. + /// + /// Copies values from physical memory into the caller-provided slice. + pub fn read_slice_at_offset( + &self, + count: usize, + values: &mut [T], + ) -> Result<(), PhysPointerError> + where + T: FromBytes, + { + if values.is_empty() { + return Ok(()); + } + if count + .checked_add(values.len()) + .is_none_or(|end| end > self.count) + { + return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); + } + let guard = self.map_and_get_ptr_guard( + count, + core::mem::size_of_val(values), + PhysPageMapPermissions::READ, + )?; + // SAFETY: `values` is valid for writes of `size_of_val(values)` bytes, which is + // the guard's mapped size. + unsafe { guard.copy_out(values.as_mut_ptr().cast::())? }; + Ok(()) + } + + /// Write the value at the given offset to the physical pointer. + pub fn write_at_offset(&self, count: usize, value: T) -> Result<(), PhysPointerError> + where + T: IntoBytes, + { + if count >= self.count { + return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); + } + let guard = self.map_and_get_ptr_guard( + count, + core::mem::size_of::(), + PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE, + )?; + // SAFETY: `value` is valid for reads of `size_of::()` bytes, which is the + // guard's mapped size. + unsafe { guard.copy_in(core::ptr::from_ref(&value).cast::())? }; + Ok(()) + } + + /// Write a slice of values at the given offset to the physical pointer. + pub fn write_slice_at_offset(&self, count: usize, values: &[T]) -> Result<(), PhysPointerError> + where + T: IntoBytes, + { + if values.is_empty() { + return Ok(()); + } + if count + .checked_add(values.len()) + .is_none_or(|end| end > self.count) + { + return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); + } + let guard = self.map_and_get_ptr_guard( + count, + core::mem::size_of_val(values), + PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE, + )?; + // SAFETY: `values` is valid for reads of `size_of_val(values)` bytes, which is + // the guard's mapped size. + unsafe { guard.copy_in(values.as_ptr().cast::())? }; + Ok(()) + } + + /// This function maps physical pages for the requested data element at a given + /// index and returns a guard that unmaps on drop. + /// + /// It bridges element-level access (used by `read_at_offset`, `write_at_offset`, etc.) + /// with page-level mapping. It determines which physical pages contain the requested + /// element, maps them into virtual memory, and returns a pointer adjusted for + /// the element's position. + /// + /// - `count`: Element index (0-based) within this physical pointer's range. + /// - `size`: Total byte size to map (must cover the data being accessed). + /// - `perms`: Required page permissions (read, write). + /// + /// The returned guard is tied to `self`'s lifetime and releases the mapping when it + /// goes out of scope. + fn map_and_get_ptr_guard( + &self, + count: usize, + size: usize, + perms: PhysPageMapPermissions, + ) -> Result, PhysPointerError> { + let skip = self + .offset + .checked_add( + count + .checked_mul(core::mem::size_of::()) + .ok_or(PhysPointerError::Overflow)?, + ) + .ok_or(PhysPointerError::Overflow)?; + let start = skip / ALIGN; + let end = skip + .checked_add(size) + .ok_or(PhysPointerError::Overflow)? + .div_ceil(ALIGN); + let map_info = self.map_range(start, end, perms)?; + let ptr = map_info.base().wrapping_add(skip % ALIGN).cast::(); + Ok(MappedGuard { + map_info: Some(map_info), + ptr, + size, + _owner: PhantomData, + }) + } + + /// Map the physical pages from `start` to `end` indexes. + fn map_range( + &self, + start: usize, + end: usize, + perms: PhysPageMapPermissions, + ) -> Result, PhysPointerError> { + if start >= end || end > self.pages.len() { + return Err(PhysPointerError::IndexOutOfBounds(end, self.pages.len())); + } + let accept_perms = PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE; + if perms.bits() & !accept_perms.bits() != 0 { + return Err(PhysPointerError::UnsupportedPermissions(perms.bits())); + } + let sub_pages = &self.pages[start..end]; + // SAFETY: This caller never creates Rust references from the returned mapped pointer. + // The mapping is wrapped in `MapInfo`, then consumed by `MappedGuard`, which accesses it + // only through fault-tolerant raw copies. That avoids relying on Rust aliasing or validity + // guarantees for the external physical memory. + unsafe { V::manager().vmap(sub_pages, perms) } + } +} + +/// RAII guard that unmaps physical pages when dropped. +/// +/// Created by `map_and_get_ptr_guard`. Its lifetime is tied to the parent +/// `PhysMutPtr`, and it owns the map info for the duration of the temporary mapping. +/// +/// # Invariant +/// +/// `ptr` points into the live mapping owned by `map_info`, and the `size` bytes starting +/// at `ptr` lie within that mapping. The mapping refers to foreign (non-Rust) physical +/// memory that another core may unmap concurrently, so `ptr` must only ever be accessed +/// through [`Self::copy_in`]/[`Self::copy_out`], which perform fault-tolerant copies. +struct MappedGuard<'a, T: Clone, const ALIGN: usize, V: GlobalVmapManager> { + map_info: Option>, + ptr: *mut T, + size: usize, + _owner: PhantomData<&'a PhysMutPtr>, +} + +impl> MappedGuard<'_, T, ALIGN, V> { + /// Copy the `self.size` mapped bytes out into `dst`. + /// + /// This is the only path through which the raw mapped pointer is dereferenced. + /// + /// # Safety + /// + /// `dst` must be valid for writes of `self.size` bytes. + unsafe fn copy_out(&self, dst: *mut u8) -> Result<(), PhysPointerError> { + // Fallible: another core may unmap this page concurrently. + let result = unsafe { + litebox::mm::exception_table::memcpy_fallible(dst, self.ptr.cast::(), self.size) + }; + debug_assert!(result.is_ok(), "fault reading from mapped physical page"); + result.map_err(|_| PhysPointerError::CopyFailed) + } + + /// Copy `self.size` bytes from `src` into the mapped memory. + /// + /// This is the only path through which the raw mapped pointer is dereferenced. + /// + /// # Safety + /// + /// `src` must be valid for reads of `self.size` bytes. + unsafe fn copy_in(&self, src: *const u8) -> Result<(), PhysPointerError> { + // Fallible: another core may unmap this page concurrently. + let result = unsafe { + litebox::mm::exception_table::memcpy_fallible(self.ptr.cast::(), src, self.size) + }; + debug_assert!(result.is_ok(), "fault writing to mapped physical page"); + result.map_err(|_| PhysPointerError::CopyFailed) + } +} + +impl> Drop + for MappedGuard<'_, T, ALIGN, V> +{ + fn drop(&mut self) { + // SAFETY: The platform is expected to handle unmapping safely. Drop cannot + // report errors. If unmapping fails, drop the returned private map_info; + // platform-specific resources that cannot be reclaimed are handled by the + // platform `vunmap` implementation. + if let Some(map_info) = self.map_info.take() { + let _ = unsafe { V::manager().vunmap(map_info) }; + } + } +} + +impl> core::fmt::Debug + for PhysMutPtr +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PhysMutPtr") + .field("pages[0]", &self.pages.first().map_or(0, |p| p.as_usize())) + .field("offset", &self.offset) + .finish_non_exhaustive() + } +} + +/// Represent a physical pointer to a read-only object. This wraps around [`PhysMutPtr`] and +/// exposes only copy-out access. +#[repr(C)] +pub struct PhysConstPtr> { + inner: PhysMutPtr, +} + +impl PhysConstPtr +where + V: GlobalVmapManager, +{ + /// Create a new `PhysConstPtr` from the given physical page array and offset. + /// + /// All addresses in `pages` should be valid and aligned to `ALIGN`, and `offset` should be smaller + /// than `ALIGN`. Also, `pages` should contain enough pages to cover at least one object of + /// type `T` starting from `offset`. If these conditions are not met, this function returns + /// `Err(PhysPointerError)`. + pub fn new(pages: &[PhysPageAddr], offset: usize) -> Result { + Ok(Self { + inner: PhysMutPtr::new(pages, offset)?, + }) + } + + /// Create a new `PhysConstPtr` from the given contiguous physical address and length. + /// + /// This is a shortcut for + /// `PhysConstPtr::new([align_down(pa), align_down(pa) + ALIGN, ..., align_up(pa + bytes) - ALIGN], pa % ALIGN)`. + /// This function assumes that `pa`, ..., `pa+bytes` are both physically and virtually contiguous. If not, + /// later accesses through `PhysConstPtr` may read data in a wrong order. + pub fn with_contiguous_pages(pa: usize, bytes: usize) -> Result { + Ok(Self { + inner: PhysMutPtr::with_contiguous_pages(pa, bytes)?, + }) + } + + /// Create a new `PhysConstPtr` from the given physical address for a single object. + /// + /// This is a shortcut for `PhysConstPtr::with_contiguous_pages(pa, size_of::())`. + /// + /// Note: This module doesn't provide `as_usize` because LiteBox should not dereference physical addresses directly. + pub fn with_usize(pa: usize) -> Result { + Ok(Self { + inner: PhysMutPtr::with_usize(pa)?, + }) + } + + /// Read the value at the given offset from the physical pointer. + /// + /// Returns an owned copy of the value read from physical memory. + pub fn read_at_offset(&self, count: usize) -> Result, PhysPointerError> + where + T: FromBytes, + { + self.inner.read_at_offset(count) + } + + /// Read a slice of values at the given offset from the physical pointer. + /// + /// Copies values from physical memory into the caller-provided slice. + pub fn read_slice_at_offset( + &self, + count: usize, + values: &mut [T], + ) -> Result<(), PhysPointerError> + where + T: FromBytes, + { + self.inner.read_slice_at_offset(count, values) + } +} + +impl> core::fmt::Debug + for PhysConstPtr +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PhysConstPtr") + .field( + "pages[0]", + &self.inner.pages.first().map_or(0, |p| p.as_usize()), + ) + .field("offset", &self.inner.offset) + .finish_non_exhaustive() + } +} diff --git a/litebox_common_linux/src/vmap.rs b/litebox_common_linux/src/vmap.rs index e747ca5a3..afaa41897 100644 --- a/litebox_common_linux/src/vmap.rs +++ b/litebox_common_linux/src/vmap.rs @@ -8,41 +8,59 @@ use thiserror::Error; /// /// `ALIGN`: The page frame size. /// -/// This provider exists to service `litebox_shim_optee::ptr::PhysMutPtr` and -/// `litebox_shim_optee::ptr::PhysConstPtr`. It can benefit other modules which need +/// This provider exists to service [`crate::physical_pointers::PhysMutPtr`] and +/// [`crate::physical_pointers::PhysConstPtr`]. It can benefit other modules which need /// Linux kernel's `vmap()` and `vunmap()` functionalities (e.g., HVCI/HEKI, drivers). -pub trait VmapManager { +/// +/// # Safety +/// +/// Implementors must uphold each unsafe method's contract and keep [`Self::MapInfo`] tied to the +/// mapping it identifies. +pub unsafe trait VmapManager { + /// Implementors use this to carry the virtual mapping and any platform-specific bookkeeping + /// needed for unmapping. + type MapInfo: PhysPageMapInfo; + /// Map the given `PhysPageAddrArray` into virtually contiguous addresses with the given - /// [`PhysPageMapPermissions`] while returning [`PhysPageMapInfo`]. + /// [`PhysPageMapPermissions`] while returning [`Self::MapInfo`]. /// /// This function is analogous to Linux kernel's `vmap()`. /// /// # Safety /// - /// The caller should ensure that `pages` are not in active use by other entities - /// (especially, there should be no read/write or write/write conflicts). - /// Unfortunately, LiteBox itself cannot fully guarantee this and it needs some helps - /// from the caller, hypervisor, or hardware. - /// Multiple LiteBox threads might concurrently call this function with overlapping - /// physical pages, so the implementation should safely handle such cases. + /// The returned pointer is a raw address; creating or holding it does not access memory or + /// create a Rust reference. Any later use of that pointer must satisfy the platform's access + /// requirements for the mapped physical pages. Even when access is logically exclusive, callers + /// must treat the mapped memory like DMA/shared physical memory rather than ordinary Rust-owned + /// RAM. unsafe fn vmap( &self, _pages: &PhysPageAddrArray, _perms: PhysPageMapPermissions, - ) -> Result, PhysPointerError> { + ) -> Result { Err(PhysPointerError::UnsupportedOperation) } - /// Unmap the previously mapped virtually contiguous addresses ([`PhysPageMapInfo`]). + /// Unmap the previously mapped virtually contiguous addresses ([`Self::MapInfo`]). /// /// This function is analogous to Linux kernel's `vunmap()`. /// + /// On failure, the unchanged `vmap_info` is returned alongside the error so the caller can + /// retry or otherwise preserve the mapping state. Dropping returned map info is not guaranteed + /// to release platform resources; each implementation owns the retention policy for resources + /// that cannot be safely reclaimed after a failed unmap. + /// /// # Safety /// - /// The caller should ensure that the virtual addresses in `vmap_info` are not in active - /// use by other entities. - unsafe fn vunmap(&self, _vmap_info: PhysPageMapInfo) -> Result<(), PhysPointerError> { - Err(PhysPointerError::UnsupportedOperation) + /// The caller must ensure there are no outstanding raw-pointer uses or Rust references derived + /// from `PhysPageMapInfo::base()`. After a successful call, the virtual mapping is invalid and + /// any platform resources tied to the mapping lifetime have been released or otherwise handled + /// by the implementation. + unsafe fn vunmap( + &self, + vmap_info: Self::MapInfo, + ) -> Result<(), (PhysPointerError, Self::MapInfo)> { + Err((PhysPointerError::UnsupportedOperation, vmap_info)) } /// Validate that the given physical pages are not owned by LiteBox. @@ -79,6 +97,22 @@ pub trait VmapManager { } } +/// A type-level handle to a platform-global [`VmapManager`]. +/// +/// `PhysMutPtr` and `PhysConstPtr` carry their provider as a type parameter +/// (`PhantomData

`), so they cannot hold a live `&VmapManager`. This trait +/// is the minimum surface that lets such a `PhantomData`-only carrier reach +/// the live manager: each platform implements this on a small unit struct +/// (e.g., `Vmap`) and points `manager()` at its global +/// platform singleton. +pub trait GlobalVmapManager: 'static { + /// The concrete `VmapManager` this marker resolves to. + type Manager: VmapManager + 'static; + + /// Return the global manager instance for this platform. + fn manager() -> &'static Self::Manager; +} + /// Data structure representing a physical address with page alignment. /// /// Currently, this is an alias to `crate::mm::linux::NonZeroAddress`. This might change if @@ -90,13 +124,39 @@ pub type PhysPageAddr = litebox::mm::linux::NonZeroAddress = [PhysPageAddr]; -/// Data structure to maintain the mapping information returned by `vmap()`. -#[derive(Clone)] -pub struct PhysPageMapInfo { +/// Mapping information returned by `vmap()`. +/// +/// Implementors use this value to track the virtual mapping and any platform-specific resources +/// tied to it. Callers must pass it back to the same platform's `vunmap()` to explicitly unmap; +/// drop behavior is implementation-specific. +pub trait PhysPageMapInfo { /// Virtual address of the mapped region which is page aligned. - pub base: *mut u8, + fn base(&self) -> *mut u8; /// The size of the mapped region in bytes. - pub size: usize, + fn size(&self) -> usize; +} + +/// A no-op [`PhysPageMapInfo`] for platforms that do not support `vmap()`/`vunmap()`. +#[derive(Debug)] +pub struct NoopPhysPageMapInfo { + base: *mut u8, + size: usize, +} + +impl NoopPhysPageMapInfo { + pub fn new(base: *mut u8, size: usize) -> Self { + Self { base, size } + } +} + +impl PhysPageMapInfo for NoopPhysPageMapInfo { + fn base(&self) -> *mut u8 { + self.base + } + + fn size(&self) -> usize { + self.size + } } bitflags::bitflags! { @@ -156,6 +216,8 @@ pub enum PhysPointerError { "The total size of the given pages ({0} bytes) is insufficient for the requested type ({1} bytes)" )] InsufficientPhysicalPages(usize, usize), + #[error("Zero-sized types are unsupported for physical pointers")] + UnsupportedZeroSizedType, #[error("Index {0} is out of bounds (count: {1})")] IndexOutOfBounds(usize, usize), #[error("Physical address {0:#x} is already mapped")] diff --git a/litebox_common_optee/src/lib.rs b/litebox_common_optee/src/lib.rs index bcff985e1..8523c6061 100644 --- a/litebox_common_optee/src/lib.rs +++ b/litebox_common_optee/src/lib.rs @@ -2058,7 +2058,7 @@ impl From<&OpteeSmcArgsPage> for OpteeSmcArgs { } /// OP-TEE SMC call arguments. -#[derive(Clone, Copy, Default, FromBytes)] +#[derive(Clone, Copy, Default, FromBytes, IntoBytes, Immutable)] pub struct OpteeSmcArgs { args: [usize; Self::NUM_OPTEE_SMC_ARGS], } diff --git a/litebox_platform_linux_userland/src/lib.rs b/litebox_platform_linux_userland/src/lib.rs index dfd5f6cbc..1ffd00e42 100644 --- a/litebox_platform_linux_userland/src/lib.rs +++ b/litebox_platform_linux_userland/src/lib.rs @@ -2373,7 +2373,9 @@ impl litebox::platform::DerivedKeyProvider for LinuxUserland { /// In general, userland platforms do not support `vmap` and `vunmap` (which are kernel functions). /// We might need to emulate these functions' behaviors using virtual addresses for development or /// testing, or use a kernel module to provide this functionality (if needed). -impl VmapManager for LinuxUserland {} +unsafe impl VmapManager for LinuxUserland { + type MapInfo = litebox_common_linux::vmap::NoopPhysPageMapInfo; +} /// Dummy `VmemPageFaultHandler`. /// diff --git a/litebox_platform_lvbs/Cargo.toml b/litebox_platform_lvbs/Cargo.toml index a6ceaee6b..c7b91e243 100644 --- a/litebox_platform_lvbs/Cargo.toml +++ b/litebox_platform_lvbs/Cargo.toml @@ -44,8 +44,7 @@ x86_64 = { version = "0.15.2", default-features = false, features = ["instructio libc = "0.2.177" [features] -default = ["optee_syscall"] -optee_syscall = [] +default = [] linux_syscall = [] devbox = [] diff --git a/litebox_platform_lvbs/src/arch/x86/mm/paging.rs b/litebox_platform_lvbs/src/arch/x86/mm/paging.rs index 82edf6824..d259e9c51 100644 --- a/litebox_platform_lvbs/src/arch/x86/mm/paging.rs +++ b/litebox_platform_lvbs/src/arch/x86/mm/paging.rs @@ -656,7 +656,6 @@ impl X64PageTable<'_, M, ALIGN> { /// # Behavior /// - Any existing mapping is treated as an error /// - On error, all pages mapped by this call are unmapped (atomic) - #[cfg(feature = "optee_syscall")] pub(crate) fn map_non_contiguous_phys_frames( &self, frames: &[PhysFrame], @@ -725,7 +724,6 @@ impl X64PageTable<'_, M, ALIGN> { /// /// Note: The caller must already hold the page table lock (`self.inner`). /// This function accepts the locked `MappedPageTable` directly. - #[cfg(feature = "optee_syscall")] fn rollback_mapped_pages( inner: &mut MappedPageTable<'_, FrameMapping>, pages: x86_64::structures::paging::page::PageRangeInclusive, diff --git a/litebox_platform_lvbs/src/lib.rs b/litebox_platform_lvbs/src/lib.rs index 145f3617c..fecee32ee 100644 --- a/litebox_platform_lvbs/src/lib.rs +++ b/litebox_platform_lvbs/src/lib.rs @@ -25,10 +25,9 @@ use litebox::{ utils::TruncateExt, }; use litebox_common_linux::errno::Errno; -#[cfg(feature = "optee_syscall")] use litebox_common_linux::vmap::{ - PhysPageAddr, PhysPageAddrArray, PhysPageMapInfo, PhysPageMapPermissions, PhysPointerError, - VmapManager, + GlobalVmapManager, PhysPageAddr, PhysPageAddrArray, PhysPageMapInfo, PhysPageMapPermissions, + PhysPointerError, VmapManager, }; use x86_64::{ VirtAddr, @@ -39,7 +38,6 @@ use x86_64::{ }; use zerocopy::{FromBytes, IntoBytes}; -#[cfg(feature = "optee_syscall")] use crate::mm::vmap::vmap_allocator; extern crate alloc; @@ -51,32 +49,29 @@ pub mod mshv; pub mod syscall_entry; -/// Allocate a zeroed `Box` directly on the heap, avoiding stack intermediaries -/// for large types (e.g., 4096-byte `HekiPage`). -/// -/// This is safe because `T: FromBytes` guarantees that all-zero bytes are a valid `T`. -/// -/// # Panics -/// -/// Panics if `T` is a zero-sized type, since `alloc_zeroed` with a zero-sized -/// layout is undefined behavior. -fn box_new_zeroed() -> alloc::boxed::Box { - assert!( - core::mem::size_of::() > 0, - "box_new_zeroed does not support zero-sized types" - ); - let layout = core::alloc::Layout::new::(); - // Safety: layout has a non-zero size and correct alignment for T. - let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }.cast::(); - if ptr.is_null() { - alloc::alloc::handle_alloc_error(layout); - } - // Safety: ptr is a valid, zeroed, properly aligned heap allocation for T. - // T: FromBytes guarantees all-zero is a valid bit pattern. - unsafe { alloc::boxed::Box::from_raw(ptr) } +static CPU_MHZ: AtomicU64 = AtomicU64::new(0); + +/// Mapping info returned by [`LinuxKernel`]'s [`VmapManager::vmap`]. +pub struct LvbsPhysPageMapInfo { + base: *mut u8, + size: usize, } -static CPU_MHZ: AtomicU64 = AtomicU64::new(0); +impl LvbsPhysPageMapInfo { + fn new(base: *mut u8, size: usize) -> Self { + Self { base, size } + } +} + +impl PhysPageMapInfo for LvbsPhysPageMapInfo { + fn base(&self) -> *mut u8 { + self.base + } + + fn size(&self) -> usize { + self.size + } +} /// Special page table ID for the base (kernel-only) page table. /// No real physical frame has address 0, so this is a safe sentinel. @@ -455,6 +450,21 @@ type UserConstPtr = type UserMutPtr = litebox::platform::common_providers::userspace_pointers::UserMutPtr; +#[derive(Clone, Copy, Debug, Default)] +pub struct Vmap; + +impl GlobalVmapManager for Vmap { + type Manager = crate::host::LvbsLinuxKernel; + fn manager() -> &'static Self::Manager { + crate::platform_low() + } +} + +pub type Vtl0PhysConstPtr = + litebox_common_linux::physical_pointers::PhysConstPtr; +pub type Vtl0PhysMutPtr = + litebox_common_linux::physical_pointers::PhysMutPtr; + impl RawPointerProvider for LinuxKernel { type RawConstPointer = UserConstPtr; type RawMutPointer = UserMutPtr; @@ -625,50 +635,14 @@ impl LinuxKernel { self.vtl1_phys_frame_range } - /// This function maps VTL0 physical page frames containing the physical addresses - /// from `phys_start` to `phys_end` to the VTL1 kernel page table. It internally page aligns - /// the input addresses to ensure the mapped memory area covers the entire input addresses - /// at the page level. It returns a page-aligned address (as `mmap` does) and the length of the mapped memory. - /// - /// Note: VTL0 physical memory is external/remote memory that this Rust binary doesn't own, - /// so mapping it doesn't create aliasing issues within the Rust memory model. - fn map_vtl0_phys_range( - &self, - phys_start: x86_64::PhysAddr, - phys_end: x86_64::PhysAddr, - flags: PageTableFlags, - ) -> Result<(*mut u8, usize), MapToError> { - let frame_range = PhysFrame::range( - PhysFrame::containing_address(phys_start), - PhysFrame::containing_address(phys_end.align_up(Size4KiB::SIZE)), - ); - - // ensure the input address range does not overlap with VTL1 memory - if frame_range.start < self.vtl1_phys_frame_range.end - && self.vtl1_phys_frame_range.start < frame_range.end - { - return Err(MapToError::FrameAllocationFailed); - } - - let flags = flags | PageTableFlags::NO_EXECUTE; - - Ok(( - self.page_table_manager - .current_page_table() - .map_phys_frame_range_direct(frame_range, flags, None)?, - usize::try_from(frame_range.len()).unwrap() * PAGE_SIZE, - )) - } - /// This function unmaps VTL0 pages from the page table. /// /// Allocator does not allocate memory frames for VTL0 pages, so frame deallocation is not needed. /// - /// Note: VTL0 physical memory is external memory not owned by LiteBox (similar to MMIO). - /// LiteBox accesses it by creating a temporary non-shared mapping, copying data to/from a - /// LiteBox-owned buffer, and unmapping immediately. No Rust references are created to the - /// mapped VTL0 memory; all accesses use raw pointer operations (read_volatile / - /// copy_nonoverlapping) to avoid violating Rust's aliasing model. + /// Note: VTL0 physical memory is external memory not owned by LiteBox, similar to DMA/shared + /// physical memory. Physical pointer APIs access it by creating a temporary mapping, copying + /// data to/from a LiteBox-owned buffer with fallible raw-pointer copies, and unmapping + /// immediately. These APIs do not create Rust references to the mapped VTL0 memory. fn unmap_vtl0_pages( &self, page_addr: *const u8, @@ -699,174 +673,6 @@ impl LinuxKernel { } } - /// Map a VTL0 physical range and return a guard that unmaps on drop. - fn map_vtl0_guard( - &self, - phys_addr: x86_64::PhysAddr, - size: u64, - flags: PageTableFlags, - ) -> Option> { - let phys_end = phys_addr - .as_u64() - .checked_add(size) - .and_then(|end| x86_64::PhysAddr::try_new(end).ok())?; - let (page_addr, page_aligned_length) = - self.map_vtl0_phys_range(phys_addr, phys_end, flags).ok()?; - let page_offset: usize = (phys_addr - phys_addr.align_down(Size4KiB::SIZE)).trunc(); - Some(Vtl0MappedGuard { - owner: self, - page_addr, - page_aligned_length, - ptr: page_addr.wrapping_add(page_offset), - size: size.trunc(), - }) - } - - /// This function copies data from VTL0 physical memory to the VTL1 kernel through `Box`. - /// Use this function instead of map/unmap functions to avoid potential TOCTTOU. - /// - /// # Safety - /// - /// The caller must ensure that the `phys_addr` is a valid VTL0 physical address - pub unsafe fn copy_from_vtl0_phys( - &self, - phys_addr: x86_64::PhysAddr, - ) -> Option> { - if core::mem::size_of::() == 0 { - return Some(alloc::boxed::Box::new(T::new_zeroed())); - } - - let src_guard = self.map_vtl0_guard( - phys_addr, - core::mem::size_of::() as u64, - PageTableFlags::PRESENT, - )?; - - let mut boxed = box_new_zeroed::(); - // Use memcpy_fallible instead of ptr::copy_nonoverlapping to handle - // the race where another core unmaps this page (via a shared page - // table) between map_vtl0_guard and the copy. The mapping is valid - // at this point, so a fault is not expected in the common case. - // TODO: Once VTL0 page-range locking is in place, this fallible copy - // may become unnecessary since the lock would prevent concurrent - // unmapping. It could still serve as a safety net against callers - // that forget to acquire the lock. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - core::ptr::from_mut::(boxed.as_mut()).cast(), - src_guard.ptr, - src_guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault copying from VTL0 mapped page"); - - result.ok().map(|()| boxed) - } - - /// This function copies data from the VTL1 kernel to VTL0 physical memory. - /// Use this function instead of map/unmap functions to avoid potential TOCTTOU. - /// # Safety - /// - /// The caller must ensure that the `phys_addr` is a valid VTL0 physical address - pub unsafe fn copy_to_vtl0_phys( - &self, - phys_addr: x86_64::PhysAddr, - value: &T, - ) -> bool { - if core::mem::size_of::() == 0 { - return true; - } - - let Some(dst_guard) = self.map_vtl0_guard( - phys_addr, - core::mem::size_of::() as u64, - PageTableFlags::PRESENT | PageTableFlags::WRITABLE, - ) else { - return false; - }; - - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - dst_guard.ptr, - core::ptr::from_ref::(value).cast::(), - dst_guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault copying to VTL0 mapped page"); - result.is_ok() - } - - /// This function copies a slice from the VTL1 kernel to VTL0 physical memory. - /// Use this function instead of map/unmap functions to avoid potential TOCTTOU. - /// - /// # Safety - /// - /// The caller must ensure that the `phys_addr` is a valid VTL0 physical address. - pub unsafe fn copy_slice_to_vtl0_phys( - &self, - phys_addr: x86_64::PhysAddr, - value: &[T], - ) -> bool { - if core::mem::size_of_val(value) == 0 { - return true; - } - - let Some(dst_guard) = self.map_vtl0_guard( - phys_addr, - core::mem::size_of_val(value) as u64, - PageTableFlags::PRESENT | PageTableFlags::WRITABLE, - ) else { - return false; - }; - - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - dst_guard.ptr, - value.as_ptr().cast::(), - dst_guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault copying to VTL0 mapped page"); - result.is_ok() - } - - /// This function copies a slice from VTL0 physical memory to the VTL1 kernel. - /// Use this function instead of map/unmap functions to avoid potential TOCTTOU. - /// - /// # Safety - /// - /// The caller must ensure that the `phys_addr` is a valid VTL0 physical address. - pub unsafe fn copy_slice_from_vtl0_phys( - &self, - phys_addr: x86_64::PhysAddr, - buf: &mut [T], - ) -> bool { - if core::mem::size_of_val(buf) == 0 { - return true; - } - - let Some(src_guard) = self.map_vtl0_guard( - phys_addr, - core::mem::size_of_val(buf) as u64, - PageTableFlags::PRESENT, - ) else { - return false; - }; - - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - buf.as_mut_ptr().cast::(), - src_guard.ptr, - src_guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault copying from VTL0 mapped page"); - result.is_ok() - } - /// Create a new task page table for VTL1 user space and returns its ID. /// /// The kernel address space is duplicated from the base page table, @@ -947,26 +753,6 @@ impl LinuxKernel { } } -/// RAII guard that unmaps VTL0 physical pages when dropped. -struct Vtl0MappedGuard<'a, Host: HostInterface> { - owner: &'a LinuxKernel, - page_addr: *mut u8, - page_aligned_length: usize, - ptr: *mut u8, - size: usize, -} - -impl Drop for Vtl0MappedGuard<'_, Host> { - fn drop(&mut self) { - assert!( - self.owner - .unmap_vtl0_pages(self.page_addr, self.page_aligned_length) - .is_ok(), - "Failed to unmap VTL0 pages" - ); - } -} - impl RawMutexProvider for LinuxKernel { type RawMutex = RawMutex; } @@ -1337,7 +1123,6 @@ impl litebox::platform::SystemInfoProvider for LinuxKernel< } } -#[cfg(feature = "optee_syscall")] /// Checks whether the given physical addresses are contiguous with respect to ALIGN. fn is_contiguous(addrs: &[PhysPageAddr]) -> bool { for window in addrs.windows(2) { @@ -1354,13 +1139,14 @@ fn is_contiguous(addrs: &[PhysPageAddr]) -> bool { true } -#[cfg(feature = "optee_syscall")] -impl VmapManager for LinuxKernel { +unsafe impl VmapManager for LinuxKernel { + type MapInfo = LvbsPhysPageMapInfo; + unsafe fn vmap( &self, pages: &PhysPageAddrArray, perms: PhysPageMapPermissions, - ) -> Result, PhysPointerError> { + ) -> Result { if pages.is_empty() { return Err(PhysPointerError::InvalidPhysicalAddress(0)); } @@ -1369,6 +1155,17 @@ impl VmapManager for LinuxKernel unimplemented!("ALIGN other than 4KiB is not supported yet"); } + // Reject duplicates early as an API-level validation. The page-table implementation also + // rejects duplicate/shared mappings, but this keeps the error local to the input array. + if !is_contiguous(pages) { + let mut seen = hashbrown::HashSet::with_capacity(pages.len()); + for page in pages { + if !seen.insert(page.as_usize()) { + return Err(PhysPointerError::DuplicatePhysicalAddress(page.as_usize())); + } + } + } + // VTL0 memory must never be executable from VTL1 (DEP). let mut flags = PageTableFlags::PRESENT | PageTableFlags::NO_EXECUTE; if perms.contains(PhysPageMapPermissions::WRITE) { @@ -1397,10 +1194,7 @@ impl VmapManager for LinuxKernel .current_page_table() .map_phys_frame_range_direct(frame_range, flags, None) { - Ok(page_addr) => Ok(PhysPageMapInfo { - base: page_addr, - size: pages.len() * ALIGN, - }), + Ok(page_addr) => Ok(LvbsPhysPageMapInfo::new(page_addr, pages.len() * ALIGN)), Err(MapToError::PageAlreadyMapped(_)) => { Err(PhysPointerError::AlreadyMapped(pages[0].as_usize())) } @@ -1412,16 +1206,6 @@ impl VmapManager for LinuxKernel ), } } else { - // Reject duplicate page addresses - { - let mut seen = hashbrown::HashSet::with_capacity(pages.len()); - for page in pages { - if !seen.insert(page.as_usize()) { - return Err(PhysPointerError::DuplicatePhysicalAddress(page.as_usize())); - } - } - } - let frames: alloc::vec::Vec> = pages .iter() .map(|p| PhysFrame::containing_address(x86_64::PhysAddr::new(p.as_usize() as u64))) @@ -1446,10 +1230,7 @@ impl VmapManager for LinuxKernel .current_page_table() .map_non_contiguous_phys_frames(&frames, base_va, flags) { - Ok(page_addr) => Ok(PhysPageMapInfo { - base: page_addr, - size: pages.len() * ALIGN, - }), + Ok(page_addr) => Ok(LvbsPhysPageMapInfo::new(page_addr, pages.len() * ALIGN)), Err(e) => { let _ = vmap_allocator().unregister_allocation(base_va); match e { @@ -1468,24 +1249,37 @@ impl VmapManager for LinuxKernel } } - unsafe fn vunmap(&self, vmap_info: PhysPageMapInfo) -> Result<(), PhysPointerError> { + unsafe fn vunmap( + &self, + vmap_info: Self::MapInfo, + ) -> Result<(), (PhysPointerError, Self::MapInfo)> { if ALIGN != PAGE_SIZE { unimplemented!("ALIGN other than 4KiB is not supported yet"); } - let base_va = x86_64::VirtAddr::new(vmap_info.base as u64); + let base = vmap_info.base(); + let size = vmap_info.size(); + let base_va = x86_64::VirtAddr::new(base as u64); // Unmap the page table entries first. Only release the VA range back // to the allocator when unmapping succeeds; if it fails, stale PTE // entries remain and recycling the VA would cause collisions. - self.unmap_vtl0_pages(vmap_info.base, vmap_info.size) - .map_err(|_| PhysPointerError::Unmapped(vmap_info.base as usize))?; + if self.unmap_vtl0_pages(base, size).is_err() { + return Err((PhysPointerError::Unmapped(base as usize), vmap_info)); + } - if crate::mm::vmap::is_vmap_address(base_va) { - crate::mm::vmap::vmap_allocator() + // PTEs are already cleared at this point, so the mapping is functionally gone + // and a retry would only re-fail against empty page-table entries. If the VA + // allocator's bookkeeping is inconsistent, surface it via `debug_assert!`. The + // VA region is leaked but cannot be safely recycled. + let unregister_ok = !crate::mm::vmap::is_vmap_address(base_va) + || crate::mm::vmap::vmap_allocator() .unregister_allocation(base_va) - .ok_or(PhysPointerError::Unmapped(vmap_info.base as usize))?; - } + .is_some(); + debug_assert!( + unregister_ok, + "vmap allocator unregister failed at {base_va:?}", + ); Ok(()) } @@ -1506,6 +1300,7 @@ impl VmapManager for LinuxKernel Ok(()) } + #[allow(dead_code, reason = "will be used soon")] unsafe fn protect( &self, pages: &PhysPageAddrArray, diff --git a/litebox_platform_lvbs/src/mm/mod.rs b/litebox_platform_lvbs/src/mm/mod.rs index df04d4209..a96d5fd12 100644 --- a/litebox_platform_lvbs/src/mm/mod.rs +++ b/litebox_platform_lvbs/src/mm/mod.rs @@ -6,7 +6,6 @@ use crate::arch::{PhysAddr, VirtAddr}; pub(crate) mod pgtable; -#[cfg(feature = "optee_syscall")] pub(crate) mod vmap; #[cfg(test)] @@ -56,7 +55,6 @@ pub trait MemoryProvider { fn pa_to_va_direct(pa: PhysAddr) -> VirtAddr { let pa = pa.as_u64() & !Self::PRIVATE_PTE_MASK; let va = VirtAddr::new_truncate(pa + Self::GVA_OFFSET.as_u64()); - #[cfg(feature = "optee_syscall")] assert!( va.as_u64() < crate::VMAP_START as u64, "VA {va:#x} is out of range for direct mapping" diff --git a/litebox_platform_lvbs/src/mshv/ringbuffer.rs b/litebox_platform_lvbs/src/mshv/ringbuffer.rs index 92a75da26..751bd19aa 100644 --- a/litebox_platform_lvbs/src/mshv/ringbuffer.rs +++ b/litebox_platform_lvbs/src/mshv/ringbuffer.rs @@ -3,7 +3,11 @@ //! RingBuffer implementation and functions +use crate::Vtl0PhysMutPtr; use core::fmt; +use litebox::mm::linux::PAGE_SIZE; +use litebox::utils::TruncateExt; +use litebox_common_linux::vmap::PhysPageAddr; use spin::{Mutex, Once}; use x86_64::PhysAddr; @@ -11,46 +15,129 @@ pub struct RingBuffer { rb_pa: PhysAddr, write_offset: usize, size: usize, + // True iff `rb_pa` is page-aligned and `size` is a non-zero page multiple, + // i.e. wraparound can be collapsed into a single non-contiguous mapping. + // Pages themselves are derived from `rb_pa + idx * PAGE_SIZE` since the ring + // is physically contiguous. + fast_path_eligible: bool, } impl RingBuffer { pub fn new(phys_addr: PhysAddr, requested_size: usize) -> Self { + let pa: usize = phys_addr.as_u64().trunc(); + let fast_path_eligible = requested_size > 0 + && requested_size.is_multiple_of(PAGE_SIZE) + && pa.is_multiple_of(PAGE_SIZE); RingBuffer { rb_pa: phys_addr, write_offset: 0, size: requested_size, + fast_path_eligible, } } pub fn write(&mut self, buf: &[u8]) { - // If the input buffer is longer than the ring buffer, fill the whole ring buffer with - // the final [ring buffer size] values from the input buffer - if buf.len() >= self.size { - let single_slice = &buf[(buf.len() - self.size)..]; - unsafe { - crate::platform_low().copy_slice_to_vtl0_phys(self.rb_pa, single_slice); - } - self.write_offset = 0; + if self.size == 0 || buf.is_empty() { return; } + self.write_offset = if self.fast_path_eligible { + write_fast(self.rb_pa, self.size, self.write_offset, buf) + } else { + write_slow(self.rb_pa, self.size, self.write_offset, buf) + }; + } +} + +/// Fast path for a page-aligned, page-sized ring buffer. Wraparound becomes a +/// single virtually-contiguous, physically non-contiguous mapping by emitting +/// the wrap span as `[rb_pa + (start_page + i) % page_count * PAGE_SIZE]`. +/// Returns the new write offset (unchanged on failure). +fn write_fast(rb_pa: PhysAddr, size: usize, write_offset: usize, buf: &[u8]) -> usize { + const MAX_SPAN_PAGES: usize = 16; + // If `buf` would force the start page into the span twice, vmap rejects the + // duplicate. Hand off to the two-write slow path before truncating `buf`. + if buf.len() < size && write_offset % PAGE_SIZE + buf.len() > size { + return write_slow(rb_pa, size, write_offset, buf); + } + + // Inputs longer than the buffer overwrite the whole ring with the trailing bytes. + let (buf, start) = if buf.len() >= size { + (&buf[(buf.len() - size)..], 0) + } else { + (buf, write_offset) + }; + + let page_count = size / PAGE_SIZE; + let start_page = start / PAGE_SIZE; + let in_page_offset = start % PAGE_SIZE; + let span_pages = (in_page_offset + buf.len()).div_ceil(PAGE_SIZE); + if span_pages > MAX_SPAN_PAGES { + return write_slow(rb_pa, size, write_offset, buf); + } + let rb_pa: usize = rb_pa.as_u64().trunc(); + let mut span: arrayvec::ArrayVec, MAX_SPAN_PAGES> = + arrayvec::ArrayVec::new(); + for i in 0..span_pages { + let page_idx = (start_page + i) % page_count; + let Some(addr) = page_idx + .checked_mul(PAGE_SIZE) + .and_then(|off| rb_pa.checked_add(off)) + .and_then(PhysPageAddr::::new) + else { + return write_offset; + }; + span.push(addr); + } + + let Ok(ptr) = Vtl0PhysMutPtr::::new(&span, in_page_offset) else { + return write_offset; + }; + if ptr.write_slice_at_offset(0, buf).is_ok() { + (start + buf.len()) % size + } else { + write_offset + } +} - // Otherwise, calculate if wraparound needed - let space_remaining: usize = self.size - self.write_offset; - if buf.len() > space_remaining { - let first_slice = &buf[..space_remaining]; - let wraparound_slice = &buf[space_remaining..]; - unsafe { - crate::platform_low() - .copy_slice_to_vtl0_phys(self.rb_pa + self.write_offset as u64, first_slice); - crate::platform_low().copy_slice_to_vtl0_phys(self.rb_pa, wraparound_slice); - } +/// Slow path used when `rb_pa` or `size` is not page-aligned/page-multiple. +/// Wraparound issues two map/unmap cycles; the returned offset advances by +/// bytes actually written so a mid-sequence failure does not strand stale data. +/// Failure return values: +/// - First slice fails: nothing written, cursor unchanged (`write_offset`). +/// - Second slice fails after first succeeded: `space_remaining` bytes were +/// written, so the cursor advances to `(write_offset + space_remaining) % size = 0`. +fn write_slow(rb_pa: PhysAddr, size: usize, write_offset: usize, buf: &[u8]) -> usize { + let write_slice = |pa: PhysAddr, slice: &[u8]| -> bool { + Vtl0PhysMutPtr::::with_contiguous_pages(pa.as_u64().trunc(), slice.len()) + .and_then(|ptr| ptr.write_slice_at_offset(0, slice)) + .is_ok() + }; + + if buf.len() >= size { + let single_slice = &buf[(buf.len() - size)..]; + return if write_slice(rb_pa, single_slice) { + 0 } else { - unsafe { - crate::platform_low() - .copy_slice_to_vtl0_phys(self.rb_pa + self.write_offset as u64, buf); - } + write_offset + }; + } + + let space_remaining = size - write_offset; + if buf.len() > space_remaining { + let first_slice = &buf[..space_remaining]; + let wraparound_slice = &buf[space_remaining..]; + if !write_slice(rb_pa + write_offset as u64, first_slice) { + return write_offset; + } + if !write_slice(rb_pa, wraparound_slice) { + // `space_remaining` bytes written; cursor wraps to 0. + return 0; } - self.write_offset = (self.write_offset + buf.len()) % self.size; + (write_offset + buf.len()) % size + } else if write_slice(rb_pa + write_offset as u64, buf) { + (write_offset + buf.len()) % size + } else { + write_offset } } diff --git a/litebox_platform_lvbs/src/mshv/vsm.rs b/litebox_platform_lvbs/src/mshv/vsm.rs index e2184c2f0..29cc0442f 100644 --- a/litebox_platform_lvbs/src/mshv/vsm.rs +++ b/litebox_platform_lvbs/src/mshv/vsm.rs @@ -7,7 +7,7 @@ use crate::mshv::mem_integrity::parse_modinfo; use crate::mshv::ringbuffer::set_ringbuffer; use crate::{ - debug_serial_println, + Vtl0PhysConstPtr, Vtl0PhysMutPtr, debug_serial_println, host::{ PRK_LEN, bootparam::get_vtl1_memory_info, @@ -49,7 +49,7 @@ use core::{ }; use hashbrown::{HashMap, HashSet}; use litebox::utils::TruncateExt; -use litebox_common_linux::errno::Errno; +use litebox_common_linux::{errno::Errno, vmap::PhysPageAddr}; use spin::Once; use thiserror::Error; use x86_64::{ @@ -57,13 +57,9 @@ use x86_64::{ structures::paging::{PageSize, PhysFrame, Size4KiB, frame::PhysFrameRange}, }; use x509_cert::{Certificate, der::Decode}; -use zerocopy::{FromBytes, FromZeros, Immutable, IntoBytes, KnownLayout}; +use zerocopy::{FromBytes, FromZeros, IntoBytes}; use zeroize::Zeroizing; -#[derive(Copy, Clone, FromBytes, Immutable, KnownLayout)] -#[repr(align(4096))] -struct AlignedPage([u8; PAGE_SIZE]); - // For now, we do not validate large kernel modules due to the VTL1's memory size limitation. const MODULE_VALIDATION_MAX_SIZE: usize = 64 * 1024 * 1024; @@ -128,11 +124,13 @@ pub fn mshv_vsm_boot_aps(cpu_online_mask_pfn: u64) -> Result { .and_then(|pa| PhysAddr::try_new(pa).ok()) .ok_or(VsmError::InvalidPhysicalAddress)?; - let Some(cpu_mask) = (unsafe { - crate::platform_low().copy_from_vtl0_phys::(cpu_online_mask_page_addr) - }) else { - return Err(VsmError::CpuOnlineMaskCopyFailed); - }; + let cpu_mask_ptr = Vtl0PhysConstPtr::::with_usize( + cpu_online_mask_page_addr.as_u64().trunc(), + ) + .map_err(|_| VsmError::CpuOnlineMaskCopyFailed)?; + let cpu_mask = cpu_mask_ptr + .read_at_offset(0) + .map_err(|_| VsmError::CpuOnlineMaskCopyFailed)?; #[cfg(debug_assertions)] { @@ -848,23 +846,27 @@ fn copy_heki_patch_from_vtl0(patch_pa_0: u64, patch_pa_1: u64) -> Result(patch_pa_0) } + let ptr = Vtl0PhysConstPtr::::with_usize(patch_pa_0.as_u64().trunc()) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + ptr.read_at_offset(0) .map(|boxed| *boxed) - .ok_or(VsmError::Vtl0CopyFailed) + .map_err(|_| VsmError::Vtl0CopyFailed) } else { let mut heki_patch = HekiPatch::new_zeroed(); let heki_patch_bytes = heki_patch.as_mut_bytes(); - unsafe { - if !crate::platform_low().copy_slice_from_vtl0_phys( - patch_pa_0, - heki_patch_bytes.get_unchecked_mut(..bytes_in_first_page), - ) || !crate::platform_low().copy_slice_from_vtl0_phys( - patch_pa_1, - heki_patch_bytes.get_unchecked_mut(bytes_in_first_page..), - ) { - return Err(VsmError::Vtl0CopyFailed); - } - } + let pages = [ + PhysPageAddr::::new(patch_pa_0.align_down(Size4KiB::SIZE).as_u64().trunc()) + .ok_or(VsmError::Vtl0CopyFailed)?, + PhysPageAddr::::new(patch_pa_1.as_u64().trunc()) + .ok_or(VsmError::Vtl0CopyFailed)?, + ]; + let ptr = Vtl0PhysConstPtr::::new( + &pages, + (patch_pa_0 - patch_pa_0.align_down(Size4KiB::SIZE)).trunc(), + ) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + ptr.read_slice_at_offset(0, heki_patch_bytes) + .map_err(|_| VsmError::Vtl0CopyFailed)?; Ok(heki_patch) }?; @@ -882,32 +884,45 @@ fn apply_vtl0_text_patch(heki_patch: HekiPatch) -> Result<(), VsmError> { let heki_patch_pa_0 = PhysAddr::new(heki_patch.pa[0]); let heki_patch_pa_1 = PhysAddr::new(heki_patch.pa[1]); - let patch_target_page_offset: usize = - (heki_patch_pa_0 - heki_patch_pa_0.align_down(Size4KiB::SIZE)).trunc(); - let bytes_in_first_page = PAGE_SIZE - patch_target_page_offset; + let patch = &heki_patch.code[..usize::from(heki_patch.size)]; + if patch.is_empty() { + return Ok(()); + } if heki_patch_pa_1.is_null() || (heki_patch_pa_0.align_up(Size4KiB::SIZE) == heki_patch_pa_1.align_down(Size4KiB::SIZE)) { - if !unsafe { - crate::platform_low().copy_slice_to_vtl0_phys( - heki_patch_pa_0, - &heki_patch.code[..usize::from(heki_patch.size)], - ) - } { - return Err(VsmError::Vtl0CopyFailed); - } + // Single contiguous span: either fits in one page (pa_1 null) or pa_1 is the + // adjacent next page. `HekiPatch::is_valid` enforces this; assert in debug builds. + debug_assert!( + !heki_patch_pa_1.is_null() + || heki_patch_pa_0.as_u64() + patch.len() as u64 + <= heki_patch_pa_0.align_down(Size4KiB::SIZE).as_u64() + Size4KiB::SIZE, + "patch crosses page boundary but pa_1 is null" + ); + let ptr = Vtl0PhysMutPtr::::with_contiguous_pages( + heki_patch_pa_0.as_u64().trunc(), + patch.len(), + ) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + ptr.write_slice_at_offset(0, patch) + .map_err(|_| VsmError::Vtl0CopyFailed)?; } else { - let (patch_first, patch_second) = - heki_patch.code[..usize::from(heki_patch.size)].split_at(bytes_in_first_page); - - unsafe { - if !crate::platform_low().copy_slice_to_vtl0_phys(heki_patch_pa_0, patch_first) - || !crate::platform_low().copy_slice_to_vtl0_phys(heki_patch_pa_1, patch_second) - { - return Err(VsmError::Vtl0CopyFailed); - } - } + let pages = [ + PhysPageAddr::::new( + heki_patch_pa_0.align_down(Size4KiB::SIZE).as_u64().trunc(), + ) + .ok_or(VsmError::Vtl0CopyFailed)?, + PhysPageAddr::::new(heki_patch_pa_1.as_u64().trunc()) + .ok_or(VsmError::Vtl0CopyFailed)?, + ]; + let ptr = Vtl0PhysMutPtr::::new( + &pages, + (heki_patch_pa_0 - heki_patch_pa_0.align_down(Size4KiB::SIZE)).trunc(), + ) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + ptr.write_slice_at_offset(0, patch) + .map_err(|_| VsmError::Vtl0CopyFailed)?; } Ok(()) } @@ -945,12 +960,14 @@ fn mshv_vsm_set_platform_root_key(key_pa: u64) -> Result { let key_pa = PhysAddr::try_new(key_pa).map_err(|_| VsmError::InvalidPhysicalAddress)?; let mut keybuf = Zeroizing::new([0u8; PRK_LEN]); - if unsafe { crate::platform_low().copy_slice_from_vtl0_phys(key_pa, &mut *keybuf) } { - set_platform_root_key(&*keybuf); - Ok(0) - } else { - Err(VsmError::Vtl0CopyFailed) - } + let key_ptr = + Vtl0PhysConstPtr::::with_contiguous_pages(key_pa.as_u64().trunc(), PRK_LEN) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + key_ptr + .read_slice_at_offset(0, &mut *keybuf) + .map_err(|_| VsmError::Vtl0CopyFailed)?; + set_platform_root_key(&*keybuf); + Ok(0) } /// VSM function dispatcher @@ -1373,7 +1390,9 @@ fn copy_heki_pages_from_vtl0(pa: u64, nranges: u64) -> Option> { if visited_pages.contains(&cur_pa.as_u64()) { return None; } - let heki_page = (unsafe { crate::platform_low().copy_from_vtl0_phys::(cur_pa) })?; + let ptr = + Vtl0PhysConstPtr::::with_usize(cur_pa.as_u64().trunc()).ok()?; + let heki_page = ptr.read_at_offset(0).ok()?; if !heki_page.is_valid() { return None; } @@ -1644,28 +1663,25 @@ impl MemoryContainer { phys_start: PhysAddr, phys_end: PhysAddr, ) -> Result<(), MemoryContainerError> { - let mut bytes_to_copy: usize = (phys_end - phys_start).trunc(); - let mut phys_cur = phys_start; - - while phys_cur < phys_end { - let phys_aligned = phys_cur.align_down(Size4KiB::SIZE); - let Some(page) = - (unsafe { crate::platform_low().copy_from_vtl0_phys::(phys_aligned) }) - else { - return Err(MemoryContainerError::CopyFromVtl0Failed); - }; + let bytes_to_copy: usize = (phys_end - phys_start).trunc(); + if bytes_to_copy == 0 { + return Ok(()); + } - let src_offset: usize = (phys_cur - phys_aligned).trunc(); - let src_len = core::cmp::min(bytes_to_copy, PAGE_SIZE - src_offset); - let src = &page.0[src_offset..src_offset + src_len]; + let ptr = Vtl0PhysConstPtr::::with_contiguous_pages( + phys_start.as_u64().trunc(), + bytes_to_copy, + ) + .map_err(|_| MemoryContainerError::CopyFromVtl0Failed)?; - self.buf.extend_from_slice(src); - phys_cur = phys_cur - .as_u64() - .checked_add(src_len as u64) - .and_then(|next| PhysAddr::try_new(next).ok()) - .ok_or(MemoryContainerError::Overflow)?; - bytes_to_copy -= src_len; + let old_len = self.buf.len(); + self.buf.resize(old_len + bytes_to_copy, 0); + if ptr + .read_slice_at_offset(0, &mut self.buf[old_len..]) + .is_err() + { + self.buf.truncate(old_len); + return Err(MemoryContainerError::CopyFromVtl0Failed); } Ok(()) } diff --git a/litebox_platform_multiplex/Cargo.toml b/litebox_platform_multiplex/Cargo.toml index 1c48c3a73..1099d334e 100644 --- a/litebox_platform_multiplex/Cargo.toml +++ b/litebox_platform_multiplex/Cargo.toml @@ -21,7 +21,7 @@ platform_linux_snp = ["dep:litebox_platform_linux_kernel"] platform_linux_userland_with_linux_syscall = ["platform_linux_userland", "litebox_platform_linux_userland/linux_syscall"] platform_linux_userland_with_optee_syscall = ["platform_linux_userland", "litebox_platform_linux_userland/optee_syscall"] platform_lvbs_with_linux_syscall = ["platform_lvbs", "litebox_platform_lvbs/linux_syscall"] -platform_lvbs_with_optee_syscall = ["platform_lvbs", "litebox_platform_lvbs/optee_syscall"] +platform_lvbs_with_optee_syscall = ["platform_lvbs"] [lints] workspace = true diff --git a/litebox_runner_lvbs/src/lib.rs b/litebox_runner_lvbs/src/lib.rs index 0ae319e59..43773ab1e 100644 --- a/litebox_runner_lvbs/src/lib.rs +++ b/litebox_runner_lvbs/src/lib.rs @@ -280,10 +280,10 @@ fn optee_smc_handler_entry_inner( // Write back the SMC arguments page to normal world memory. // All OP-TEE return codes (success or error) are delivered via smc_args.args[0]. - let mut smc_args_ptr = NormalWorldMutPtr::::with_usize(smc_args_addr) + let smc_args_ptr = NormalWorldMutPtr::::with_usize(smc_args_addr) .map_err(|_| litebox_common_linux::errno::Errno::EINVAL)?; - // SAFETY: The SMC args are written back to normal world memory. - unsafe { smc_args_ptr.write_at_offset(0, smc_args_updated) } + smc_args_ptr + .write_at_offset(0, smc_args_updated) .map_err(|_| litebox_common_linux::errno::Errno::EFAULT)?; Ok(0) } @@ -422,13 +422,12 @@ fn optee_smc_handler(smc_args_addr: usize) -> OpteeSmcArgs { args }; - let Ok(mut smc_args_ptr) = + let Ok(smc_args_ptr) = NormalWorldConstPtr::::with_usize(smc_args_addr) else { return make_error_response(OpteeSmcReturnCode::EBadAddr); }; - // SAFETY: The SMC args are read from normal world memory into an owned copy. - let Ok(mut smc_args) = (unsafe { smc_args_ptr.read_at_offset(0) }) else { + let Ok(mut smc_args) = smc_args_ptr.read_at_offset(0) else { return make_error_response(OpteeSmcReturnCode::EBadAddr); }; let Ok(smc_result) = handle_optee_smc_args(&mut smc_args) else { @@ -1297,13 +1296,11 @@ fn write_msg_args_to_normal_world( let mut blob = vec![0u8; msg_args_size]; msg_args.serialize(&mut blob)?; - let mut ptr = NormalWorldMutPtr::::with_contiguous_pages( + let ptr = NormalWorldMutPtr::::with_contiguous_pages( msg_args_phys_addr.trunc(), msg_args_size, )?; - // SAFETY: Writing msg_args back to normal world memory at a valid physical address. - // The blob contains the serialized variable-length optee_msg_arg structure(s). - unsafe { ptr.write_slice_at_offset(0, &blob) }?; + ptr.write_slice_at_offset(0, &blob)?; Ok(()) } @@ -1323,13 +1320,11 @@ fn write_non_ta_msg_args_to_normal_world( let mut blob = vec![0u8; msg_args_size]; msg_args.serialize(&mut blob)?; - let mut ptr = NormalWorldMutPtr::::with_contiguous_pages( + let ptr = NormalWorldMutPtr::::with_contiguous_pages( msg_args_phys_addr.trunc(), msg_args_size, )?; - // SAFETY: Writing msg_args back to normal world memory at a valid physical address. - // The blob contains the serialized variable-length optee_msg_arg structure(s). - unsafe { ptr.write_slice_at_offset(0, &blob) }?; + ptr.write_slice_at_offset(0, &blob)?; Ok(()) } @@ -1355,10 +1350,8 @@ fn write_rpc_args_to_normal_world( let rpc_pa: usize = >::trunc(msg_args_phys_addr) .checked_add(msg_args_size) .ok_or(OpteeSmcReturnCode::EBadAddr)?; // RPC args are placed right after the main msg_args blob - let mut ptr = NormalWorldMutPtr::::with_contiguous_pages(rpc_pa, rpc_args_size)?; - // SAFETY: Writing rpc_args back to normal world memory at a valid physical address. - // The blob contains the serialized variable-length optee_msg_arg structure(s). - unsafe { ptr.write_slice_at_offset(0, &blob) }?; + let ptr = NormalWorldMutPtr::::with_contiguous_pages(rpc_pa, rpc_args_size)?; + ptr.write_slice_at_offset(0, &blob)?; Ok(()) } diff --git a/litebox_shim_optee/src/lib.rs b/litebox_shim_optee/src/lib.rs index af70c43b6..ec0b2c161 100644 --- a/litebox_shim_optee/src/lib.rs +++ b/litebox_shim_optee/src/lib.rs @@ -21,7 +21,7 @@ use litebox::{ shim::ContinueOperation, utils::{ReinterpretUnsignedExt, TruncateExt}, }; -use litebox_common_linux::{MapFlags, ProtFlags, errno::Errno}; +use litebox_common_linux::{MapFlags, ProtFlags, errno::Errno, vmap::GlobalVmapManager}; use litebox_common_optee::{ LdelfArg, LdelfSyscallRequest, SyscallRequest, TaFlags, TeeAlgorithm, TeeAlgorithmClass, TeeAttributeType, TeeCrypStateHandle, TeeHandleFlag, TeeIdentity, TeeLogin, TeeObjHandle, @@ -34,7 +34,6 @@ pub mod session; pub(crate) mod syscalls; pub mod msg_handler; -pub mod ptr; // Re-export session management types for convenience pub use session::{ @@ -1408,8 +1407,20 @@ impl SessionIdPool { } } -pub type NormalWorldConstPtr = crate::ptr::PhysConstPtr; -pub type NormalWorldMutPtr = crate::ptr::PhysMutPtr; +#[derive(Clone, Copy, Debug, Default)] +pub struct Vmap; + +impl GlobalVmapManager for Vmap { + type Manager = litebox_platform_multiplex::Platform; + fn manager() -> &'static Self::Manager { + litebox_platform_multiplex::platform() + } +} + +pub type NormalWorldConstPtr = + litebox_common_linux::physical_pointers::PhysConstPtr; +pub type NormalWorldMutPtr = + litebox_common_linux::physical_pointers::PhysMutPtr; #[cfg(test)] mod test_utils { diff --git a/litebox_shim_optee/src/msg_handler.rs b/litebox_shim_optee/src/msg_handler.rs index 9187d5b7f..826839b04 100644 --- a/litebox_shim_optee/src/msg_handler.rs +++ b/litebox_shim_optee/src/msg_handler.rs @@ -181,10 +181,11 @@ pub fn read_optee_msg_args_from_phys( let mut blob = alloc::vec![0u8; copy_size]; - let mut blob_ptr = + let blob_ptr = NormalWorldConstPtr::::with_contiguous_pages(phys_addr, copy_size) .map_err(|_| OpteeSmcReturnCode::EBadAddr)?; - unsafe { blob_ptr.read_slice_at_offset(0, &mut blob) } + blob_ptr + .read_slice_at_offset(0, &mut blob) .map_err(|_| OpteeSmcReturnCode::EBadAddr)?; parse_optee_msg_args(&blob, has_rpc_arg) @@ -695,11 +696,8 @@ impl ShmInfo { { return Err(OpteeSmcReturnCode::EBadAddr); } - let mut ptr = NormalWorldConstPtr::::new(&self.page_addrs, self.page_offset)?; - // SAFETY: bounds validated above; copy lands in a buffer owned by LiteBox to avoid TOCTOU issues. - unsafe { - ptr.read_slice_at_offset(offset, buffer)?; - } + let ptr = NormalWorldConstPtr::::new(&self.page_addrs, self.page_offset)?; + ptr.read_slice_at_offset(offset, buffer)?; Ok(()) } @@ -710,11 +708,8 @@ impl ShmInfo { if buffer.len() > self.len { return Err(OpteeSmcReturnCode::EBadAddr); } - let mut ptr = NormalWorldMutPtr::::new(&self.page_addrs, self.page_offset)?; - // SAFETY: bounds validated above; data comes from a buffer owned by LiteBox. - unsafe { - ptr.write_slice_at_offset(0, buffer)?; - } + let ptr = NormalWorldMutPtr::::new(&self.page_addrs, self.page_offset)?; + ptr.write_slice_at_offset(0, buffer)?; Ok(()) } } @@ -791,10 +786,11 @@ impl ShmRefMap { return Err(OpteeSmcReturnCode::EBadAddr); } visited_pages_data.insert(cur_addr); - let mut cur_ptr = NormalWorldConstPtr::::with_usize(cur_addr) + let cur_ptr = NormalWorldConstPtr::::with_usize(cur_addr) + .map_err(|_| OpteeSmcReturnCode::EBadAddr)?; + let pages_data = cur_ptr + .read_at_offset(0) .map_err(|_| OpteeSmcReturnCode::EBadAddr)?; - let pages_data = - unsafe { cur_ptr.read_at_offset(0) }.map_err(|_| OpteeSmcReturnCode::EBadAddr)?; let pages_len_before = pages.len(); for page in &pages_data.pages_list { if *page == 0 || pages.len() == num_pages { diff --git a/litebox_shim_optee/src/ptr.rs b/litebox_shim_optee/src/ptr.rs deleted file mode 100644 index 06a492506..000000000 --- a/litebox_shim_optee/src/ptr.rs +++ /dev/null @@ -1,585 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -//! Physical Pointer Abstraction with On-demand Mapping -//! -//! This module adds supports for accessing physical addresses (e.g., VTL0 or -//! normal-world physical memory) from LiteBox with on-demand mapping. -//! In the context of LVBS and OP-TEE, accessing physical memory is necessary -//! because VTL0 and VTL1 as well as normal world and secure world do not share -//! the same virtual address space, but they still have to share data through memory. -//! VTL1 and secure world receive physical addresses from VTL0 and normal world, -//! respectively, and they need to read from or write to those addresses. -//! -//! To simplify all these, we could persistently map the entire VTL0/normal-world -//! physical memory into VTL1/secure-world address space at once and just access them -//! through corresponding virtual addresses. However, this module does not take these -//! approaches due to scalability (e.g., how to deal with a system with terabytes of -//! physical memory?) and security concerns (e.g., data corruption or information -//! leakage due to concurrent or persistent access). -//! -//! Instead, the approach this module takes is to map the required physical memory -//! region on-demand when accessing them while using a LiteBox-owned buffer to copy -//! data to/from those regions. This way, this module can ensure that data must be -//! copied into LiteBox-owned memory before being used while avoiding any unknown -//! side effects due to persistent memory mapping. -//! -//! Considerations: -//! -//! Ideally, this module should be able to validate whether a given physical address -//! is okay to access or even exists in the first place. For example, accessing -//! LiteBox's own memory with this physical pointer abstraction must be prohibited to -//! prevent the Boomerang attack and any other undefined memory access. Also, some -//! device memory is mapped to certain physical address ranges and LiteBox should not -//! touch them without in-depth knowledge. However, this is a bit tricky because, in -//! many cases, LiteBox does not directly interact with the underlying hardware or -//! BIOS/UEFI such that it does not have complete knowledge of the physical memory -//! layout. In the case of LVBS, LiteBox obtains the physical memory information -//! from VTL0 including the total physical memory size and the memory range assigned -//! to VTL1/LiteBox. Thus, this module can at least confirm a given physical address -//! does not belong to VTL1's physical memory. -//! -//! This module should allow byte-level access while transparently handling page -//! mapping and data access across page boundaries. This could become complicated -//! when we consider multiple page sizes (e.g., 4 KiB, 2 MiB, 1 GiB). Also, -//! unaligned access is a matter to be considered. -//! -//! In addition, often times, this physical pointer abstraction is involved with -//! a list of physical addresses (i.e., scatter-gather list). For example, in -//! the worse case, a two-byte value can span across two non-contiguous physical -//! pages (the last byte of the first page and the first byte of the second page). -//! Thus, to enhance the performance, we may need to consider mapping multiple pages -//! at once, copy data from/to them, and unmap them later. -//! -//! When this module needs to access data across physical page boundaries, it assumes -//! that those physical pages are virtually contiguous in VTL0 or normal-world address -//! space. Otherwise, this module could end up with accessing misordered data. This is -//! best-effort assumption and ensuring this is the caller's responsibility (e.g., even -//! if this module always requires a list of physical addresses, the caller might -//! provide a wrong list by mistake or intentionally). - -// TODO: Since the below `PhysMutPtr` and `PhysConstPtr` are not OP-TEE specific, -// we can move them to a different crate (e.g., `litebox`) if needed. - -use litebox_common_linux::vmap::{ - PhysPageAddr, PhysPageMapInfo, PhysPageMapPermissions, PhysPointerError, VmapManager, -}; -use litebox_platform_multiplex::platform; -use zerocopy::FromBytes; - -/// Allocate a zeroed `Box` on the heap. -/// -/// # Panics -/// -/// Panics if `T` is a zero-sized type, since `alloc_zeroed` with a zero-sized -/// layout is undefined behavior. -fn box_new_zeroed() -> alloc::boxed::Box { - assert!( - core::mem::size_of::() > 0, - "box_new_zeroed does not support zero-sized types" - ); - let layout = core::alloc::Layout::new::(); - // Safety: layout has a non-zero size and correct alignment for T. - let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }.cast::(); - if ptr.is_null() { - alloc::alloc::handle_alloc_error(layout); - } - // Safety: ptr is a valid, zeroed, properly aligned heap allocation for T. - // T: FromBytes guarantees all-zero is a valid bit pattern. - unsafe { alloc::boxed::Box::from_raw(ptr) } -} - -#[inline] -fn align_down(address: usize, align: usize) -> usize { - address & !(align - 1) -} - -/// Represent a physical pointer to an object with on-demand mapping. -/// - `pages`: An array of page-aligned physical addresses. We expect physical addresses in this array are -/// virtually contiguous. -/// - `offset`: The offset within `pages[0]` where the object starts. It should be smaller than `ALIGN`. -/// - `count`: The number of objects of type `T` that can be accessed from this pointer. -/// - `map_info`: The mapping information of the currently mapped physical pages, if any. -/// - `T`: The type of the object being pointed to. `pages` with respect to `offset` should cover enough -/// memory for an object of type `T`. -#[derive(Clone)] -#[repr(C)] -pub struct PhysMutPtr { - pages: alloc::boxed::Box<[PhysPageAddr]>, - offset: usize, - count: usize, - map_info: Option>, - _type: core::marker::PhantomData, -} - -impl PhysMutPtr { - /// Create a new `PhysMutPtr` from the given physical page array and offset. - /// - /// All addresses in `pages` should be valid and aligned to `ALIGN`, and `offset` should be - /// smaller than `ALIGN`. Also, `pages` should contain enough pages to cover at least one - /// object of type `T` starting from `offset`. If these conditions are not met, this function - /// returns `Err(PhysPointerError)`. - pub fn new(pages: &[PhysPageAddr], offset: usize) -> Result { - if offset >= ALIGN { - return Err(PhysPointerError::InvalidBaseOffset(offset, ALIGN)); - } - let size = if pages.is_empty() { - 0 - } else { - pages - .len() - .checked_mul(ALIGN) - .ok_or(PhysPointerError::Overflow)? - - offset - }; - if size < core::mem::size_of::() { - return Err(PhysPointerError::InsufficientPhysicalPages( - size, - core::mem::size_of::(), - )); - } - platform().validate_unowned(pages)?; - Ok(Self { - pages: pages.into(), - offset, - count: size / core::mem::size_of::(), - map_info: None, - _type: core::marker::PhantomData, - }) - } - - /// Create a new `PhysMutPtr` from the given contiguous physical address and length. - /// - /// This is a shortcut for - /// `PhysMutPtr::new([align_down(pa), align_down(pa) + ALIGN, ..., align_up(pa + bytes) - ALIGN], pa % ALIGN)`. - /// This function assumes that `pa`, ..., `pa+bytes` are both physically and virtually contiguous. If not, - /// later accesses through `PhysMutPtr` may read/write data in a wrong order. - pub fn with_contiguous_pages(pa: usize, bytes: usize) -> Result { - if bytes < core::mem::size_of::() { - return Err(PhysPointerError::InsufficientPhysicalPages( - bytes, - core::mem::size_of::(), - )); - } - let start_page = align_down(pa, ALIGN); - let end_page = pa - .checked_add(bytes) - .and_then(|end| end.checked_next_multiple_of(ALIGN)) - .ok_or(PhysPointerError::Overflow)?; - let span = end_page - .checked_sub(start_page) - .ok_or(PhysPointerError::Overflow)?; - let mut pages = alloc::vec::Vec::with_capacity(span / ALIGN); - let mut current_page = start_page; - while current_page < end_page { - pages.push( - PhysPageAddr::::new(current_page) - .ok_or(PhysPointerError::InvalidPhysicalAddress(current_page))?, - ); - current_page = current_page - .checked_add(ALIGN) - .ok_or(PhysPointerError::Overflow)?; - } - Self::new(&pages, pa - start_page) - } - - /// Create a new `PhysMutPtr` from the given physical address for a single object. - /// - /// This is a shortcut for `PhysMutPtr::with_contiguous_pages(pa, size_of::())`. - /// - /// Note: This module doesn't provide `as_usize` because LiteBox should not dereference physical addresses directly. - pub fn with_usize(pa: usize) -> Result { - Self::with_contiguous_pages(pa, core::mem::size_of::()) - } - - /// Read the value at the given offset from the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, it might read corrupt data. - /// `FromBytes` is required to ensure T is valid for any bit pattern from untrusted physical memory. - pub unsafe fn read_at_offset( - &mut self, - count: usize, - ) -> Result, PhysPointerError> - where - T: FromBytes, - { - if count >= self.count { - return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); - } - let guard = unsafe { - self.map_and_get_ptr_guard( - count, - core::mem::size_of::(), - PhysPageMapPermissions::READ, - )? - }; - let mut boxed = box_new_zeroed::(); - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - core::ptr::from_mut::(boxed.as_mut()).cast::(), - guard.ptr.cast::(), - guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault reading from mapped physical page"); - result.map_err(|_| PhysPointerError::CopyFailed)?; - Ok(boxed) - } - - /// Read a slice of values at the given offset from the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, it might read corrupt data. - /// `FromBytes` is required to ensure T is valid for any bit pattern from untrusted physical memory. - pub unsafe fn read_slice_at_offset( - &mut self, - count: usize, - values: &mut [T], - ) -> Result<(), PhysPointerError> - where - T: FromBytes, - { - if count - .checked_add(values.len()) - .is_none_or(|end| end > self.count) - { - return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); - } - let guard = unsafe { - self.map_and_get_ptr_guard( - count, - core::mem::size_of_val(values), - PhysPageMapPermissions::READ, - )? - }; - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - values.as_mut_ptr().cast::(), - guard.ptr.cast::(), - guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault reading from mapped physical page"); - result.map_err(|_| PhysPointerError::CopyFailed)?; - Ok(()) - } - - /// Write the value at the given offset to the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, data it writes might be overwritten. - pub unsafe fn write_at_offset( - &mut self, - count: usize, - value: T, - ) -> Result<(), PhysPointerError> { - if count >= self.count { - return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); - } - let guard = unsafe { - self.map_and_get_ptr_guard( - count, - core::mem::size_of::(), - PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE, - )? - }; - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - guard.ptr.cast::(), - core::ptr::from_ref(&value).cast::(), - guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault writing to mapped physical page"); - result.map_err(|_| PhysPointerError::CopyFailed)?; - Ok(()) - } - - /// Write a slice of values at the given offset to the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, data it writes might be overwritten. - pub unsafe fn write_slice_at_offset( - &mut self, - count: usize, - values: &[T], - ) -> Result<(), PhysPointerError> { - if count - .checked_add(values.len()) - .is_none_or(|end| end > self.count) - { - return Err(PhysPointerError::IndexOutOfBounds(count, self.count)); - } - let guard = unsafe { - self.map_and_get_ptr_guard( - count, - core::mem::size_of_val(values), - PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE, - )? - }; - // Fallible: another core may unmap this page concurrently. - let result = unsafe { - litebox::mm::exception_table::memcpy_fallible( - guard.ptr.cast::(), - values.as_ptr().cast::(), - guard.size, - ) - }; - debug_assert!(result.is_ok(), "fault writing to mapped physical page"); - result.map_err(|_| PhysPointerError::CopyFailed)?; - Ok(()) - } - - /// This function maps physical pages for the requested data element at a given - /// index and returns a guard that unmaps on drop. - /// - /// It bridges element-level access (used by `read_at_offset`, `write_at_offset`, etc.) - /// with page-level mapping. It determines which physical pages contain the requested - /// element, maps them into virtual memory, and returns a pointer adjusted for - /// the element's position. - /// - /// - `count`: Element index (0-based) within this physical pointer's range. - /// - `size`: Total byte size to map (must cover the data being accessed). - /// - `perms`: Required page permissions (read, write). - /// - /// # Safety - /// - /// Same as [`Self::map_range`]. The returned guard borrows `self` mutably, ensuring - /// the mapping is released when the guard goes out of scope. - unsafe fn map_and_get_ptr_guard( - &mut self, - count: usize, - size: usize, - perms: PhysPageMapPermissions, - ) -> Result, PhysPointerError> { - let skip = self - .offset - .checked_add( - count - .checked_mul(core::mem::size_of::()) - .ok_or(PhysPointerError::Overflow)?, - ) - .ok_or(PhysPointerError::Overflow)?; - let start = skip / ALIGN; - let end = skip - .checked_add(size) - .ok_or(PhysPointerError::Overflow)? - .div_ceil(ALIGN); - unsafe { - self.map_range(start, end, perms)?; - } - let map_info = self - .map_info - .as_ref() - .ok_or(PhysPointerError::NoMappingInfo)?; - let ptr = map_info.base.wrapping_add(skip % ALIGN).cast::(); - let _ = map_info; - Ok(MappedGuard { - owner: self, - ptr, - size, - }) - } - - /// Map the physical pages from `start` to `end` indexes. - /// - /// # Safety - /// - /// This function assumes that the underlying platform safely handles concurrent mapping/unmapping - /// requests for the same physical pages. - unsafe fn map_range( - &mut self, - start: usize, - end: usize, - perms: PhysPageMapPermissions, - ) -> Result<(), PhysPointerError> { - if start >= end || end > self.pages.len() { - return Err(PhysPointerError::IndexOutOfBounds(end, self.pages.len())); - } - let accept_perms = PhysPageMapPermissions::READ | PhysPageMapPermissions::WRITE; - if perms.bits() & !accept_perms.bits() != 0 { - return Err(PhysPointerError::UnsupportedPermissions(perms.bits())); - } - if self.map_info.is_none() { - let sub_pages = &self.pages[start..end]; - unsafe { - self.map_info = Some(platform().vmap(sub_pages, perms)?); - } - Ok(()) - } else { - Err(PhysPointerError::AlreadyMapped( - self.pages.first().map_or(0, |p| p.as_usize()), - )) - } - } - - /// Unmap the physical pages if mapped. - /// - /// # Safety - /// - /// This function assumes that the underlying platform safely handles concurrent mapping/unmapping - /// requests for the same physical pages. - unsafe fn unmap(&mut self) -> Result<(), PhysPointerError> { - if let Some(map_info) = self.map_info.take() { - unsafe { - platform().vunmap(map_info)?; - } - Ok(()) - } else { - Err(PhysPointerError::Unmapped( - self.pages.first().map_or(0, |p| p.as_usize()), - )) - } - } -} - -/// RAII guard that unmaps physical pages when dropped. -/// -/// Created by `map_and_get_ptr_guard`. Holds a mutable borrow on the parent -/// `PhysMutPtr` and provides the mapped base pointer for the duration of the mapping. -struct MappedGuard<'a, T: Clone, const ALIGN: usize> { - owner: &'a mut PhysMutPtr, - ptr: *mut T, - size: usize, -} - -impl Drop for MappedGuard<'_, T, ALIGN> { - fn drop(&mut self) { - // SAFETY: The platform is expected to handle unmapping safely, including - // the case where pages were never mapped (returns Unmapped error, ignored). - let result = unsafe { self.owner.unmap() }; - debug_assert!( - result.is_ok() || matches!(result, Err(PhysPointerError::Unmapped(_))), - "unexpected error during unmap in drop: {result:?}", - ); - } -} - -impl Drop for PhysMutPtr { - fn drop(&mut self) { - // SAFETY: The platform is expected to handle unmapping safely, including - // the case where pages were never mapped (returns Unmapped error, ignored). - let result = unsafe { self.unmap() }; - debug_assert!( - result.is_ok() || matches!(result, Err(PhysPointerError::Unmapped(_))), - "unexpected error during unmap in drop: {result:?}", - ); - } -} - -impl core::fmt::Debug for PhysMutPtr { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("PhysMutPtr") - .field("pages[0]", &self.pages.first().map_or(0, |p| p.as_usize())) - .field("offset", &self.offset) - .finish_non_exhaustive() - } -} - -/// Represent a physical pointer to a read-only object. This wraps around [`PhysMutPtr`] and -/// exposes only read access. -#[derive(Clone)] -#[repr(C)] -pub struct PhysConstPtr { - inner: PhysMutPtr, -} - -impl PhysConstPtr { - /// Create a new `PhysConstPtr` from the given physical page array and offset. - /// - /// All addresses in `pages` should be valid and aligned to `ALIGN`, and `offset` should be smaller - /// than `ALIGN`. Also, `pages` should contain enough pages to cover at least one object of - /// type `T` starting from `offset`. If these conditions are not met, this function returns - /// `Err(PhysPointerError)`. - pub fn new(pages: &[PhysPageAddr], offset: usize) -> Result { - Ok(Self { - inner: PhysMutPtr::new(pages, offset)?, - }) - } - - /// Create a new `PhysConstPtr` from the given contiguous physical address and length. - /// - /// This is a shortcut for - /// `PhysConstPtr::new([align_down(pa), align_down(pa) + ALIGN, ..., align_up(pa + bytes) - ALIGN], pa % ALIGN)`. - /// This function assumes that `pa`, ..., `pa+bytes` are both physically and virtually contiguous. If not, - /// later accesses through `PhysConstPtr` may read data in a wrong order. - pub fn with_contiguous_pages(pa: usize, bytes: usize) -> Result { - Ok(Self { - inner: PhysMutPtr::with_contiguous_pages(pa, bytes)?, - }) - } - - /// Create a new `PhysConstPtr` from the given physical address for a single object. - /// - /// This is a shortcut for `PhysConstPtr::with_contiguous_pages(pa, size_of::())`. - /// - /// Note: This module doesn't provide `as_usize` because LiteBox should not dereference physical addresses directly. - pub fn with_usize(pa: usize) -> Result { - Ok(Self { - inner: PhysMutPtr::with_usize(pa)?, - }) - } - - /// Read the value at the given offset from the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, it might read corrupt data. - pub unsafe fn read_at_offset( - &mut self, - count: usize, - ) -> Result, PhysPointerError> - where - T: FromBytes, - { - unsafe { self.inner.read_at_offset(count) } - } - - /// Read a slice of values at the given offset from the physical pointer. - /// - /// # Safety - /// - /// The caller should be aware that the given physical address might be concurrently written by - /// other entities (e.g., the normal world kernel) if there is no extra security mechanism - /// in place (e.g., by the hypervisor or hardware). That is, it might read corrupt data. - pub unsafe fn read_slice_at_offset( - &mut self, - count: usize, - values: &mut [T], - ) -> Result<(), PhysPointerError> - where - T: FromBytes, - { - unsafe { self.inner.read_slice_at_offset(count, values) } - } -} - -impl core::fmt::Debug for PhysConstPtr { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("PhysConstPtr") - .field( - "pages[0]", - &self.inner.pages.first().map_or(0, |p| p.as_usize()), - ) - .field("offset", &self.inner.offset) - .finish_non_exhaustive() - } -}