diff --git a/benches/mock_bench/mmapper.rs b/benches/mock_bench/mmapper.rs index ba0cfbb4e1..3bf0165f0e 100644 --- a/benches/mock_bench/mmapper.rs +++ b/benches/mock_bench/mmapper.rs @@ -3,8 +3,7 @@ pub use criterion::Criterion; use mmtk::{ memory_manager, mmap_anno_test, util::{ - constants::BYTES_IN_PAGE, memory::MmapStrategy, test_private::MMAPPER, - test_util::fixtures::*, Address, + constants::BYTES_IN_PAGE, os::*, test_private::MMAPPER, test_util::fixtures::*, Address, }, }; @@ -76,10 +75,17 @@ pub fn bench(c: &mut Criterion) { c.bench_function("ensure_mapped_regular", |b| { let start = regular.align_down(BYTES_IN_PAGE); assert!(start.is_mapped()); - let strategy = MmapStrategy::new(false, mmtk::util::memory::MmapProtection::ReadWrite); let anno = mmap_anno_test!(); b.iter(|| { - MMAPPER.ensure_mapped(start, 1, strategy, anno).unwrap(); + MMAPPER + .ensure_mapped( + start, + 1, + HugePageSupport::No, + MmapProtection::ReadWrite, + anno, + ) + .unwrap(); }) }); } diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index fc16ba727b..af3dd502da 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -4,6 +4,7 @@ use crate::plan::PlanTraceObject; use crate::plan::VectorObjectQueue; use crate::policy::gc_work::TraceKind; use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage}; +use crate::util::os::*; use crate::util::ObjectReference; use crate::vm::slot::{MemorySlice, Slot}; use crate::vm::*; @@ -117,7 +118,7 @@ impl GCWork for ProcessModBuf { !gen.is_object_in_nursery(*obj), "{} was logged but is not mature. Dumping process memory maps:\n{}", *obj, - crate::util::memory::get_process_memory_maps(), + OS::get_process_memory_maps().unwrap(), ); ::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::( *obj, diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 4f0564b9e5..c07f96839f 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -10,10 +10,10 @@ use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, MetadataSpec}; use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; +use crate::util::os::*; use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; -use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -293,8 +293,8 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect(start.to_mut_ptr(), extent, PROT_NONE); + if let Err(e) = OS::set_memory_access(start, extent, MmapProtection::NoAccess) { + panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); } @@ -308,12 +308,8 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect( - start.to_mut_ptr(), - extent, - PROT_READ | PROT_WRITE | PROT_EXEC, - ); + if let Err(e) = OS::set_memory_access(start, extent, self.common().mmap_protection()) { + panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); } diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index e30c12e18a..6960a93721 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -292,7 +292,7 @@ impl LargeObjectSpace { FreeListPageResource::new_contiguous(common.start, common.extent, vm_map) }; pr.protect_memory_on_release = if protect_memory_on_release { - Some(common.mmap_strategy().prot) + Some(common.mmap_protection()) } else { None }; diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index 5395273d91..6f6636dfc2 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -14,12 +14,11 @@ use crate::util::heap::gc_trigger::GCTrigger; use crate::util::heap::layout::vm_layout::vm_layout; use crate::util::heap::PageResource; use crate::util::heap::VMRequest; -use crate::util::memory::MmapAnnotation; -use crate::util::memory::MmapStrategy; use crate::util::metadata::side_metadata::SideMetadataContext; use crate::util::metadata::side_metadata::SideMetadataSanity; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::ObjectReference; use crate::vm::VMBinding; @@ -255,11 +254,12 @@ impl LockFreeImmortalSpace { }; // Eagerly memory map the entire heap (also zero all the memory) - let strategy = MmapStrategy::new( - *args.options.transparent_hugepages, - crate::util::memory::MmapProtection::ReadWrite, - ); - crate::util::memory::dzmmap_noreplace( + let strategy = MmapStrategy::default() + .transparent_hugepages(*args.options.transparent_hugepages) + .prot(crate::util::os::MmapProtection::ReadWrite) + .replace(false) + .reserve(true); + crate::util::os::OS::dzmmap( start, aligned_total_bytes, strategy, diff --git a/src/policy/space.rs b/src/policy/space.rs index 4582fcfb3f..80d39d5589 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -30,7 +30,7 @@ use crate::util::heap::layout::Mmapper; use crate::util::heap::layout::VMMap; use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::heap::HeapMeta; -use crate::util::memory::{self, HugePageSupport, MmapProtection, MmapStrategy}; +use crate::util::os::*; use crate::vm::VMBinding; use std::marker::PhantomData; @@ -191,8 +191,13 @@ pub trait Space: 'static + SFT + Sync + Downcast { .ensure_mapped( res.start, res.pages, - self.common().mmap_strategy(), - &memory::MmapAnnotation::Space { + if *self.common().options.transparent_hugepages { + HugePageSupport::TransparentHugePages + } else { + HugePageSupport::No + }, + self.common().mmap_protection(), + &MmapAnnotation::Space { name: self.get_name(), }, ) @@ -202,7 +207,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { self.get_name(), )) { - memory::handle_mmap_error::(mmap_error, tls, res.start, bytes); + OS::handle_mmap_error::(mmap_error, tls, res.start, bytes); } }; let grow_space = || { @@ -227,7 +232,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { // TODO: Concurrent zeroing if self.common().zeroed { - memory::zero(res.start, bytes); + crate::util::memory::zero(res.start, bytes); } // Some assertions @@ -755,18 +760,11 @@ impl CommonSpace { self.vm_map } - pub fn mmap_strategy(&self) -> MmapStrategy { - MmapStrategy { - huge_page: if *self.options.transparent_hugepages { - HugePageSupport::TransparentHugePages - } else { - HugePageSupport::No - }, - prot: if self.permission_exec || cfg!(feature = "exec_permission_on_all_spaces") { - MmapProtection::ReadWriteExec - } else { - MmapProtection::ReadWrite - }, + pub fn mmap_protection(&self) -> MmapProtection { + if self.permission_exec || cfg!(feature = "exec_permission_on_all_spaces") { + MmapProtection::ReadWriteExec + } else { + MmapProtection::ReadWrite } } diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index 331ec330b8..8655805f20 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -1,30 +1,6 @@ use super::worker::ThreadId; use crate::util::options::AffinityKind; -#[cfg(target_os = "linux")] -use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; - -/// Represents the ID of a logical CPU on a system. -pub type CoreId = u16; - -// XXX: Maybe in the future we can use a library such as https://github.com/Elzair/core_affinity_rs -// to have an OS agnostic way of setting thread affinity. -#[cfg(target_os = "linux")] -/// Return the total number of cores allocated to the program. -pub fn get_total_num_cpus() -> u16 { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - sched_getaffinity(0, std::mem::size_of::(), &mut cs); - CPU_COUNT(&cs) as u16 - } -} - -#[cfg(not(target_os = "linux"))] -/// Return the total number of cores allocated to the program. -pub fn get_total_num_cpus() -> u16 { - unimplemented!() -} +use crate::util::os::*; impl AffinityKind { /// Resolve affinity of GC thread. Has a side-effect of calling into the kernel to set the @@ -36,51 +12,13 @@ impl AffinityKind { AffinityKind::AllInSet(cpuset) => { // Bind the current thread to all the cores in the set debug!("Set affinity for thread {} to cpuset {:?}", thread, cpuset); - bind_current_thread_to_cpuset(cpuset.as_slice()); + OS::bind_current_thread_to_cpuset(cpuset.as_slice()); } AffinityKind::RoundRobin(cpuset) => { let cpu = cpuset[thread % cpuset.len()]; debug!("Set affinity for thread {} to core {}", thread, cpu); - bind_current_thread_to_core(cpu); + OS::bind_current_thread_to_core(cpu); } } } } - -#[cfg(target_os = "linux")] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_core(cpu: CoreId) { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - CPU_SET(cpu as usize, &mut cs); - sched_setaffinity(0, std::mem::size_of::(), &cs); - } -} - -#[cfg(not(target_os = "linux"))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_core(_cpu: CoreId) { - unimplemented!() -} - -#[cfg(any(target_os = "linux", target_os = "android"))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - for cpu in cpuset { - CPU_SET(*cpu as usize, &mut cs); - } - sched_setaffinity(0, std::mem::size_of::(), &cs); - } -} - -#[cfg(not(any(target_os = "linux", target_os = "android")))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_cpuset(_cpuset: &[CoreId]) { - unimplemented!() -} diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index 2bf4fc700c..9608973902 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -2,6 +2,7 @@ use std::sync::Arc; +use super::allocator::AllocatorContext; use crate::policy::marksweepspace::native_ms::*; use crate::util::alloc::allocator; use crate::util::alloc::Allocator; @@ -10,8 +11,6 @@ use crate::util::Address; use crate::util::VMThread; use crate::vm::VMBinding; -use super::allocator::AllocatorContext; - /// A MiMalloc free list allocator #[repr(C)] pub struct FreeListAllocator { diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index 1c9dabf460..b80b522dda 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -14,8 +14,8 @@ use crate::util::heap::layout::vm_layout::*; use crate::util::heap::layout::CreateFreeListResult; use crate::util::heap::pageresource::CommonPageResource; use crate::util::heap::space_descriptor::SpaceDescriptor; -use crate::util::memory; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::vm::*; use std::marker::PhantomData; @@ -27,7 +27,7 @@ pub struct FreeListPageResource { sync: Mutex, _p: PhantomData, /// Protect memory on release, and unprotect on re-allocate. - pub(crate) protect_memory_on_release: Option, + pub(crate) protect_memory_on_release: Option, } unsafe impl Send for FreeListPageResource {} @@ -219,7 +219,11 @@ impl FreeListPageResource { // > (e.g., read versus read/write protection) exceeding the // > allowed maximum. assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = memory::mprotect(start, conversions::pages_to_bytes(pages)) { + if let Err(e) = OS::set_memory_access( + start, + conversions::pages_to_bytes(pages), + MmapProtection::NoAccess, + ) { panic!( "Failed at protecting memory (starting at {}): {:?}", start, e @@ -230,7 +234,7 @@ impl FreeListPageResource { /// Unprotect the memory fn munprotect(&self, start: Address, pages: usize) { assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = memory::munprotect( + if let Err(e) = OS::set_memory_access( start, conversions::pages_to_bytes(pages), self.protect_memory_on_release.unwrap(), diff --git a/src/util/heap/layout/map64.rs b/src/util/heap/layout/map64.rs index b6e0e5993c..de9e16ddc5 100644 --- a/src/util/heap/layout/map64.rs +++ b/src/util/heap/layout/map64.rs @@ -6,7 +6,7 @@ use crate::util::freelist::FreeList; use crate::util::heap::layout::heap_parameters::*; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::space_descriptor::SpaceDescriptor; -use crate::util::memory::MmapStrategy; +use crate::util::os::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::util::Address; use std::cell::UnsafeCell; diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index c69963b925..04cf041b7a 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -2,7 +2,7 @@ use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::conversions::raw_is_aligned; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::layout::Mmapper; -use crate::util::memory::*; +use crate::util::os::*; use crate::util::Address; use bytemuck::NoUninit; use std::{io::Result, sync::Mutex}; @@ -146,7 +146,7 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -162,7 +162,12 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { trace!("Trying to quarantine {group_range}"); - mmap_noreserve(group_start, group_bytes, strategy, anno)?; + let mmap_strategy = MmapStrategy::default() + .huge_page(huge_page_option) + .prot(MmapProtection::NoAccess) + .reserve(false) + .replace(false); + OS::dzmmap(group_start, group_bytes, mmap_strategy, anno)?; Ok(Some(MapState::Quarantined)) } MapState::Quarantined => { @@ -181,7 +186,8 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, + prot: MmapProtection, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -189,6 +195,11 @@ impl Mmapper for ChunkStateMmapper { let bytes = pages << LOG_BYTES_IN_PAGE; let range = ChunkRange::new_unaligned(start, bytes); + let mmap_strategy = MmapStrategy::default() + .huge_page(huge_page_option) + .prot(prot) + .reserve(true); + self.storage .bulk_transition_state(range, |group_range, state| { let group_start: Address = group_range.start; @@ -196,11 +207,11 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - dzmmap_noreplace(group_start, group_bytes, strategy, anno)?; + OS::dzmmap(group_start, group_bytes, mmap_strategy.replace(false), anno)?; Ok(Some(MapState::Mapped)) } MapState::Quarantined => { - unsafe { dzmmap(group_start, group_bytes, strategy, anno) }?; + OS::dzmmap(group_start, group_bytes, mmap_strategy.replace(true), anno)?; Ok(Some(MapState::Mapped)) } MapState::Mapped => Ok(None), @@ -231,7 +242,6 @@ mod tests { use super::*; use crate::mmap_anno_test; use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::memory; use crate::util::test_util::CHUNK_STATE_MMAPPER_TEST_REGION; use crate::util::test_util::{serial_test, with_cleanup}; use crate::util::{conversions, Address}; @@ -256,7 +266,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -271,7 +287,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -284,7 +300,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -299,7 +321,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -313,7 +335,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -328,7 +356,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs index c981677ab1..580b2e26b1 100644 --- a/src/util/heap/layout/mmapper/mod.rs +++ b/src/util/heap/layout/mmapper/mod.rs @@ -1,7 +1,4 @@ -use crate::util::{ - memory::{MmapAnnotation, MmapStrategy}, - Address, -}; +use crate::util::{os::*, Address}; use std::io::Result; #[allow(unused)] // Used in doc comment. @@ -77,7 +74,7 @@ pub trait Mmapper: Sync { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, anno: &MmapAnnotation, ) -> Result<()>; @@ -97,7 +94,8 @@ pub trait Mmapper: Sync { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, + prot: MmapProtection, anno: &MmapAnnotation, ) -> Result<()>; diff --git a/src/util/memory.rs b/src/util/memory.rs index 9a9d4d16dc..e3e04efc53 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -1,167 +1,4 @@ -use crate::util::alloc::AllocationError; -use crate::util::opaque_pointer::*; use crate::util::Address; -use crate::vm::{Collection, VMBinding}; -use bytemuck::NoUninit; -use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; -use std::io::{Error, Result}; -use sysinfo::MemoryRefreshKind; -use sysinfo::{RefreshKind, System}; - -#[cfg(target_os = "linux")] -// MAP_FIXED_NOREPLACE returns EEXIST if already mapped -const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED_NOREPLACE; -#[cfg(target_os = "macos")] -// MAP_FIXED is used instead of MAP_FIXED_NOREPLACE (which is not available on macOS). We are at the risk of overwriting pre-existing mappings. -const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; - -/// Strategy for performing mmap -#[derive(Debug, Copy, Clone)] -pub struct MmapStrategy { - /// Do we support huge pages? - pub huge_page: HugePageSupport, - /// The protection flags for mmap - pub prot: MmapProtection, -} - -impl MmapStrategy { - /// Create a new strategy - pub fn new(transparent_hugepages: bool, prot: MmapProtection) -> Self { - Self { - huge_page: if transparent_hugepages { - HugePageSupport::TransparentHugePages - } else { - HugePageSupport::No - }, - prot, - } - } - - /// The strategy for MMTk's own internal memory - pub const INTERNAL_MEMORY: Self = Self { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - }; - - /// The strategy for MMTk side metadata - pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; - - /// The strategy for MMTk's test memory - #[cfg(test)] - pub const TEST: Self = Self::INTERNAL_MEMORY; -} - -/// The protection flags for Mmap -#[repr(i32)] -#[derive(Debug, Copy, Clone)] -pub enum MmapProtection { - /// Allow read + write - ReadWrite, - /// Allow read + write + code execution - ReadWriteExec, - /// Do not allow any access - NoAccess, -} - -impl MmapProtection { - /// Turn the protection enum into the native flags - pub fn into_native_flags(self) -> libc::c_int { - match self { - Self::ReadWrite => PROT_READ | PROT_WRITE, - Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, - Self::NoAccess => PROT_NONE, - } - } -} - -/// Support for huge pages -#[repr(u8)] -#[derive(Debug, Copy, Clone, NoUninit)] -pub enum HugePageSupport { - /// No support for huge page - No, - /// Enable transparent huge pages for the pages that are mapped. This option is only for linux. - TransparentHugePages, -} - -/// Annotation for an mmap entry. -/// -/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed` -/// require an annotation that indicates the purpose of the memory mapping. -/// -/// This is for debugging. On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the -/// human-readable name for the given mmap region. The annotation is ignored on other platforms. -/// -/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous -/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but -/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is -/// called for a chunk for the first time, which reflects which space first attempted the mmap, not -/// which space is currently using the chunk. Use `crate::policy::space::print_vm_map` to print a -/// more accurate mapping between address ranges and spaces. -/// -/// On 32-bit architecture, side metadata are allocated in a chunked fasion. One single `mmap` -/// region will contain many different metadata. In that case, we simply annotate the whole region -/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`. -pub enum MmapAnnotation<'a> { - /// The mmap is for a space. - Space { - /// The name of the space. - name: &'a str, - }, - /// The mmap is for a side metadata. - SideMeta { - /// The name of the space. - space: &'a str, - /// The name of the side metadata. - meta: &'a str, - }, - /// The mmap is for a test case. Usually constructed using the [`mmap_anno_test!`] macro. - Test { - /// The source file. - file: &'a str, - /// The line number. - line: u32, - }, - /// For all other use cases. - Misc { - /// A human-readable descriptive name. - name: &'a str, - }, -} - -/// Construct an `MmapAnnotation::Test` with the current file name and line number. -#[macro_export] -macro_rules! mmap_anno_test { - () => { - &$crate::util::memory::MmapAnnotation::Test { - file: file!(), - line: line!(), - } - }; -} - -// Export this to external crates -pub use mmap_anno_test; - -impl std::fmt::Display for MmapAnnotation<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"), - MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"), - MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"), - MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"), - } - } -} - -/// Check the result from an mmap function in this module. -/// Return true if the mmap has failed due to an existing conflicting mapping. -pub(crate) fn result_is_mapped(result: Result<()>) -> bool { - match result { - Ok(_) => false, - Err(err) => err.raw_os_error().unwrap() == libc::EEXIST, - } -} /// Set a range of memory to 0. pub fn zero(start: Address, len: usize) { @@ -171,502 +8,6 @@ pub fn zero(start: Address, len: usize) { /// Set a range of memory to the given value. Similar to memset. pub fn set(start: Address, val: u8, len: usize) { unsafe { - std::ptr::write_bytes::(start.to_mut_ptr(), val, len); - } -} - -/// Demand-zero mmap: -/// This function mmaps the memory and guarantees to zero all mapped memory. -/// This function WILL overwrite existing memory mapping. The user of this function -/// needs to be aware of this, and use it cautiously. -/// -/// # Safety -/// This function WILL overwrite existing memory mapping if there is any. So only use this function if you know -/// the memory has been reserved by mmtk (e.g. after the use of mmap_noreserve()). Otherwise using this function -/// may corrupt others' data. -#[allow(clippy::let_and_return)] // Zeroing is not neceesary for some OS/s -pub unsafe fn dzmmap( - start: Address, - size: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - let flags = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; - let ret = mmap_fixed(start, size, flags, strategy, anno); - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] - if ret.is_ok() { - zero(start, size) - } - ret -} -/// Demand-zero mmap (no replace): -/// This function mmaps the memory and guarantees to zero all mapped memory. -/// This function will not overwrite existing memory mapping, and it will result Err if there is an existing mapping. -#[allow(clippy::let_and_return)] // Zeroing is not neceesary for some OS/s -pub fn dzmmap_noreplace( - start: Address, - size: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - let flags = MMAP_FLAGS; - let ret = mmap_fixed(start, size, flags, strategy, anno); - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] - if ret.is_ok() { - zero(start, size) - } - ret -} - -/// mmap with no swap space reserve: -/// This function does not reserve swap space for this mapping, which means there is no guarantee that writes to the -/// mapping can always be successful. In case of out of physical memory, one may get a segfault for writing to the mapping. -/// We can use this to reserve the address range, and then later overwrites the mapping with dzmmap(). -pub fn mmap_noreserve( - start: Address, - size: usize, - mut strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - strategy.prot = MmapProtection::NoAccess; - let flags = MMAP_FLAGS | libc::MAP_NORESERVE; - mmap_fixed(start, size, flags, strategy, anno) -} - -fn mmap_fixed( - start: Address, - size: usize, - flags: libc::c_int, - strategy: MmapStrategy, - _anno: &MmapAnnotation, -) -> Result<()> { - let ptr = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, - ptr, - )?; - - #[cfg(all( - any(target_os = "linux", target_os = "android"), - not(feature = "no_mmap_annotation") - ))] - { - // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that - // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not - // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` - // for other reasons, too. That includes `start` being an invalid address, and the - // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But - // since this prctl is used for debugging, we log the error instead of panicking. - let anno_str = _anno.to_string(); - let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( - &|| unsafe { - libc::prctl( - libc::PR_SET_VMA, - libc::PR_SET_VMA_ANON_NAME, - start.to_ptr::(), - size, - anno_cstr.as_ptr(), - ) - }, - 0, - ); - if let Err(e) = result { - debug!("Error while calling prctl: {e}"); - } - } - - match strategy.huge_page { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - #[cfg(target_os = "linux")] - { - wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) - } - // Setting the transparent hugepage option to true will not pass - // the validation on non-Linux OSes - #[cfg(not(target_os = "linux"))] - unreachable!() - } - } -} - -/// Unmap the given memory (in page granularity). This wraps the unsafe libc munmap call. -pub fn munmap(start: Address, size: usize) -> Result<()> { - wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) -} - -/// Properly handle errors from a mmap Result, including invoking the binding code in the case of -/// an OOM error. -pub fn handle_mmap_error( - error: Error, - tls: VMThread, - addr: Address, - bytes: usize, -) -> ! { - use std::io::ErrorKind; - - eprintln!("Failed to mmap {}, size {}", addr, bytes); - eprintln!("{}", get_process_memory_maps()); - - match error.kind() { - // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. - ErrorKind::OutOfMemory => { - // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. - // TODO: We may be able to remove this now. - ErrorKind::Other => { - // further check the error - if let Some(os_errno) = error.raw_os_error() { - // If it is OOM, we invoke out_of_memory() through the VM interface. - if os_errno == libc::ENOMEM { - // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - } - } - ErrorKind::AlreadyExists => { - panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); - } - _ => {} - } - panic!("Unexpected mmap failure: {:?}", error) -} - -/// Checks if the memory has already been mapped. If not, we panic. -/// -/// Note that the checking has a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped. -/// Be very careful about using this function. -/// -/// This function is currently left empty for non-linux, and should be implemented in the future. -/// As the function is only used for assertions, MMTk will still run even if we never panic. -pub(crate) fn panic_if_unmapped(_start: Address, _size: usize, _anno: &MmapAnnotation) { - #[cfg(target_os = "linux")] - { - let flags = MMAP_FLAGS; - match mmap_fixed( - _start, - _size, - flags, - MmapStrategy { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - }, - _anno, - ) { - Ok(_) => panic!("{} of size {} is not mapped", _start, _size), - Err(e) => { - assert!( - e.kind() == std::io::ErrorKind::AlreadyExists, - "Failed to check mapped: {:?}", - e - ); - } - } - } -} - -/// Unprotect the given memory (in page granularity) to allow access (PROT_READ/WRITE/EXEC). -pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - let prot = prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) -} - -/// Protect the given memory (in page granularity) to forbid any access (PROT_NONE). -pub fn mprotect(start: Address, size: usize) -> Result<()> { - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, - 0, - ) -} - -fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { - let ret = f(); - if ret == expect { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn get_process_memory_maps() -> String { - // print map - use std::fs::File; - use std::io::Read; - let mut data = String::new(); - let mut f = File::open("/proc/self/maps").unwrap(); - f.read_to_string(&mut data).unwrap(); - data -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(target_os = "macos")] -pub fn get_process_memory_maps() -> String { - // Get the current process ID (replace this with a specific PID if needed) - let pid = std::process::id(); - - // Execute the vmmap command - let output = std::process::Command::new("vmmap") - .arg(pid.to_string()) // Pass the PID as an argument - .output() // Capture the output - .expect("Failed to execute vmmap command"); - - // Check if the command was successful - if output.status.success() { - // Convert the command output to a string - let output_str = - std::str::from_utf8(&output.stdout).expect("Failed to convert output to string"); - output_str.to_string() - } else { - // Handle the error case - let error_message = - std::str::from_utf8(&output.stderr).expect("Failed to convert error message to string"); - panic!("Failed to get process memory map: {}", error_message) - } -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] -pub fn get_process_memory_maps() -> String { - "(process map unavailable)".to_string() -} - -/// Returns the total physical memory for the system in bytes. -pub(crate) fn get_system_total_memory() -> u64 { - // TODO: Note that if we want to get system info somewhere else in the future, we should - // refactor this instance into some global struct. sysinfo recommends sharing one instance of - // `System` instead of making multiple instances. - // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info - // - // If we refactor the `System` instance to use it for other purposes, please make sure start-up - // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using - // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to - // start-up time. During start-up, MMTk core only needs the total memory to initialize the - // `Options`. If we only load memory-related components on start-up, it should only take <1ms - // to initialize the `System` instance. - let sys = System::new_with_specifics( - RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), - ); - sys.total_memory() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::util::constants::BYTES_IN_PAGE; - use crate::util::test_util::MEMORY_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - - // In the tests, we will mmap this address. This address should not be in our heap (in case we mess up with other tests) - const START: Address = MEMORY_TEST_REGION.start; - - #[test] - fn test_mmap() { - serial_test(|| { - with_cleanup( - || { - let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - // We can overwrite with dzmmap - let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ); - }); - } - - #[test] - fn test_munmap() { - serial_test(|| { - with_cleanup( - || { - let res = dzmmap_noreplace( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!(), - ); - assert!(res.is_ok()); - let res = munmap(START, BYTES_IN_PAGE); - assert!(res.is_ok()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - fn test_mmap_noreplace() { - serial_test(|| { - with_cleanup( - || { - // Make sure we mmapped the memory - let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - // Use dzmmap_noreplace will fail - let res = dzmmap_noreplace( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!(), - ); - assert!(res.is_err()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }); - } - - #[test] - fn test_mmap_noreserve() { - serial_test(|| { - with_cleanup( - || { - let res = - mmap_noreserve(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()); - assert!(res.is_ok()); - // Try reserve it - let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - #[should_panic] - fn test_check_is_mmapped_for_unmapped() { - serial_test(|| { - with_cleanup( - || { - // We expect this call to panic - panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[test] - fn test_check_is_mmapped_for_mapped() { - serial_test(|| { - with_cleanup( - || { - assert!(dzmmap_noreplace( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!() - ) - .is_ok()); - panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - #[should_panic] - fn test_check_is_mmapped_for_unmapped_next_to_mapped() { - serial_test(|| { - with_cleanup( - || { - // map 1 page from START - assert!(dzmmap_noreplace( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .is_ok()); - - // check if the next page is mapped - which should panic - panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE, mmap_anno_test!()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE * 2).is_ok()); - }, - ) - }) - } - - #[test] - #[should_panic] - // This is a bug we need to fix. We need to figure out a way to properly check if a piece of memory is mapped or not. - // Alternatively, we should remove the code that calls the function. - #[ignore] - fn test_check_is_mmapped_for_partial_mapped() { - serial_test(|| { - with_cleanup( - || { - // map 1 page from START - assert!(dzmmap_noreplace( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!() - ) - .is_ok()); - - // check if the 2 pages from START are mapped. The second page is unmapped, so it should panic. - panic_if_unmapped(START, BYTES_IN_PAGE * 2, mmap_anno_test!()); - }, - || { - assert!(munmap(START, BYTES_IN_PAGE * 2).is_ok()); - }, - ) - }) - } - - #[test] - fn test_get_system_total_memory() { - let total = get_system_total_memory(); - println!("Total memory: {:?}", total); + std::ptr::write_bytes(start.to_mut_ptr::(), val, len); } } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 65ce7443e1..956ced34c5 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -2,10 +2,10 @@ use super::*; use crate::util::constants::{BYTES_IN_PAGE, BYTES_IN_WORD, LOG_BITS_IN_BYTE}; use crate::util::conversions::raw_align_up; use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK; -use crate::util::memory::{self, MmapAnnotation}; use crate::util::metadata::metadata_val_traits::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC; +use crate::util::os::*; use crate::util::Address; use num_traits::FromPrimitive; use ranges::BitByteRange; @@ -122,13 +122,7 @@ impl SideMetadataSpec { meta_start ); - memory::panic_if_unmapped( - meta_start, - BYTES_IN_PAGE, - &MmapAnnotation::Misc { - name: "assert_metadata_mapped", - }, - ); + OS::panic_if_unmapped(meta_start, BYTES_IN_PAGE); } #[cfg(debug_assertions)] @@ -180,7 +174,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - memory::zero(start, end - start); + crate::util::memory::zero(start, end - start); false } BitByteRange::BitsInByte { @@ -217,7 +211,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - memory::set(start, 0xff, end - start); + crate::util::memory::set(start, 0xff, end - start); false } BitByteRange::BitsInByte { @@ -1662,7 +1656,6 @@ mod tests { use crate::util::heap::layout::vm_layout; use crate::util::test_util::{serial_test, with_cleanup}; - use memory::MmapStrategy; use paste::paste; const TEST_LOG_BYTES_IN_REGION: usize = 12; @@ -1689,14 +1682,20 @@ mod tests { let data_addr = vm_layout::vm_layout().heap_start; // Make sure the address is mapped. crate::MMAPPER - .ensure_mapped(data_addr, 1, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped( + data_addr, + 1, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let meta_addr = address_to_meta_address(&spec, data_addr); with_cleanup( || { let mmap_result = context.try_map_metadata_space(data_addr, BYTES_IN_PAGE, "test_space"); - assert!(mmap_result.is_ok()); + assert!(mmap_result.is_ok(), "{:?}", mmap_result); f(&spec, data_addr, meta_addr); }, diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 24e5366dc8..bc1403003f 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -4,9 +4,9 @@ use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::{BITS_IN_WORD, BYTES_IN_PAGE, LOG_BITS_IN_BYTE}; use crate::util::conversions::rshift_align_up; use crate::util::heap::layout::vm_layout::VMLayout; -use crate::util::memory::{MmapAnnotation, MmapStrategy}; #[cfg(target_pointer_width = "32")] use crate::util::metadata::side_metadata::address_to_chunked_meta_address; +use crate::util::os::*; use crate::util::Address; use crate::MMAPPER; use std::io::Result; @@ -93,10 +93,9 @@ pub(super) fn align_metadata_address( /// Unmaps the specified metadata range, or panics. #[cfg(test)] pub(crate) fn ensure_munmap_metadata(start: Address, size: usize) { - use crate::util::memory; trace!("ensure_munmap_metadata({}, 0x{:x})", start, size); - assert!(memory::munmap(start, size).is_ok()) + assert!(OS::munmap(start, size).is_ok()) } /// Unmaps a metadata space (`spec`) for the specified data address range (`start` and `size`) @@ -143,14 +142,15 @@ pub(super) fn try_mmap_contiguous_metadata_space( MMAPPER.ensure_mapped( mmap_start, mmap_size >> LOG_BYTES_IN_PAGE, - MmapStrategy::SIDE_METADATA, + HugePageSupport::No, + MmapProtection::ReadWrite, anno, ) } else { MMAPPER.quarantine_address_range( mmap_start, mmap_size >> LOG_BYTES_IN_PAGE, - MmapStrategy::SIDE_METADATA, + HugePageSupport::No, anno, ) } diff --git a/src/util/metadata/side_metadata/helpers_32.rs b/src/util/metadata/side_metadata/helpers_32.rs index 1372d5916c..87ad6ee35a 100644 --- a/src/util/metadata/side_metadata/helpers_32.rs +++ b/src/util/metadata/side_metadata/helpers_32.rs @@ -2,7 +2,7 @@ use super::SideMetadataSpec; use crate::util::{ constants::{self, LOG_BITS_IN_BYTE}, heap::layout::vm_layout::{BYTES_IN_CHUNK, CHUNK_MASK, LOG_BYTES_IN_CHUNK}, - memory::{self, MmapAnnotation}, + os::*, Address, }; use std::io::Result; @@ -102,7 +102,7 @@ pub(super) const fn metadata_bytes_per_chunk( pub(crate) fn ensure_munmap_metadata_chunk(start: Address, local_per_chunk: usize) { if local_per_chunk != 0 { let policy_meta_start = address_to_meta_chunk_addr(start); - assert!(memory::munmap(policy_meta_start, local_per_chunk).is_ok()) + assert!(OS::munmap(policy_meta_start, local_per_chunk).is_ok()) } } @@ -156,7 +156,8 @@ pub(super) fn try_map_per_chunk_metadata_space( } if munmap_first_chunk.is_none() { // if first chunk is newly mapped, it needs munmap on failure - munmap_first_chunk = Some(memory::result_is_mapped(res)); + let map_exists = res.is_err_and(|e| e.kind() == std::io::ErrorKind::AlreadyExists); + munmap_first_chunk = Some(map_exists); } aligned_start += BYTES_IN_CHUNK; total_mapped += local_per_chunk; @@ -187,15 +188,11 @@ pub(super) fn try_mmap_metadata_chunk( MMAPPER.ensure_mapped( policy_meta_start, pages, - memory::MmapStrategy::SIDE_METADATA, + HugePageSupport::No, + MmapProtection::ReadWrite, anno, ) } else { - MMAPPER.quarantine_address_range( - policy_meta_start, - pages, - memory::MmapStrategy::SIDE_METADATA, - anno, - ) + MMAPPER.quarantine_address_range(policy_meta_start, pages, HugePageSupport::No, anno) } } diff --git a/src/util/mod.rs b/src/util/mod.rs index d22c29a2e3..6a542648f8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -28,7 +28,7 @@ pub mod is_mmtk_object; pub mod linear_scan; /// Various malloc implementations (conditionally compiled by features) pub mod malloc; -/// Wrapper functions for memory syscalls such as mmap, mprotect, etc. +/// Memory utilities (non-OS dependent). OS dependent memory utilities can be found in [`crate::util::os::OSMemory`]. pub mod memory; /// Metadata (OnSide or InHeader) implementation. pub mod metadata; @@ -36,6 +36,8 @@ pub mod metadata; pub mod opaque_pointer; /// MMTk command line options. pub mod options; +/// Operating system abstractions. +pub mod os; #[cfg(feature = "test_private")] pub mod test_private; /// Test utilities. We need this module for `MockVM` in criterion benches, which does not include code with `cfg(test)`. diff --git a/src/util/options.rs b/src/util/options.rs index fd8952d1da..216b162d61 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,5 +1,5 @@ -use crate::scheduler::affinity::{get_total_num_cpus, CoreId}; use crate::util::constants::LOG_BYTES_IN_MBYTE; +use crate::util::os::*; use crate::util::Address; use std::default::Default; use std::fmt::Debug; @@ -455,7 +455,7 @@ impl AffinityKind { /// maximum number of cores allocated to the program. Assumes core ids on the system are /// 0-indexed. pub fn validate(&self) -> bool { - let num_cpu = get_total_num_cpus(); + let num_cpu = OS::get_total_num_cpus(); if let AffinityKind::RoundRobin(cpuset) = self { for cpu in cpuset { @@ -950,7 +950,7 @@ options! { thread_affinity: AffinityKind [|v: &AffinityKind| v.validate()] = AffinityKind::OsDefault, /// Set the GC trigger. This defines the heap size and how MMTk triggers a GC. /// Default to a fixed heap size of 0.5x physical memory. - gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((crate::util::memory::get_system_total_memory() as f64 * 0.5f64) as usize), + gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((OS::get_system_total_memory().unwrap_or(4 * 1024 * 1024 * 1024) as f64 * 0.5f64) as usize), /// Enable transparent hugepage support for MMTk spaces via madvise (only Linux is supported) /// This only affects the memory for MMTk spaces. transparent_hugepages: bool [|v: &bool| !v || cfg!(target_os = "linux")] = false, @@ -1210,7 +1210,7 @@ mod tests { || { let mut vec = vec![0_u16]; let mut cpu_list = String::new(); - let num_cpus = get_total_num_cpus(); + let num_cpus = OS::get_total_num_cpus(); cpu_list.push('0'); for cpu in 1..num_cpus { diff --git a/src/util/os/imp/mod.rs b/src/util/os/imp/mod.rs new file mode 100644 index 0000000000..5e1fdc9680 --- /dev/null +++ b/src/util/os/imp/mod.rs @@ -0,0 +1,11 @@ +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] +pub(crate) mod unix_like; + +#[cfg(target_os = "linux")] +pub use unix_like::linux_like::linux::Linux as OS; + +#[cfg(target_os = "android")] +pub use unix_like::linux_like::android::Android as OS; + +#[cfg(target_os = "macos")] +pub use unix_like::macos::MacOS as OS; diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs new file mode 100644 index 0000000000..4fca815705 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -0,0 +1,65 @@ +use crate::util::address::Address; +use crate::util::os::imp::unix_like::linux_like::linux_common; +use crate::util::os::imp::unix_like::unix_common; +use crate::util::os::*; + +use std::io::Result; + +/// Android implementation of the `OS` trait. +pub struct Android; + +impl OSMemory for Android { + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
{ + linux_common::dzmmap(start, size, strategy, annotation) + } + + fn munmap(start: Address, size: usize) -> Result<()> { + unix_common::munmap(start, size) + } + + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + unix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(start: Address, size: usize) { + linux_common::panic_if_unmapped(start, size) + } +} + +impl OSProcess for Android { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + + fn get_process_memory_maps() -> Result { + linux_common::get_process_memory_maps() + } + + fn get_process_id() -> Result { + unix_common::get_process_id() + } + + fn get_thread_id() -> Result { + unix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + linux_common::get_total_num_cpus() + } + + fn bind_current_thread_to_core(core_id: CoreId) { + linux_common::bind_current_thread_to_core(core_id) + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + linux_common::bind_current_thread_to_cpuset(core_ids) + } +} diff --git a/src/util/os/imp/unix_like/linux_like/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs new file mode 100644 index 0000000000..a75a3d3ee4 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -0,0 +1,65 @@ +use crate::util::address::Address; +use crate::util::os::imp::unix_like::linux_like::linux_common; +use crate::util::os::imp::unix_like::unix_common; +use crate::util::os::*; + +use std::io::Result; + +/// Linux implementation of the `OS` trait. +pub struct Linux; + +impl OSMemory for Linux { + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
{ + linux_common::dzmmap(start, size, strategy, annotation) + } + + fn munmap(start: Address, size: usize) -> Result<()> { + unix_common::munmap(start, size) + } + + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + unix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(start: Address, size: usize) { + linux_common::panic_if_unmapped(start, size) + } +} + +impl OSProcess for Linux { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + + fn get_process_memory_maps() -> Result { + linux_common::get_process_memory_maps() + } + + fn get_process_id() -> Result { + unix_common::get_process_id() + } + + fn get_thread_id() -> Result { + unix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + linux_common::get_total_num_cpus() + } + + fn bind_current_thread_to_core(core_id: CoreId) { + linux_common::bind_current_thread_to_core(core_id) + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + linux_common::bind_current_thread_to_cpuset(core_ids) + } +} diff --git a/src/util/os/imp/unix_like/linux_like/linux_common.rs b/src/util/os/imp/unix_like/linux_like/linux_common.rs new file mode 100644 index 0000000000..d9c013bb65 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/linux_common.rs @@ -0,0 +1,140 @@ +use crate::util::address::Address; +use crate::util::os::imp::unix_like::unix_common; +use crate::util::os::*; +use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; +use std::io::Result; + +pub fn set_vma_name(start: Address, size: usize, annotation: &MmapAnnotation) { + // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that + // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not + // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` + // for other reasons, too. That includes `start` being an invalid address, and the + // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But + // since this prctl is used for debugging, we log the error instead of panicking. + let anno_str = annotation.to_string(); + let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); + let result = unix_common::wrap_libc_call( + &|| unsafe { + libc::prctl( + libc::PR_SET_VMA, + libc::PR_SET_VMA_ANON_NAME, + start.to_ptr::(), + size, + anno_cstr.as_ptr(), + ) + }, + 0, + ); + if let Err(e) = result { + debug!("Error while calling prctl: {e}"); + } +} + +/// Set huge page option for the given memory. +pub fn set_hugepage(start: Address, size: usize, options: HugePageSupport) -> Result<()> { + match options { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => unix_common::wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ), + } +} + +impl MmapStrategy { + /// get the flags for POSIX mmap. + pub fn get_posix_mmap_flags(&self) -> i32 { + let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; + if self.replace { + flags |= libc::MAP_FIXED; + } else { + flags |= libc::MAP_FIXED_NOREPLACE + } + if !self.reserve { + flags |= libc::MAP_NORESERVE; + } + flags + } +} + +/// Get the memory maps for the process. The returned string is a multi-line string. +/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. +pub fn get_process_memory_maps() -> Result { + // print map + use std::fs::File; + use std::io::Read; + let mut data = String::new(); + let mut f = File::open("/proc/self/maps")?; + f.read_to_string(&mut data)?; + Ok(data) +} + +pub fn get_total_num_cpus() -> CoreNum { + use std::mem::MaybeUninit; + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + sched_getaffinity(0, std::mem::size_of::(), &mut cs); + CPU_COUNT(&cs) as u16 + } +} + +pub fn bind_current_thread_to_core(core_id: CoreId) { + use std::mem::MaybeUninit; + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + CPU_SET(core_id as usize, &mut cs); + sched_setaffinity(0, std::mem::size_of::(), &cs); + } +} + +pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { + use std::mem::MaybeUninit; + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + for cpu in cpuset { + CPU_SET(*cpu as usize, &mut cs); + } + sched_setaffinity(0, std::mem::size_of::(), &cs); + } +} + +pub fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, +) -> Result
{ + let addr = unix_common::mmap(start, size, strategy)?; + + if !cfg!(feature = "no_mmap_annotation") { + set_vma_name(addr, size, annotation); + } + + set_hugepage(addr, size, strategy.huge_page)?; + + // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) + + Ok(addr) +} + +pub fn panic_if_unmapped(start: Address, size: usize) { + let strategy = MmapStrategy { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + match unix_common::mmap(start, size, strategy) { + Ok(_) => panic!("{} of size {} is not mapped", start, size), + Err(e) => { + assert!( + e.kind() == std::io::ErrorKind::AlreadyExists, + "Failed to check mapped: {:?}", + e + ); + } + } +} diff --git a/src/util/os/imp/unix_like/linux_like/mod.rs b/src/util/os/imp/unix_like/linux_like/mod.rs new file mode 100644 index 0000000000..59f3bbb521 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/mod.rs @@ -0,0 +1,7 @@ +pub mod linux_common; + +#[cfg(target_os = "linux")] +pub mod linux; + +#[cfg(target_os = "android")] +pub mod android; diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs new file mode 100644 index 0000000000..5d73ddd01b --- /dev/null +++ b/src/util/os/imp/unix_like/macos.rs @@ -0,0 +1,107 @@ +use crate::util::address::Address; +use crate::util::os::imp::unix_like::unix_common; +use crate::util::os::*; +use std::io::Result; + +/// MacOS implementation of the `OS` trait. +pub struct MacOS; + +impl OSMemory for MacOS { + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + _annotation: &MmapAnnotation<'_>, + ) -> Result
{ + let addr = unix_common::mmap(start, size, strategy)?; + + // Annotation is ignored on macOS + // Huge page is ignored on macOS + + // Zero memory if we actually reserve the memory + if strategy.reserve { + crate::util::memory::zero(start, size); + } + Ok(addr) + } + + fn munmap(start: Address, size: usize) -> Result<()> { + unix_common::munmap(start, size) + } + + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + unix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(_start: Address, _size: usize) { + // Unimplemented for now + } +} + +impl MmapStrategy { + /// get the flags for POSIX mmap. + pub fn get_posix_mmap_flags(&self) -> i32 { + let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED; + // replace is isgnored on macOS + if !self.reserve { + flags |= libc::MAP_NORESERVE; + } + flags + } +} + +impl OSProcess for MacOS { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + + fn get_process_memory_maps() -> Result { + // Get the current process ID (replace this with a specific PID if needed) + let pid = std::process::id(); + + // Execute the vmmap command + let output = std::process::Command::new("vmmap") + .arg(pid.to_string()) // Pass the PID as an argument + .output() // Capture the output + .expect("Failed to execute vmmap command"); + + // Check if the command was successful + if output.status.success() { + // Convert the command output to a string + let output_str = + std::str::from_utf8(&output.stdout).expect("Failed to convert output to string"); + Ok(output_str.to_string()) + } else { + // Handle the error case + let error_message = std::str::from_utf8(&output.stderr) + .expect("Failed to convert error message to string"); + Err(std::io::Error::other(format!( + "Failed to get process memory map: {}", + error_message + ))) + } + } + + fn get_process_id() -> Result { + unix_common::get_process_id() + } + + fn get_thread_id() -> Result { + unix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + unimplemented!() + } + + fn bind_current_thread_to_core(_core_id: CoreId) { + unimplemented!() + } + + fn bind_current_thread_to_cpuset(_core_ids: &[CoreId]) { + unimplemented!() + } +} diff --git a/src/util/os/imp/unix_like/mod.rs b/src/util/os/imp/unix_like/mod.rs new file mode 100644 index 0000000000..ba1c7f9c7d --- /dev/null +++ b/src/util/os/imp/unix_like/mod.rs @@ -0,0 +1,7 @@ +pub mod unix_common; + +#[cfg(target_os = "macos")] +pub mod macos; + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod linux_like; diff --git a/src/util/os/imp/unix_like/unix_common.rs b/src/util/os/imp/unix_like/unix_common.rs new file mode 100644 index 0000000000..6fa643dc48 --- /dev/null +++ b/src/util/os/imp/unix_like/unix_common.rs @@ -0,0 +1,60 @@ +use crate::util::address::Address; +use crate::util::os::*; +use std::io::Result; + +impl MmapProtection { + fn get_native_flags(&self) -> i32 { + use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; + match self { + Self::ReadWrite => PROT_READ | PROT_WRITE, + Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, + Self::NoAccess => PROT_NONE, + } + } +} + +pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.get_native_flags(); + let flags = strategy.get_posix_mmap_flags(); + wrap_libc_call( + &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, + ptr, + )?; + Ok(start) +} + +pub fn is_mmap_oom(os_errno: i32) -> bool { + os_errno == libc::ENOMEM +} + +pub fn munmap(start: Address, size: usize) -> Result<()> { + wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) +} + +pub fn mprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.get_native_flags()) }, + 0, + ) +} + +pub type ProcessIDType = libc::pid_t; +pub type ThreadIDType = libc::pthread_t; + +pub fn get_process_id() -> Result { + Ok(unsafe { libc::getpid() }) +} + +pub fn get_thread_id() -> Result { + Ok(unsafe { libc::pthread_self() }) +} + +pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { + let ret = f(); + if ret == expect { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs new file mode 100644 index 0000000000..006d4059e0 --- /dev/null +++ b/src/util/os/memory.rs @@ -0,0 +1,300 @@ +use bytemuck::NoUninit; +use std::io::Result; + +use crate::util::os::*; +use crate::vm::*; +use crate::{ + util::{address::Address, VMThread}, + vm::VMBinding, +}; + +/// Abstraction for OS memory operations. +pub trait OSMemory { + /// Perform a demand-zero mmap. + /// + /// Fallback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. + /// Fallback: see [`crate::util::os::MmapStrategy`] for fallbacks for `strategy`. + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
; + + /// Handle mmap errors, possibly by signaling the VM about an out-of-memory condition. + fn handle_mmap_error( + error: std::io::Error, + tls: VMThread, + addr: Address, + bytes: usize, + ) { + use crate::util::alloc::AllocationError; + use std::io::ErrorKind; + + eprintln!("Failed to mmap {}, size {}", addr, bytes); + eprintln!("{}", OS::get_process_memory_maps().unwrap()); + + let call_binding_oom = || { + // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + }; + + match error.kind() { + // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. + ErrorKind::OutOfMemory => { + call_binding_oom(); + } + // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. + // TODO: We may be able to remove this now. + ErrorKind::Other => { + // further check the error + if let Some(os_errno) = error.raw_os_error() { + if OS::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + ErrorKind::AlreadyExists => { + panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); + } + _ => { + if let Some(os_errno) = error.raw_os_error() { + if OS::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + } + panic!("Unexpected mmap failure: {:?}", error) + } + + /// Check whether the given OS error number indicates an out-of-memory condition. + fn is_mmap_oom(os_errno: i32) -> bool; + + /// Unmap a memory region. + fn munmap(start: Address, size: usize) -> Result<()>; + + /// Change the protection of a memory region to the specified protection. + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()>; + + /// Checks if the memory has already been mapped. If not, we panic. + /// + /// Note that the checking may have a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped. + /// Be very careful about using this function. + /// + /// Fallback: As the function is only used for assertions, it can be a no-op, and MMTk will still run and never panics in this function. + fn panic_if_unmapped(start: Address, size: usize); + + /// Get the total memory of the system in bytes. + fn get_system_total_memory() -> Result { + use sysinfo::MemoryRefreshKind; + use sysinfo::{RefreshKind, System}; + + // TODO: Note that if we want to get system info somewhere else in the future, we should + // refactor this instance into some global struct. sysinfo recommends sharing one instance of + // `System` instead of making multiple instances. + // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info + // + // If we refactor the `System` instance to use it for other purposes, please make sure start-up + // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using + // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to + // start-up time. During start-up, MMTk core only needs the total memory to initialize the + // `Options`. If we only load memory-related components on start-up, it should only take <1ms + // to initialize the `System` instance. + let sys = System::new_with_specifics( + RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), + ); + Ok(sys.total_memory()) + } +} + +/// Strategy for performing mmap +#[derive(Debug, Copy, Clone)] +pub struct MmapStrategy { + /// Whether we should use huge page for this mmapping. + /// Fallback: for platforms that do not support huge pages, this option can be ignored. + pub huge_page: HugePageSupport, + /// The protection flags for mmap. + pub prot: MmapProtection, + /// Whether this mmap allows replacing existing mappings. + /// Fallback: for platforms that cannot replace existing mappings, or always replace existing mappings, this option can be ignored. + pub replace: bool, + /// Whether this mmap allows reserve/commit physical memory. + /// This has to be implemented properly for a platform. Otherwise, we will see huge unrealistic memory consumption. + pub reserve: bool, +} + +impl std::default::Default for MmapStrategy { + fn default() -> Self { + Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + } + } +} + +impl MmapStrategy { + /// Create a new strategy + pub fn new( + huge_page: HugePageSupport, + prot: MmapProtection, + replace: bool, + reserve: bool, + ) -> Self { + Self { + huge_page, + prot, + replace, + reserve, + } + } + + // Builder methods + + /// Set huge page option. + pub fn huge_page(self, huge_page: HugePageSupport) -> Self { + Self { huge_page, ..self } + } + + /// Set huge page option by a boolean flag. + pub fn transparent_hugepages(self, enable: bool) -> Self { + let huge_page = if enable { + HugePageSupport::TransparentHugePages + } else { + HugePageSupport::No + }; + Self { huge_page, ..self } + } + + /// Set protection option. + pub fn prot(self, prot: MmapProtection) -> Self { + Self { prot, ..self } + } + + /// Set the replace flag. + pub fn replace(self, replace: bool) -> Self { + Self { replace, ..self } + } + + /// Set the reserve flag. + pub fn reserve(self, reserve: bool) -> Self { + Self { reserve, ..self } + } + + #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings. + /// The strategy for MMTk's own internal memory (test) + pub const INTERNAL_MEMORY: Self = Self::TEST; + #[cfg(not(test))] + /// The strategy for MMTk's own internal memory + pub const INTERNAL_MEMORY: Self = Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + + /// The strategy for MMTk's test memory + #[cfg(test)] + pub const TEST: Self = Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: true, + reserve: true, + }; +} + +/// The protection flags for Mmap +#[repr(i32)] +#[derive(Debug, Copy, Clone)] +pub enum MmapProtection { + /// Allow read + write + ReadWrite, + /// Allow read + write + code execution + ReadWriteExec, + /// Do not allow any access + NoAccess, +} + +/// Support for huge pages +#[repr(u8)] +#[derive(Debug, Copy, Clone, NoUninit)] +pub enum HugePageSupport { + /// No support for huge page + No, + /// Enable transparent huge pages for the pages that are mapped. This option is only for linux. + TransparentHugePages, +} + +/// Annotation for an mmap entry. +/// +/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed` +/// require an annotation that indicates the purpose of the memory mapping. +/// +/// This is for debugging. On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the +/// human-readable name for the given mmap region. The annotation is ignored on other platforms. +/// +/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous +/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but +/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is +/// called for a chunk for the first time, which reflects which space first attempted the mmap, not +/// which space is currently using the chunk. Use `crate::policy::space::print_vm_map` to print a +/// more accurate mapping between address ranges and spaces. +/// +/// On 32-bit architecture, side metadata are allocated in a chunked fasion. One single `mmap` +/// region will contain many different metadata. In that case, we simply annotate the whole region +/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`. +pub enum MmapAnnotation<'a> { + /// The mmap is for a space. + Space { + /// The name of the space. + name: &'a str, + }, + /// The mmap is for a side metadata. + SideMeta { + /// The name of the space. + space: &'a str, + /// The name of the side metadata. + meta: &'a str, + }, + /// The mmap is for a test case. Usually constructed using the [`mmap_anno_test!`] macro. + Test { + /// The source file. + file: &'a str, + /// The line number. + line: u32, + }, + /// For all other use cases. + Misc { + /// A human-readable descriptive name. + name: &'a str, + }, +} + +/// Construct an `MmapAnnotation::Test` with the current file name and line number. +#[macro_export] +macro_rules! mmap_anno_test { + () => { + &$crate::util::os::MmapAnnotation::Test { + file: file!(), + line: line!(), + } + }; +} + +// Export this to external crates +pub use mmap_anno_test; + +impl std::fmt::Display for MmapAnnotation<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"), + MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"), + MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"), + MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"), + } + } +} diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs new file mode 100644 index 0000000000..328186d788 --- /dev/null +++ b/src/util/os/mod.rs @@ -0,0 +1,17 @@ +//! Operating System abstractions for MMTk. + +// Note: +// 1. For functions that return `Result`, an error value should only be used for exceptional cases. If a function returns +// a placeholder value, that should not be considered as 'exceptional cases', and should return Ok. +// 2. Some functions or arguments (e.g. [`crate::util::os::memory::MmapStrategy`]) allow fallback behaviors for platforms where certain features +// are not supported, or unimplemented. + +mod memory; +pub use memory::*; +mod process; +pub use process::*; + +mod imp; +pub use imp::OS; + +trait OperatingSystem: OSMemory + OSProcess {} diff --git a/src/util/os/process.rs b/src/util/os/process.rs new file mode 100644 index 0000000000..da2f78fc93 --- /dev/null +++ b/src/util/os/process.rs @@ -0,0 +1,36 @@ +use std::fmt::Display; +use std::io::Result; + +/// Representation of a CPU core identifier. +pub type CoreId = u16; +/// Representation of number of CPU cores. +pub type CoreNum = u16; + +/// Abstraction for OS process operations. +pub trait OSProcess { + /// The process ID type for the OS. + type ProcessIDType: Display + Eq + Copy; + /// The thread ID type for the OS. + type ThreadIDType: Display + Eq + Copy; + + /// Get the memory maps for the process. The returned string is a multi-line string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. + fn get_process_memory_maps() -> Result; + + /// Get the process ID as a string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. + fn get_process_id() -> Result; + + //// Get the thread ID as a string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. + fn get_thread_id() -> Result; + + /// Return the total number of cores allocated to the program. + fn get_total_num_cpus() -> CoreNum; + + /// Bind the current thread to the specified core. + fn bind_current_thread_to_core(core_id: CoreId); + + /// Bind the current thread to the specified core set. + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]); +} diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 23f6844ed7..b0d51803e7 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -1,9 +1,8 @@ use super::freelist::*; -use super::memory::MmapStrategy; use crate::util::address::Address; use crate::util::constants::*; use crate::util::conversions; -use crate::util::memory::MmapAnnotation; +use crate::util::os::*; /** log2 of the number of bits used by a free list entry (two entries per unit) */ const LOG_ENTRY_BITS: usize = LOG_BITS_IN_INT as _; @@ -199,7 +198,7 @@ impl RawMemoryFreeList { } fn mmap(&self, start: Address, bytes: usize) { - let res = super::memory::dzmmap_noreplace( + let res = OS::dzmmap( start, bytes, self.strategy, @@ -222,9 +221,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - unsafe { - ::libc::munmap(self.base.as_usize() as _, len); - } + let _ = OS::munmap(self.base, len); } } } diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index f31f68b5ec..00a972ac2e 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -108,19 +108,12 @@ unsafe impl Sync for InitializeOnce {} /// Create a formatted string that makes the best effort idenfying the current process and thread. pub fn debug_process_thread_id() -> String { - let pid = unsafe { libc::getpid() }; - #[cfg(target_os = "linux")] - { - // `gettid()` is Linux-specific. - let tid = unsafe { libc::gettid() }; - format!("PID: {}, TID: {}", pid, tid) - } - #[cfg(not(target_os = "linux"))] - { - // TODO: When we support other platforms, use platform-specific methods to get thread - // identifiers. - format!("PID: {}", pid) - } + use crate::util::os::*; + format!( + "PID: {}, TID: {}", + OS::get_process_id().map_or("(Failed to get PID)".to_string(), |pid| format!("{}", pid)), + OS::get_thread_id().map_or("(Failed to get TID)".to_string(), |tid| format!("{}", tid)), + ) } #[cfg(test)] diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs index 1dc6a5759e..f343a8d9e2 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs @@ -1,7 +1,7 @@ use super::mock_test_prelude::*; -use crate::util::memory; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::Address; #[test] @@ -11,23 +11,21 @@ pub fn test_handle_mmap_conflict() { || { let start = unsafe { Address::from_usize(0x100_0000) }; let one_megabyte = 1000000; - let mmap1_res = memory::dzmmap_noreplace( - start, - one_megabyte, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), - ); + let mmap1_res = OS::dzmmap(start, one_megabyte, MmapStrategy::TEST, mmap_anno_test!()); assert!(mmap1_res.is_ok()); let panic_res = std::panic::catch_unwind(|| { - let mmap2_res = memory::dzmmap_noreplace( + let mmap2_res = OS::dzmmap( start, one_megabyte, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), + MmapStrategy { + replace: false, + ..MmapStrategy::TEST + }, + mmap_anno_test!(), ); assert!(mmap2_res.is_err()); - memory::handle_mmap_error::( + OS::handle_mmap_error::( mmap2_res.err().unwrap(), VMThread::UNINITIALIZED, start, diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs index b810c47a43..e47b2a44b2 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs @@ -1,7 +1,7 @@ use super::mock_test_prelude::*; -use crate::util::memory; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::Address; #[cfg(target_pointer_width = "32")] @@ -18,14 +18,9 @@ pub fn test_handle_mmap_oom() { let start = unsafe { Address::from_usize(0x100_0000) }; // mmap 1 terabyte memory - we expect this will fail due to out of memory. // If that's not the case, increase the size we mmap. - let mmap_res = memory::dzmmap_noreplace( - start, - LARGE_SIZE, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), - ); + let mmap_res = OS::dzmmap(start, LARGE_SIZE, MmapStrategy::TEST, mmap_anno_test!()); - memory::handle_mmap_error::( + OS::handle_mmap_error::( mmap_res.err().unwrap(), VMThread::UNINITIALIZED, start, diff --git a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs index 1e386bc0d1..003575dc6e 100644 --- a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs +++ b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs @@ -19,6 +19,7 @@ fn test_malloc() { assert!(!bool1); assert!(!bool2); + assert!(bool3); assert!(bool4);