From 7895c882690d96b2b30eed37e585aec139c1e401 Mon Sep 17 00:00:00 2001 From: sepcnt <30561671+sepcnt@users.noreply.github.com> Date: Wed, 26 Nov 2025 14:23:52 +0800 Subject: [PATCH 01/27] feat: add Windows platform support --- .github/scripts/ci-common.sh | 12 +- .github/workflows/minimal-tests-core.yml | 12 + Cargo.toml | 9 + src/policy/copyspace.rs | 17 +- src/scheduler/affinity.rs | 42 ++- src/util/malloc/library.rs | 54 +++- src/util/malloc/malloc_ms_util.rs | 57 ++-- src/util/memory.rs | 291 ++++++++++++++---- src/util/raw_memory_freelist.rs | 4 +- src/util/rust_util/mod.rs | 11 +- src/util/test_util/mod.rs | 3 + .../tests/mock_tests/mock_test_malloc_ms.rs | 25 +- 12 files changed, 432 insertions(+), 105 deletions(-) diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index 35e450bdbd..287d1c980b 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -67,7 +67,12 @@ init_non_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - features[i]=$(echo $feature) + feature_name=$(echo $feature) + # jemalloc does not support Windows + if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then + continue + fi + features[i]=$feature_name let "i++" fi fi @@ -118,6 +123,11 @@ init_exclusive_features() { if [[ ! -z "$feature" ]]; then # Trim whitespaces features[i]=$(echo $feature) + # jemalloc does not support Windows + if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then + continue + fi + features[i]=$feature_name let "i++" fi fi diff --git a/.github/workflows/minimal-tests-core.yml b/.github/workflows/minimal-tests-core.yml index 2a113ca850..4bb8a77367 100644 --- a/.github/workflows/minimal-tests-core.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -37,11 +37,16 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } + - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} + defaults: + run: + shell: bash + env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" @@ -69,6 +74,7 @@ jobs: # Setup Environments - name: Setup Environments + if: runner.os != 'Windows' run: ./.github/scripts/ci-setup-${{ matrix.target.triple }}.sh # Build @@ -88,11 +94,16 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } + - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: style-check/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} + defaults: + run: + shell: bash + env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" @@ -120,6 +131,7 @@ jobs: # Setup Environments - name: Setup Environments + if: runner.os != 'Windows' run: ./.github/scripts/ci-setup-${{ matrix.target.triple }}.sh # Style checks diff --git a/Cargo.toml b/Cargo.toml index 5ca7a0358e..e244adb175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,15 @@ strum = "0.27.1" strum_macros = "0.27.1" sysinfo = "0.33.1" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = [ + "Win32_Foundation", + "Win32_System_Memory", + "Win32_System_SystemInformation", + "Win32_System_Threading", + "Win32_System_Diagnostics_Debug", +] } + [dev-dependencies] paste = "1.0.8" rand = "0.9.0" diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 4f0564b9e5..6505fd2e60 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -13,7 +13,6 @@ use crate::util::object_forwarding; use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; -use libc::{mprotect, PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -293,8 +292,8 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect(start.to_mut_ptr(), extent, PROT_NONE); + if let Err(e) = crate::util::memory::mprotect(start, extent) { + panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); } @@ -308,12 +307,12 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - unsafe { - mprotect( - start.to_mut_ptr(), - extent, - PROT_READ | PROT_WRITE | PROT_EXEC, - ); + if let Err(e) = crate::util::memory::munprotect( + start, + extent, + crate::util::memory::MmapProtection::ReadWriteExec, + ) { + panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); } diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index 331ec330b8..819489c128 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -20,7 +20,17 @@ pub fn get_total_num_cpus() -> u16 { } } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "windows")] +/// Return the total number of cores allocated to the program. +pub fn get_total_num_cpus() -> u16 { + unsafe { + windows_sys::Win32::System::Threading::GetActiveProcessorCount( + windows_sys::Win32::System::Threading::ALL_PROCESSOR_GROUPS, + ) as u16 + } +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] /// Return the total number of cores allocated to the program. pub fn get_total_num_cpus() -> u16 { unimplemented!() @@ -59,7 +69,18 @@ fn bind_current_thread_to_core(cpu: CoreId) { } } -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "windows")] +/// Bind the current thread to the specified core. +fn bind_current_thread_to_core(cpu: CoreId) { + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + 1 << cpu, + ); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "windows")))] /// Bind the current thread to the specified core. fn bind_current_thread_to_core(_cpu: CoreId) { unimplemented!() @@ -79,7 +100,22 @@ fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { } } -#[cfg(not(any(target_os = "linux", target_os = "android")))] +#[cfg(target_os = "windows")] +/// Bind the current thread to the specified core. +fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { + let mut mask = 0; + for cpu in cpuset { + mask |= 1 << cpu; + } + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + mask, + ); + } +} + +#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "windows")))] /// Bind the current thread to the specified core. fn bind_current_thread_to_cpuset(_cpuset: &[CoreId]) { unimplemented!() diff --git a/src/util/malloc/library.rs b/src/util/malloc/library.rs index 2177a06144..e716502a04 100644 --- a/src/util/malloc/library.rs +++ b/src/util/malloc/library.rs @@ -2,10 +2,18 @@ #[cfg(feature = "malloc_jemalloc")] pub use self::jemalloc::*; -#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] pub use self::libc_malloc::*; #[cfg(feature = "malloc_mimalloc")] pub use self::mimalloc::*; +#[cfg(all( + target_os = "windows", + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] +pub use self::win_malloc::*; /// When we count page usage of library malloc, we assume they allocate in pages. For some malloc implementations, /// they may use a larger page (e.g. mimalloc's 64K page). For libraries that we are not sure, we assume they use @@ -43,7 +51,10 @@ mod mimalloc { } /// If no malloc lib is specified, use the libc implementation -#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] mod libc_malloc { // Normal 4K page pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; @@ -61,3 +72,42 @@ mod libc_malloc { #[cfg(target_os = "macos")] pub use self::malloc_size as malloc_usable_size; } + +/// Windows malloc implementation using HeapAlloc +#[cfg(all( + target_os = "windows", + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] +mod win_malloc { + // Normal 4K page + pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; + + use std::ffi::c_void; + use windows_sys::Win32::System::Memory::*; + + pub unsafe fn malloc(size: usize) -> *mut c_void { + HeapAlloc(GetProcessHeap(), 0, size) + } + + pub unsafe fn free(ptr: *mut c_void) { + if !ptr.is_null() { + HeapFree(GetProcessHeap(), 0, ptr); + } + } + + pub unsafe fn calloc(nmemb: usize, size: usize) -> *mut c_void { + let total = nmemb * size; + HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, total) + } + + pub unsafe fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { + if ptr.is_null() { + return malloc(size); + } + HeapReAlloc(GetProcessHeap(), 0, ptr, size) + } + + pub unsafe fn malloc_usable_size(ptr: *const c_void) -> usize { + HeapSize(GetProcessHeap(), 0, ptr) + } +} diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 5d27cf79cb..5c08426fe7 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -4,6 +4,10 @@ use crate::util::Address; use crate::vm::VMBinding; /// Allocate with alignment. This also guarantees the memory is zero initialized. +#[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +))] pub fn align_alloc(size: usize, align: usize) -> Address { let mut ptr = std::ptr::null_mut::(); let ptr_ptr = std::ptr::addr_of_mut!(ptr); @@ -71,28 +75,37 @@ pub fn alloc(size: usize, align: usize, offset: usize) -> (Addres let mut is_offset_malloc = false; // malloc returns 16 bytes aligned address. // So if the alignment is smaller than 16 bytes, we do not need to align. - if align <= 16 && offset == 0 { - let raw = unsafe { calloc(1, size) }; - address = Address::from_mut_ptr(raw); - debug_assert!(address.is_aligned_to(align)); - } else if align > 16 && offset == 0 { - address = align_alloc(size, align); - debug_assert!( - address.is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {}", - address, - align - ); - } else { - address = align_offset_alloc::(size, align, offset); - is_offset_malloc = true; - debug_assert!( - (address + offset).is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {} at offset: {}", - address, - align, - offset - ); + + match (align, offset) { + (a, 0) if a <= 16 => { + let raw = unsafe { calloc(1, size) }; + address = Address::from_mut_ptr(raw); + debug_assert!(address.is_aligned_to(align)); + } + #[cfg(all( + not(target_os = "windows"), + not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) + ))] + (a, 0) if a > 16 => { + address = align_alloc(size, align); + debug_assert!( + address.is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {}", + address, + align + ); + } + _ => { + address = align_offset_alloc::(size, align, offset); + is_offset_malloc = true; + debug_assert!( + (address + offset).is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {} at offset: {}", + address, + align, + offset + ); + } } (address, is_offset_malloc) } diff --git a/src/util/memory.rs b/src/util/memory.rs index 9a9d4d16dc..6e877087cb 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -3,6 +3,7 @@ use crate::util::opaque_pointer::*; use crate::util::Address; use crate::vm::{Collection, VMBinding}; use bytemuck::NoUninit; +#[cfg(not(target_os = "windows"))] use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; use std::io::{Error, Result}; use sysinfo::MemoryRefreshKind; @@ -14,6 +15,10 @@ const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_F #[cfg(target_os = "macos")] // MAP_FIXED is used instead of MAP_FIXED_NOREPLACE (which is not available on macOS). We are at the risk of overwriting pre-existing mappings. const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; +#[cfg(target_os = "windows")] +const MMAP_FLAGS: libc::c_int = 0; // Not used on Windows +#[cfg(target_os = "windows")] +const MAP_NORESERVE: libc::c_int = 0x4000; // Custom flag for Windows emulation /// Strategy for performing mmap #[derive(Debug, Copy, Clone)] @@ -64,7 +69,8 @@ pub enum MmapProtection { } impl MmapProtection { - /// Turn the protection enum into the native flags + /// Turn the protection enum into the native flags on non-Windows platforms + #[cfg(not(target_os = "windows"))] pub fn into_native_flags(self) -> libc::c_int { match self { Self::ReadWrite => PROT_READ | PROT_WRITE, @@ -72,6 +78,17 @@ impl MmapProtection { Self::NoAccess => PROT_NONE, } } + + /// Turn the protection enum into the native flags on Windows platforms + #[cfg(target_os = "windows")] + pub fn into_native_flags(self) -> u32 { + use windows_sys::Win32::System::Memory::*; + match self { + Self::ReadWrite => PAGE_READWRITE, + Self::ReadWriteExec => PAGE_EXECUTE_READWRITE, + Self::NoAccess => PAGE_NOACCESS, + } + } } /// Support for huge pages @@ -157,10 +174,19 @@ impl std::fmt::Display for MmapAnnotation<'_> { /// Check the result from an mmap function in this module. /// Return true if the mmap has failed due to an existing conflicting mapping. pub(crate) fn result_is_mapped(result: Result<()>) -> bool { + #[cfg(not(target_os = "windows"))] match result { Ok(_) => false, Err(err) => err.raw_os_error().unwrap() == libc::EEXIST, } + #[cfg(target_os = "windows")] + match result { + Ok(_) => false, + Err(err) => { + // ERROR_INVALID_ADDRESS may be returned if the address is already mapped or invalid + err.raw_os_error().unwrap() == 487 // ERROR_INVALID_ADDRESS + } + } } /// Set a range of memory to 0. @@ -191,10 +217,14 @@ pub unsafe fn dzmmap( strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { + #[cfg(not(target_os = "windows"))] let flags = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; + #[cfg(target_os = "windows")] + let flags = 0; // Not used let ret = mmap_fixed(start, size, flags, strategy, anno); // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] + // On Windows, MEM_COMMIT guarantees zero-initialized pages. + #[cfg(not(any(target_os = "linux", target_os = "windows")))] if ret.is_ok() { zero(start, size) } @@ -210,10 +240,15 @@ pub fn dzmmap_noreplace( strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { + #[cfg(not(target_os = "windows"))] let flags = MMAP_FLAGS; + #[cfg(target_os = "windows")] + let flags = 0; // Not used + let ret = mmap_fixed(start, size, flags, strategy, anno); // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - #[cfg(not(target_os = "linux"))] + // On Windows, MEM_COMMIT guarantees zero-initialized pages. + #[cfg(not(any(target_os = "linux", target_os = "windows")))] if ret.is_ok() { zero(start, size) } @@ -231,7 +266,10 @@ pub fn mmap_noreserve( anno: &MmapAnnotation, ) -> Result<()> { strategy.prot = MmapProtection::NoAccess; + #[cfg(not(target_os = "windows"))] let flags = MMAP_FLAGS | libc::MAP_NORESERVE; + #[cfg(target_os = "windows")] + let flags = MAP_NORESERVE; mmap_fixed(start, size, flags, strategy, anno) } @@ -242,64 +280,148 @@ fn mmap_fixed( strategy: MmapStrategy, _anno: &MmapAnnotation, ) -> Result<()> { - let ptr = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, - ptr, - )?; - - #[cfg(all( - any(target_os = "linux", target_os = "android"), - not(feature = "no_mmap_annotation") - ))] + #[cfg(not(target_os = "windows"))] { - // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that - // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not - // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` - // for other reasons, too. That includes `start` being an invalid address, and the - // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But - // since this prctl is used for debugging, we log the error instead of panicking. - let anno_str = _anno.to_string(); - let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( - &|| unsafe { - libc::prctl( - libc::PR_SET_VMA, - libc::PR_SET_VMA_ANON_NAME, - start.to_ptr::(), - size, - anno_cstr.as_ptr(), - ) - }, - 0, - ); - if let Err(e) = result { - debug!("Error while calling prctl: {e}"); + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.into_native_flags(); + wrap_libc_call( + &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, + ptr, + )?; + + #[cfg(all( + any(target_os = "linux", target_os = "android"), + not(feature = "no_mmap_annotation") + ))] + { + // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that + // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not + // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` + // for other reasons, too. That includes `start` being an invalid address, and the + // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But + // since this prctl is used for debugging, we log the error instead of panicking. + let anno_str = _anno.to_string(); + let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); + let result = wrap_libc_call( + &|| unsafe { + libc::prctl( + libc::PR_SET_VMA, + libc::PR_SET_VMA_ANON_NAME, + start.to_ptr::(), + size, + anno_cstr.as_ptr(), + ) + }, + 0, + ); + if let Err(e) = result { + debug!("Error while calling prctl: {e}"); + } + } + + match strategy.huge_page { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => { + #[cfg(target_os = "linux")] + { + wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ) + } + // Setting the transparent hugepage option to true will not pass + // the validation on non-Linux OSes + #[cfg(not(target_os = "linux"))] + unreachable!() + } } } - match strategy.huge_page { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - #[cfg(target_os = "linux")] - { - wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.into_native_flags(); + let commit = + (flags & MAP_NORESERVE) == 0 && !matches!(strategy.prot, MmapProtection::NoAccess); + + // Query the state of the memory region + let mut mbi: MEMORY_BASIC_INFORMATION = unsafe { std::mem::zeroed() }; + let query_res = unsafe { + VirtualQuery( + ptr, + &mut mbi, + std::mem::size_of::(), + ) + }; + if query_res == 0 { + return Err(std::io::Error::last_os_error()); + } + + let res = unsafe { + match mbi.State { + MEM_FREE => { + let mut allocation_type = MEM_RESERVE; + if commit { + allocation_type |= MEM_COMMIT; + } + VirtualAlloc(ptr, size, allocation_type, prot) + } + MEM_RESERVE => { + if commit { + VirtualAlloc(ptr, size, MEM_COMMIT, prot) + } else { + // Already reserved, nothing to do + ptr + } + } + MEM_COMMIT => { + // Already committed, just return success. + // We could change protection here if needed, but VirtualAlloc returns success + // if the memory is already committed. + ptr + } + _ => { + // Should not happen + return Err(std::io::Error::other("Unexpected memory state")); + } } - // Setting the transparent hugepage option to true will not pass - // the validation on non-Linux OSes - #[cfg(not(target_os = "linux"))] - unreachable!() + }; + + if res.is_null() { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) } } } /// Unmap the given memory (in page granularity). This wraps the unsafe libc munmap call. pub fn munmap(start: Address, size: usize) -> Result<()> { - wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) + #[cfg(not(target_os = "windows"))] + return wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0); + + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + // Using MEM_DECOMMIT will decommit the memory but leave the address space reserved. + // This is the safest way to emulate munmap on Windows, as MEM_RELEASE would free + // the entire allocation, which could be larger than the requested size. + let res = unsafe { VirtualFree(start.to_mut_ptr(), size, MEM_DECOMMIT) }; + if res == 0 { + // If decommit fails, we try to release the memory. This might happen if the memory was + // only reserved. + let res_release = unsafe { VirtualFree(start.to_mut_ptr(), 0, MEM_RELEASE) }; + if res_release == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } else { + Ok(()) + } + } } /// Properly handle errors from a mmap Result, including invoking the binding code in the case of @@ -329,18 +451,37 @@ pub fn handle_mmap_error( // further check the error if let Some(os_errno) = error.raw_os_error() { // If it is OOM, we invoke out_of_memory() through the VM interface. + #[cfg(not(target_os = "windows"))] if os_errno == libc::ENOMEM { // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. trace!("Signal MmapOutOfMemory!"); VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); unreachable!() } + #[cfg(target_os = "windows")] + if os_errno == 8 { + // ERROR_NOT_ENOUGH_MEMORY + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + } } } ErrorKind::AlreadyExists => { panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); } - _ => {} + _ => { + #[cfg(target_os = "windows")] + if let Some(os_errno) = error.raw_os_error() { + // If it is invalid address, we provide a more specific panic message. + if os_errno == 487 { + // ERROR_INVALID_ADDRESS + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + } + } + } } panic!("Unexpected mmap failure: {:?}", error) } @@ -380,19 +521,49 @@ pub(crate) fn panic_if_unmapped(_start: Address, _size: usize, _anno: &MmapAnnot /// Unprotect the given memory (in page granularity) to allow access (PROT_READ/WRITE/EXEC). pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - let prot = prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) + #[cfg(not(target_os = "windows"))] + { + let prot = prot.into_native_flags(); + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, + 0, + ) + } + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + let prot = prot.into_native_flags(); + let mut old_protect = 0; + let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } } /// Protect the given memory (in page granularity) to forbid any access (PROT_NONE). pub fn mprotect(start: Address, size: usize) -> Result<()> { - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, - 0, - ) + #[cfg(not(target_os = "windows"))] + { + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, + 0, + ) + } + #[cfg(target_os = "windows")] + { + use windows_sys::Win32::System::Memory::*; + let mut old_protect = 0; + let res = + unsafe { VirtualProtect(start.to_mut_ptr(), size, PAGE_NOACCESS, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } } fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 23f6844ed7..f02e6ad971 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -222,9 +222,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - unsafe { - ::libc::munmap(self.base.as_usize() as _, len); - } + let _ = crate::util::memory::munmap(self.base, len); } } } diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index f31f68b5ec..10f57c560b 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -108,15 +108,22 @@ unsafe impl Sync for InitializeOnce {} /// Create a formatted string that makes the best effort idenfying the current process and thread. pub fn debug_process_thread_id() -> String { - let pid = unsafe { libc::getpid() }; #[cfg(target_os = "linux")] { + let pid = unsafe { libc::getpid() }; // `gettid()` is Linux-specific. let tid = unsafe { libc::gettid() }; format!("PID: {}, TID: {}", pid, tid) } - #[cfg(not(target_os = "linux"))] + #[cfg(target_os = "windows")] { + let pid = unsafe { windows_sys::Win32::System::Threading::GetCurrentProcessId() }; + let tid = unsafe { windows_sys::Win32::System::Threading::GetCurrentThreadId() }; + format!("PID: {}, TID: {}", pid, tid) + } + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + { + let pid = unsafe { libc::getpid() }; // TODO: When we support other platforms, use platform-specific methods to get thread // identifiers. format!("PID: {}", pid) diff --git a/src/util/test_util/mod.rs b/src/util/test_util/mod.rs index 4540cee3b8..be71905b97 100644 --- a/src/util/test_util/mod.rs +++ b/src/util/test_util/mod.rs @@ -53,6 +53,9 @@ const TEST_ADDRESS: Address = #[cfg(target_os = "macos")] const TEST_ADDRESS: Address = crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x2_0000_0000) }); +#[cfg(target_os = "windows")] +const TEST_ADDRESS: Address = + crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x5_0000_0000) }); // util::heap::layout::mmapper::csm pub(crate) const CHUNK_STATE_MMAPPER_TEST_REGION: MmapTestRegion = diff --git a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs index 1e386bc0d1..e9c9d28b1e 100644 --- a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs +++ b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs @@ -18,7 +18,21 @@ fn test_malloc() { assert!((address4 + 4_isize).is_aligned_to(64)); assert!(!bool1); - assert!(!bool2); + + // Since Windows HeapAlloc only guarantees 16-byte alignment, the allocation with 32-byte alignment + // without offset will be treated as an offset allocation. + if cfg!(all( + not(target_os = "windows"), + not(any( + feature = "malloc_jemalloc", + feature = "malloc_mimalloc" + )) + )) { + assert!(!bool2); + } else { + assert!(bool2); + } + assert!(bool3); assert!(bool4); @@ -30,8 +44,13 @@ fn test_malloc() { unsafe { malloc_ms_util::free(address1.to_mut_ptr()); } - unsafe { - malloc_ms_util::free(address2.to_mut_ptr()); + + if !bool2 { + unsafe { + malloc_ms_util::free(address2.to_mut_ptr()); + } + } else { + malloc_ms_util::offset_free(address2); } malloc_ms_util::offset_free(address3); malloc_ms_util::offset_free(address4); From b11ef2131b430939a58f1650b93e2fa5ce5fbe4f Mon Sep 17 00:00:00 2001 From: sepcnt <30561671+sepcnt@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:06:38 +0800 Subject: [PATCH 02/27] fix: use constants, CI placeholder, detailed comment and better `mmap_fixed` --- .../ci-setup-x86_64-pc-windows-msvc.sh | 0 .github/workflows/minimal-tests-core.yml | 2 - src/util/malloc/malloc_ms_util.rs | 6 + src/util/memory.rs | 130 ++++++++++++------ 4 files changed, 91 insertions(+), 47 deletions(-) create mode 100644 .github/scripts/ci-setup-x86_64-pc-windows-msvc.sh diff --git a/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh b/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/minimal-tests-core.yml b/.github/workflows/minimal-tests-core.yml index 4bb8a77367..69edf67362 100644 --- a/.github/workflows/minimal-tests-core.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -74,7 +74,6 @@ jobs: # Setup Environments - name: Setup Environments - if: runner.os != 'Windows' run: ./.github/scripts/ci-setup-${{ matrix.target.triple }}.sh # Build @@ -131,7 +130,6 @@ jobs: # Setup Environments - name: Setup Environments - if: runner.os != 'Windows' run: ./.github/scripts/ci-setup-${{ matrix.target.triple }}.sh # Style checks diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 5c08426fe7..19400a485b 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -4,6 +4,8 @@ use crate::util::Address; use crate::vm::VMBinding; /// Allocate with alignment. This also guarantees the memory is zero initialized. +/// This uses posix_memalign, which is not available on Windows. +/// This would somehow affect `MallocMarkSweep` performance on Windows. #[cfg(all( not(target_os = "windows"), not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) @@ -86,6 +88,10 @@ pub fn alloc(size: usize, align: usize, offset: usize) -> (Addres not(target_os = "windows"), not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) ))] + // On non-Windows platforms with posix_memalign, we can use align_alloc for alignments > 16 + // However, on Windows, there is no equivalent function. + // The memory alloc by `align_alloc` may not be freed correctly by `free`. + // So we use offset allocation for all alignments > 16 on Windows. (a, 0) if a > 16 => { address = align_alloc(size, align); debug_assert!( diff --git a/src/util/memory.rs b/src/util/memory.rs index 6e877087cb..2e267ae17b 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -184,7 +184,8 @@ pub(crate) fn result_is_mapped(result: Result<()>) -> bool { Ok(_) => false, Err(err) => { // ERROR_INVALID_ADDRESS may be returned if the address is already mapped or invalid - err.raw_os_error().unwrap() == 487 // ERROR_INVALID_ADDRESS + err.raw_os_error().unwrap() + == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 } } } @@ -339,60 +340,99 @@ fn mmap_fixed( #[cfg(target_os = "windows")] { - use windows_sys::Win32::System::Memory::*; + use std::io; + use windows_sys::Win32::System::Memory::{ + VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, + }; - let ptr = start.to_mut_ptr(); + let ptr: *mut u8 = start.to_mut_ptr(); let prot = strategy.prot.into_native_flags(); + + // Has to COMMIT inmediately if: + // - not MAP_NORESERVE + // - and protection is not NoAccess let commit = (flags & MAP_NORESERVE) == 0 && !matches!(strategy.prot, MmapProtection::NoAccess); - // Query the state of the memory region - let mut mbi: MEMORY_BASIC_INFORMATION = unsafe { std::mem::zeroed() }; - let query_res = unsafe { - VirtualQuery( - ptr, - &mut mbi, - std::mem::size_of::(), - ) - }; - if query_res == 0 { - return Err(std::io::Error::last_os_error()); - } + // Scan the region [ptr, ptr + size) to understand its current state + unsafe { + let mut addr = ptr; + let end = ptr.add(size); + + let mut saw_free = false; + let mut saw_reserved = false; + let mut saw_committed = false; + + while addr < end { + let mut mbi: MEMORY_BASIC_INFORMATION = std::mem::zeroed(); + let q = VirtualQuery( + addr as *const _, + &mut mbi, + std::mem::size_of::(), + ); + if q == 0 { + return Err(io::Error::last_os_error()); + } - let res = unsafe { - match mbi.State { - MEM_FREE => { - let mut allocation_type = MEM_RESERVE; - if commit { - allocation_type |= MEM_COMMIT; + let region_base = mbi.BaseAddress as *mut u8; + let region_size = mbi.RegionSize; + let region_end = region_base.add(region_size); + + // Calculate the intersection of [addr, end) and [region_base, region_end) + let _sub_begin = if addr > region_base { + addr + } else { + region_base + }; + let _sub_end = if end < region_end { end } else { region_end }; + + match mbi.State { + MEM_FREE => saw_free = true, + MEM_RESERVE => saw_reserved = true, + MEM_COMMIT => saw_committed = true, + _ => { + return Err(io::Error::other("Unexpected memory state in mmap_fixed")); } - VirtualAlloc(ptr, size, allocation_type, prot) } - MEM_RESERVE => { - if commit { - VirtualAlloc(ptr, size, MEM_COMMIT, prot) - } else { - // Already reserved, nothing to do - ptr - } + + // Jump to the next region (VirtualQuery always returns "continuous regions with the same attributes") + addr = region_end; + } + + // 1. All FREE: make a new mapping in the region + // 2. All RESERVE/COMMIT: treat as an existing mapping, can just COMMIT or succeed directly + // 3. MIX of FREE + others: not allowed (semantically similar to MAP_FIXED_NOREPLACE) + if saw_free && (saw_reserved || saw_committed) { + return Err(io::Error::from_raw_os_error( + windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32, + )); + } + + if saw_free && !saw_reserved && !saw_committed { + // All FREE: make a new mapping in the region + let mut allocation_type = MEM_RESERVE; + if commit { + allocation_type |= MEM_COMMIT; } - MEM_COMMIT => { - // Already committed, just return success. - // We could change protection here if needed, but VirtualAlloc returns success - // if the memory is already committed. - ptr + + let res = VirtualAlloc(ptr as *mut _, size, allocation_type, prot); + if res.is_null() { + return Err(io::Error::last_os_error()); } - _ => { - // Should not happen - return Err(std::io::Error::other("Unexpected memory state")); + + Ok(()) + } else { + // This behavior is similar to mmap with MAP_FIXED on Linux. + // If the region is already mapped, we just ensure the required commitment. + // If commit is not needed, we just return Ok. + if commit { + let res = VirtualAlloc(ptr as *mut _, size, MEM_COMMIT, prot); + if res.is_null() { + return Err(io::Error::last_os_error()); + } } + Ok(()) } - }; - - if res.is_null() { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) } } } @@ -459,7 +499,7 @@ pub fn handle_mmap_error( unreachable!() } #[cfg(target_os = "windows")] - if os_errno == 8 { + if os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 { // ERROR_NOT_ENOUGH_MEMORY trace!("Signal MmapOutOfMemory!"); VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); @@ -474,7 +514,7 @@ pub fn handle_mmap_error( #[cfg(target_os = "windows")] if let Some(os_errno) = error.raw_os_error() { // If it is invalid address, we provide a more specific panic message. - if os_errno == 487 { + if os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 { // ERROR_INVALID_ADDRESS trace!("Signal MmapOutOfMemory!"); VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); From e7e5f74300a8dc2ffdc285f01a98fc41a750286e Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 1 Jan 2026 11:21:56 +1300 Subject: [PATCH 03/27] Add OS abstraction --- src/util/mod.rs | 2 + src/util/os/linux.rs | 27 +++++++ src/util/os/memory.rs | 153 ++++++++++++++++++++++++++++++++++++ src/util/os/mod.rs | 15 ++++ src/util/os/posix_common.rs | 68 ++++++++++++++++ src/util/os/process.rs | 6 ++ src/util/os/windows.rs | 27 +++++++ 7 files changed, 298 insertions(+) create mode 100644 src/util/os/linux.rs create mode 100644 src/util/os/memory.rs create mode 100644 src/util/os/mod.rs create mode 100644 src/util/os/posix_common.rs create mode 100644 src/util/os/process.rs create mode 100644 src/util/os/windows.rs diff --git a/src/util/mod.rs b/src/util/mod.rs index d22c29a2e3..a12a8ff6c7 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -36,6 +36,8 @@ pub mod metadata; pub mod opaque_pointer; /// MMTk command line options. pub mod options; +/// Operating system abstractions. +pub mod os; #[cfg(feature = "test_private")] pub mod test_private; /// Test utilities. We need this module for `MockVM` in criterion benches, which does not include code with `cfg(test)`. diff --git a/src/util/os/linux.rs b/src/util/os/linux.rs new file mode 100644 index 0000000000..d9d6474f40 --- /dev/null +++ b/src/util/os/linux.rs @@ -0,0 +1,27 @@ +use crate::util::os::memory::*; +use crate::util::address::Address; +use std::io::Result; + +pub struct LinuxMemory; + +impl Memory for LinuxMemory { + fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ + // Windows-specific implementation of dzmmap + unimplemented!() + } + + fn munmap(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of munmap + unimplemented!() + } + + fn mprotect(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of mprotect + unimplemented!() + } + + fn munprotect(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of munprotect + unimplemented!() + } +} diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs new file mode 100644 index 0000000000..7df144649a --- /dev/null +++ b/src/util/os/memory.rs @@ -0,0 +1,153 @@ +use bytemuck::NoUninit; +use std::io::{Error, Result}; + +use crate::util::address::Address; + +pub trait Memory { + fn zero(start: Address, len: usize) { + Self::set(start, 0, len); + } + fn set(start: Address, val: u8, len: usize) { + unsafe { + std::ptr::write_bytes::(start.to_mut_ptr(), val, len); + } + } + fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
; + fn munmap(start: Address, size: usize) -> Result<()>; + fn mprotect(start: Address, size: usize) -> Result<()>; + fn munprotect(start: Address, size: usize) -> Result<()>; +} + +/// Strategy for performing mmap +#[derive(Debug, Copy, Clone)] +pub struct MmapStrategy { + /// Do we support huge pages? + pub huge_page: HugePageSupport, + /// The protection flags for mmap + pub prot: MmapProtection, + pub replace: bool, + pub reserve: bool, +} + +impl MmapStrategy { + /// Create a new strategy + pub fn new(transparent_hugepages: bool, prot: MmapProtection, replace: bool, reserve: bool) -> Self { + Self { + huge_page: if transparent_hugepages { + HugePageSupport::TransparentHugePages + } else { + HugePageSupport::No + }, + prot, + replace, + reserve, + } + } + + /// The strategy for MMTk's own internal memory + pub const INTERNAL_MEMORY: Self = Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + + /// The strategy for MMTk side metadata + pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; + + /// The strategy for MMTk's test memory + #[cfg(test)] + pub const TEST: Self = Self::INTERNAL_MEMORY; +} + +/// The protection flags for Mmap +#[repr(i32)] +#[derive(Debug, Copy, Clone)] +pub enum MmapProtection { + /// Allow read + write + ReadWrite, + /// Allow read + write + code execution + ReadWriteExec, + /// Do not allow any access + NoAccess, +} + +/// Support for huge pages +#[repr(u8)] +#[derive(Debug, Copy, Clone, NoUninit)] +pub enum HugePageSupport { + /// No support for huge page + No, + /// Enable transparent huge pages for the pages that are mapped. This option is only for linux. + TransparentHugePages, +} + +/// Annotation for an mmap entry. +/// +/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed` +/// require an annotation that indicates the purpose of the memory mapping. +/// +/// This is for debugging. On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the +/// human-readable name for the given mmap region. The annotation is ignored on other platforms. +/// +/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous +/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but +/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is +/// called for a chunk for the first time, which reflects which space first attempted the mmap, not +/// which space is currently using the chunk. Use `crate::policy::space::print_vm_map` to print a +/// more accurate mapping between address ranges and spaces. +/// +/// On 32-bit architecture, side metadata are allocated in a chunked fasion. One single `mmap` +/// region will contain many different metadata. In that case, we simply annotate the whole region +/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`. +pub enum MmapAnnotation<'a> { + /// The mmap is for a space. + Space { + /// The name of the space. + name: &'a str, + }, + /// The mmap is for a side metadata. + SideMeta { + /// The name of the space. + space: &'a str, + /// The name of the side metadata. + meta: &'a str, + }, + /// The mmap is for a test case. Usually constructed using the [`mmap_anno_test!`] macro. + Test { + /// The source file. + file: &'a str, + /// The line number. + line: u32, + }, + /// For all other use cases. + Misc { + /// A human-readable descriptive name. + name: &'a str, + }, +} + +/// Construct an `MmapAnnotation::Test` with the current file name and line number. +#[macro_export] +macro_rules! mmap_anno_test_2 { + () => { + &$crate::util::memory::MmapAnnotation::Test { + file: file!(), + line: line!(), + } + }; +} + +// Export this to external crates +pub use mmap_anno_test_2; + +impl std::fmt::Display for MmapAnnotation<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"), + MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"), + MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"), + MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"), + } + } +} diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs new file mode 100644 index 0000000000..8669e4e546 --- /dev/null +++ b/src/util/os/mod.rs @@ -0,0 +1,15 @@ +pub mod memory; +pub mod process; + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] +pub(crate) mod posix_common; +#[cfg(target_os = "linux")] +pub(crate) mod linux; + +#[cfg(target_os = "windows")] +pub(crate) mod windows; + +pub trait OperatingSystem { + type OSMemory: memory::Memory; + type OSProcess: process::Process; +} diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs new file mode 100644 index 0000000000..9983121e8f --- /dev/null +++ b/src/util/os/posix_common.rs @@ -0,0 +1,68 @@ +use crate::util::os::memory::*; +use crate::util::address::Address; +use std::io::Result; + +pub fn posix_mmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.into_native_flags(); + wrap_libc_call( + &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, + ptr, + )?; + + #[cfg(all( + any(target_os = "linux", target_os = "android"), + not(feature = "no_mmap_annotation") + ))] + { + // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that + // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not + // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` + // for other reasons, too. That includes `start` being an invalid address, and the + // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But + // since this prctl is used for debugging, we log the error instead of panicking. + let anno_str = _anno.to_string(); + let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); + let result = wrap_libc_call( + &|| unsafe { + libc::prctl( + libc::PR_SET_VMA, + libc::PR_SET_VMA_ANON_NAME, + start.to_ptr::(), + size, + anno_cstr.as_ptr(), + ) + }, + 0, + ); + if let Err(e) = result { + debug!("Error while calling prctl: {e}"); + } + } + + match strategy.huge_page { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => { + #[cfg(target_os = "linux")] + { + wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ) + } + // Setting the transparent hugepage option to true will not pass + // the validation on non-Linux OSes + #[cfg(not(target_os = "linux"))] + unreachable!() + } + } +} + +fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { + let ret = f(); + if ret == expect { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} \ No newline at end of file diff --git a/src/util/os/process.rs b/src/util/os/process.rs new file mode 100644 index 0000000000..34fc26909d --- /dev/null +++ b/src/util/os/process.rs @@ -0,0 +1,6 @@ +use std::io::Result; + +pub trait Process { + fn get_process_memory_maps() -> Result; + fn get_system_total_memory() -> Result; +} diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs new file mode 100644 index 0000000000..cbcdecc538 --- /dev/null +++ b/src/util/os/windows.rs @@ -0,0 +1,27 @@ +use crate::util::os::memory::*; +use crate::util::address::Address; +use std::io::Result; + +pub struct WindowsMemory; + +impl Memory for WindowsMemory { + fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ + // Windows-specific implementation of dzmmap + unimplemented!() + } + + fn munmap(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of munmap + unimplemented!() + } + + fn mprotect(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of mprotect + unimplemented!() + } + + fn munprotect(start: Address, size: usize) -> Result<()> { + // Windows-specific implementation of munprotect + unimplemented!() + } +} From 7cfd2b30dd7edecf91b981841b920631da638bd4 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 1 Jan 2026 13:41:41 +1300 Subject: [PATCH 04/27] Implement OS interface for Windows --- src/util/os/memory.rs | 2 +- src/util/os/mod.rs | 3 + src/util/os/process.rs | 21 ++++- src/util/os/windows.rs | 174 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 186 insertions(+), 14 deletions(-) diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 7df144649a..9de27446cf 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -15,7 +15,7 @@ pub trait Memory { fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
; fn munmap(start: Address, size: usize) -> Result<()>; fn mprotect(start: Address, size: usize) -> Result<()>; - fn munprotect(start: Address, size: usize) -> Result<()>; + fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()>; } /// Strategy for performing mmap diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index 8669e4e546..d0cea730d4 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -9,6 +9,9 @@ pub(crate) mod linux; #[cfg(target_os = "windows")] pub(crate) mod windows; +#[cfg(target_os = "windows")] +pub use windows::Windows as OS; + pub trait OperatingSystem { type OSMemory: memory::Memory; type OSProcess: process::Process; diff --git a/src/util/os/process.rs b/src/util/os/process.rs index 34fc26909d..cd6f6d7719 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -2,5 +2,24 @@ use std::io::Result; pub trait Process { fn get_process_memory_maps() -> Result; - fn get_system_total_memory() -> Result; + fn get_system_total_memory() -> Result { + use sysinfo::MemoryRefreshKind; + use sysinfo::{RefreshKind, System}; + + // TODO: Note that if we want to get system info somewhere else in the future, we should + // refactor this instance into some global struct. sysinfo recommends sharing one instance of + // `System` instead of making multiple instances. + // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info + // + // If we refactor the `System` instance to use it for other purposes, please make sure start-up + // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using + // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to + // start-up time. During start-up, MMTk core only needs the total memory to initialize the + // `Options`. If we only load memory-related components on start-up, it should only take <1ms + // to initialize the `System` instance. + let sys = System::new_with_specifics( + RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), + ); + Ok(sys.total_memory()) + } } diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index cbcdecc538..1880259df8 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -1,27 +1,177 @@ -use crate::util::os::memory::*; +use crate::util::os::{OperatingSystem, memory::*}; use crate::util::address::Address; use std::io::Result; -pub struct WindowsMemory; +pub struct Windows; +impl OperatingSystem for Windows { + type OSMemory = WindowsMemoryImpl; + type OSProcess = WindowsProcessImpl; +} + +pub struct WindowsMemoryImpl; -impl Memory for WindowsMemory { +impl Memory for WindowsMemoryImpl { fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ - // Windows-specific implementation of dzmmap - unimplemented!() + use std::io; + use windows_sys::Win32::System::Memory::{ + VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, + }; + + let ptr: *mut u8 = start.to_mut_ptr(); + + // Has to COMMIT immediately if: + // - not MAP_NORESERVE + // - and protection is not NoAccess + let commit = strategy.reserve && !matches!(strategy.prot, MmapProtection::NoAccess); + + // Scan the region [ptr, ptr + size) to understand its current state + unsafe { + let mut addr = ptr; + let end = ptr.add(size); + + let mut saw_free = false; + let mut saw_reserved = false; + let mut saw_committed = false; + + while addr < end { + let mut mbi: MEMORY_BASIC_INFORMATION = std::mem::zeroed(); + let q = VirtualQuery( + addr as *const _, + &mut mbi, + std::mem::size_of::(), + ); + if q == 0 { + return Err(io::Error::last_os_error()); + } + + let region_base = mbi.BaseAddress as *mut u8; + let region_size = mbi.RegionSize; + let region_end = region_base.add(region_size); + + // Calculate the intersection of [addr, end) and [region_base, region_end) + let _sub_begin = if addr > region_base { + addr + } else { + region_base + }; + let _sub_end = if end < region_end { end } else { region_end }; + + match mbi.State { + MEM_FREE => saw_free = true, + MEM_RESERVE => saw_reserved = true, + MEM_COMMIT => saw_committed = true, + _ => { + return Err(io::Error::other("Unexpected memory state in mmap_fixed")); + } + } + + // Jump to the next region (VirtualQuery always returns "continuous regions with the same attributes") + addr = region_end; + } + + // 1. All FREE: make a new mapping in the region + // 2. All RESERVE/COMMIT: treat as an existing mapping, can just COMMIT or succeed directly + // 3. MIX of FREE + others: not allowed (semantically similar to MAP_FIXED_NOREPLACE) + if saw_free && (saw_reserved || saw_committed) { + return Err(io::Error::from_raw_os_error( + windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32, + )); + } + + if saw_free && !saw_reserved && !saw_committed { + // All FREE: make a new mapping in the region + let mut allocation_type = MEM_RESERVE; + if commit { + allocation_type |= MEM_COMMIT; + } + + let res = VirtualAlloc(ptr as *mut _, size, allocation_type, strategy.prot.into_native_flags()); + if res.is_null() { + return Err(io::Error::last_os_error()); + } + + Ok(start) + } else { + // This behavior is similar to mmap with MAP_FIXED on Linux. + // If the region is already mapped, we just ensure the required commitment. + // If commit is not needed, we just return Ok. + if commit { + let res = VirtualAlloc(ptr as *mut _, size, MEM_COMMIT, strategy.prot.into_native_flags()); + if res.is_null() { + return Err(io::Error::last_os_error()); + } + } + Ok(start) + } + } } fn munmap(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of munmap - unimplemented!() + use windows_sys::Win32::System::Memory::*; + // Using MEM_DECOMMIT will decommit the memory but leave the address space reserved. + // This is the safest way to emulate munmap on Windows, as MEM_RELEASE would free + // the entire allocation, which could be larger than the requested size. + let res = unsafe { VirtualFree(start.to_mut_ptr(), size, MEM_DECOMMIT) }; + if res == 0 { + // If decommit fails, we try to release the memory. This might happen if the memory was + // only reserved. + let res_release = unsafe { VirtualFree(start.to_mut_ptr(), 0, MEM_RELEASE) }; + if res_release == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } else { + Ok(()) + } } fn mprotect(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of mprotect - unimplemented!() + use windows_sys::Win32::System::Memory::*; + let mut old_protect = 0; + let res = + unsafe { VirtualProtect(start.to_mut_ptr(), size, PAGE_NOACCESS, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } } - fn munprotect(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of munprotect - unimplemented!() + fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + use windows_sys::Win32::System::Memory::*; + let prot = prot.into_native_flags(); + let mut old_protect = 0; + let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; + if res == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } + } +} + +impl MmapProtection { + fn into_native_flags(&self) -> u32 { + use windows_sys::Win32::System::Memory::*; + match self { + Self::ReadWrite => PAGE_READWRITE, + Self::ReadWriteExec => PAGE_EXECUTE_READWRITE, + Self::NoAccess => PAGE_NOACCESS, + } + } +} + +use crate::util::os::process::*; + +pub struct WindowsProcessImpl; + +impl Process for WindowsProcessImpl { + fn get_process_memory_maps() -> Result { + // Windows-specific implementation to get process memory maps + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "get_process_memory_maps not implemented for Windows", + )) } } From 0cba6819153ba67a4584843ef99c8f2af51ce932 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Sun, 4 Jan 2026 21:11:16 +1300 Subject: [PATCH 05/27] Able to build on Windows --- .github/scripts/ci-common.sh | 23 +++++++- src/plan/generational/gc_work.rs | 3 +- src/policy/copyspace.rs | 7 ++- src/policy/immix/block.rs | 5 +- src/policy/lockfreeimmortalspace.rs | 11 ++-- src/policy/markcompactspace.rs | 3 +- src/policy/space.rs | 58 +++++++++++++++++-- src/util/alloc/free_list_allocator.rs | 4 +- src/util/alloc/immix_allocator.rs | 3 +- src/util/heap/freelistpageresource.rs | 8 +-- src/util/heap/layout/map64.rs | 2 +- src/util/heap/layout/mmapper/csm/mod.rs | 18 +++--- src/util/heap/layout/mmapper/mod.rs | 2 +- src/util/malloc/malloc_ms_util.rs | 2 +- src/util/memory.rs | 2 +- src/util/metadata/side_metadata/global.rs | 8 +-- src/util/metadata/side_metadata/helpers.rs | 4 +- src/util/options.rs | 3 +- src/util/os/memory.rs | 26 ++++++++- src/util/os/mod.rs | 18 +++--- src/util/os/posix_common.rs | 23 ++++++++ src/util/os/process.rs | 22 +------ src/util/os/windows.rs | 18 +++--- src/util/raw_memory_freelist.rs | 7 +-- .../mock_test_handle_mmap_conflict.rs | 2 +- .../mock_tests/mock_test_handle_mmap_oom.rs | 2 +- 26 files changed, 199 insertions(+), 85 deletions(-) diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index 287d1c980b..4aa3f13e7e 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -13,6 +13,19 @@ dummyvm_toml=$project_root/docs/dummyvm/Cargo.toml cargo update -p home@0.5.11 --precise 0.5.5 # This can be removed once we move to Rust 1.81 or newer cargo update -p home@0.5.12 --precise 0.5.5 # This requires Rust edition 2024 +# Read a line and strip trailing CR (works for CRLF and LF files) +strip_cr() { + local s=$1 + printf '%s' "${s%$'\r'}" +} + +# Trim all whitespace (for feature keys) +trim_ws() { + local s=$1 + # remove all whitespace characters + printf '%s' "${s//[[:space:]]/}" +} + # Repeat a command for all the features. Requires the command as one argument (with double quotes) for_all_features() { # without mutually exclusive features @@ -47,6 +60,8 @@ init_non_exclusive_features() { i=0 while IFS= read -r line; do + line=$(strip_cr "$line") + # Only parse non mutally exclusive features if [[ $line == *"-- Non mutually exclusive features --"* ]]; then parse_features=true @@ -67,7 +82,7 @@ init_non_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - feature_name=$(echo $feature) + feature_name=$(trim_ws "$feature") # jemalloc does not support Windows if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then continue @@ -79,6 +94,8 @@ init_non_exclusive_features() { done < $cargo_toml non_exclusive_features=$(IFS=$','; echo "${features[*]}") + + echo "Non exclusive features: $non_exclusive_features" } # Get exclusive features @@ -94,6 +111,8 @@ init_exclusive_features() { declare -a features=() while IFS= read -r line; do + line=$(strip_cr "$line") + # Only parse mutally exclusive features if [[ $line == *"-- Mutally exclusive features --"* ]]; then parse_features=true @@ -122,7 +141,7 @@ init_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - features[i]=$(echo $feature) + feature_name=$(trim_ws "$feature") # jemalloc does not support Windows if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then continue diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index fc16ba727b..96a7f60c38 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -7,6 +7,7 @@ use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage}; use crate::util::ObjectReference; use crate::vm::slot::{MemorySlice, Slot}; use crate::vm::*; +use crate::util::os::*; use crate::MMTK; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; @@ -117,7 +118,7 @@ impl GCWork for ProcessModBuf { !gen.is_object_in_nursery(*obj), "{} was logged but is not mature. Dumping process memory maps:\n{}", *obj, - crate::util::memory::get_process_memory_maps(), + OSProcess::get_process_memory_maps().unwrap(), ); ::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::( *obj, diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 6505fd2e60..8cf1b30c4e 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -12,6 +12,7 @@ use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; +use crate::util::os::*; use crate::vm::*; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; @@ -292,7 +293,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::memory::mprotect(start, extent) { + if let Err(e) = crate::util::os::OSMemory::mprotect(start, extent) { panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); @@ -307,10 +308,10 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::memory::munprotect( + if let Err(e) = crate::util::os::OSMemory::munprotect( start, extent, - crate::util::memory::MmapProtection::ReadWriteExec, + crate::util::os::MmapProtection::ReadWriteExec, ) { panic!("Failed to unprotect memory: {:?}", e); } diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index ad6bc12de4..6853db1916 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -253,7 +253,10 @@ impl Block { line.mark(0); } #[cfg(feature = "immix_zero_on_release")] - crate::util::memory::zero(line.start(), Line::BYTES); + { + use crate::util::os::*; + OSMemory::zero(line.start(), Line::BYTES); + } // We need to clear the pin bit if it is on the side, as this line can be reused #[cfg(feature = "object_pinning")] diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index 5395273d91..f6dcd98879 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -14,8 +14,7 @@ use crate::util::heap::gc_trigger::GCTrigger; use crate::util::heap::layout::vm_layout::vm_layout; use crate::util::heap::PageResource; use crate::util::heap::VMRequest; -use crate::util::memory::MmapAnnotation; -use crate::util::memory::MmapStrategy; +use crate::util::os::*; use crate::util::metadata::side_metadata::SideMetadataContext; use crate::util::metadata::side_metadata::SideMetadataSanity; use crate::util::object_enum::ObjectEnumerator; @@ -158,7 +157,7 @@ impl Space for LockFreeImmortalSpace { } } if self.slow_path_zeroing { - crate::util::memory::zero(start, bytes); + crate::util::os::OSMemory::zero(start, bytes); } start } @@ -257,9 +256,11 @@ impl LockFreeImmortalSpace { // Eagerly memory map the entire heap (also zero all the memory) let strategy = MmapStrategy::new( *args.options.transparent_hugepages, - crate::util::memory::MmapProtection::ReadWrite, + crate::util::os::MmapProtection::ReadWrite, + false, + true, ); - crate::util::memory::dzmmap_noreplace( + crate::util::os::OSMemory::dzmmap( start, aligned_total_bytes, strategy, diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 626422b6a4..fc78a14be5 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -13,6 +13,7 @@ use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, vo_bit}; use crate::util::object_enum::{self, ObjectEnumerator}; use crate::util::{Address, ObjectReference}; +use crate::util::os::*; use crate::{vm::*, ObjectQueue}; use atomic::Ordering; @@ -225,7 +226,7 @@ impl MarkCompactSpace { // Clear header forwarding pointer for an object fn clear_header_forwarding_pointer(object: ObjectReference) { - crate::util::memory::zero( + OSMemory::zero( Self::header_forwarding_pointer_address(object), GC_EXTRA_HEADER_BYTES, ); diff --git a/src/policy/space.rs b/src/policy/space.rs index 4582fcfb3f..4bbfee0762 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -30,7 +30,7 @@ use crate::util::heap::layout::Mmapper; use crate::util::heap::layout::VMMap; use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::heap::HeapMeta; -use crate::util::memory::{self, HugePageSupport, MmapProtection, MmapStrategy}; +use crate::util::os::*; use crate::vm::VMBinding; use std::marker::PhantomData; @@ -192,7 +192,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { res.start, res.pages, self.common().mmap_strategy(), - &memory::MmapAnnotation::Space { + &MmapAnnotation::Space { name: self.get_name(), }, ) @@ -202,7 +202,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { self.get_name(), )) { - memory::handle_mmap_error::(mmap_error, tls, res.start, bytes); + handle_space_mmap_error::(mmap_error, tls, res.start, bytes); } }; let grow_space = || { @@ -227,7 +227,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { // TODO: Concurrent zeroing if self.common().zeroed { - memory::zero(res.start, bytes); + OSMemory::zero(res.start, bytes); } // Some assertions @@ -532,6 +532,54 @@ pub(crate) fn print_vm_map( Ok(()) } +fn handle_space_mmap_error( + error: std::io::Error, + tls: VMThread, + addr: Address, + bytes: usize, +) { + use std::io::ErrorKind; + use crate::util::alloc::AllocationError; + + eprintln!("Failed to mmap {}, size {}", addr, bytes); + eprintln!("{}", OSProcess::get_process_memory_maps().unwrap()); + + let call_binding_oom = || { + // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + }; + + match error.kind() { + // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. + ErrorKind::OutOfMemory => { + call_binding_oom(); + } + // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. + // TODO: We may be able to remove this now. + ErrorKind::Other => { + // further check the error + if let Some(os_errno) = error.raw_os_error() { + if OSMemory::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + ErrorKind::AlreadyExists => { + panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); + } + _ => { + if let Some(os_errno) = error.raw_os_error() { + if OSMemory::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + } + panic!("Unexpected mmap failure: {:?}", error) +} + impl_downcast!(Space where VM: VMBinding); pub struct CommonSpace { @@ -767,6 +815,8 @@ impl CommonSpace { } else { MmapProtection::ReadWrite }, + replace: false, + reserve: true, } } diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index b443fe0c6a..6e637c8314 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -9,7 +9,7 @@ use crate::util::linear_scan::Region; use crate::util::Address; use crate::util::VMThread; use crate::vm::VMBinding; - +use crate::util::os::*; use super::allocator::AllocatorContext; /// A MiMalloc free list allocator @@ -166,7 +166,7 @@ impl FreeListAllocator { // Zeroing memory right before we return it. // If we move the zeroing to somewhere else, we need to clear the list link here: cell.store::
(Address::ZERO) let cell_size = block.load_block_cell_size(); - crate::util::memory::zero(cell, cell_size); + crate::util::os::OSMemory::zero(cell, cell_size); // Make sure the memory is zeroed. This looks silly as we zero the cell right before this check. // But we would need to move the zeroing to somewhere so we can do zeroing at a coarser grainularity. diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index eb2e5235fa..c47dbd6b26 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -12,6 +12,7 @@ use crate::util::linear_scan::Region; use crate::util::opaque_pointer::VMThread; use crate::util::rust_util::unlikely; use crate::util::Address; +use crate::util::os::*; use crate::vm::*; /// Immix allocator @@ -250,7 +251,7 @@ impl ImmixAllocator { end_line, self.tls ); - crate::util::memory::zero( + OSMemory::zero( self.bump_pointer.cursor, self.bump_pointer.limit - self.bump_pointer.cursor, ); diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index 1c9dabf460..00a75b54d1 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -14,7 +14,7 @@ use crate::util::heap::layout::vm_layout::*; use crate::util::heap::layout::CreateFreeListResult; use crate::util::heap::pageresource::CommonPageResource; use crate::util::heap::space_descriptor::SpaceDescriptor; -use crate::util::memory; +use crate::util::os::*; use crate::util::opaque_pointer::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::vm::*; @@ -27,7 +27,7 @@ pub struct FreeListPageResource { sync: Mutex, _p: PhantomData, /// Protect memory on release, and unprotect on re-allocate. - pub(crate) protect_memory_on_release: Option, + pub(crate) protect_memory_on_release: Option, } unsafe impl Send for FreeListPageResource {} @@ -219,7 +219,7 @@ impl FreeListPageResource { // > (e.g., read versus read/write protection) exceeding the // > allowed maximum. assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = memory::mprotect(start, conversions::pages_to_bytes(pages)) { + if let Err(e) = OSMemory::mprotect(start, conversions::pages_to_bytes(pages)) { panic!( "Failed at protecting memory (starting at {}): {:?}", start, e @@ -230,7 +230,7 @@ impl FreeListPageResource { /// Unprotect the memory fn munprotect(&self, start: Address, pages: usize) { assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = memory::munprotect( + if let Err(e) = OSMemory::munprotect( start, conversions::pages_to_bytes(pages), self.protect_memory_on_release.unwrap(), diff --git a/src/util/heap/layout/map64.rs b/src/util/heap/layout/map64.rs index b6e0e5993c..de9e16ddc5 100644 --- a/src/util/heap/layout/map64.rs +++ b/src/util/heap/layout/map64.rs @@ -6,7 +6,7 @@ use crate::util::freelist::FreeList; use crate::util::heap::layout::heap_parameters::*; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::space_descriptor::SpaceDescriptor; -use crate::util::memory::MmapStrategy; +use crate::util::os::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::util::Address; use std::cell::UnsafeCell; diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index c69963b925..607e17d504 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -2,7 +2,7 @@ use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::conversions::raw_is_aligned; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::layout::Mmapper; -use crate::util::memory::*; +use crate::util::os::*; use crate::util::Address; use bytemuck::NoUninit; use std::{io::Result, sync::Mutex}; @@ -146,7 +146,7 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - strategy: MmapStrategy, + mut strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -162,7 +162,8 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { trace!("Trying to quarantine {group_range}"); - mmap_noreserve(group_start, group_bytes, strategy, anno)?; + strategy.reserve = false; + OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; Ok(Some(MapState::Quarantined)) } MapState::Quarantined => { @@ -181,7 +182,7 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - strategy: MmapStrategy, + mut strategy: MmapStrategy, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -196,11 +197,14 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - dzmmap_noreplace(group_start, group_bytes, strategy, anno)?; + strategy.replace = false; + OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; Ok(Some(MapState::Mapped)) } MapState::Quarantined => { - unsafe { dzmmap(group_start, group_bytes, strategy, anno) }?; + strategy.replace = true; + strategy.reserve = true; + OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; Ok(Some(MapState::Mapped)) } MapState::Mapped => Ok(None), @@ -231,7 +235,7 @@ mod tests { use super::*; use crate::mmap_anno_test; use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::memory; + use crate::util::os::memory; use crate::util::test_util::CHUNK_STATE_MMAPPER_TEST_REGION; use crate::util::test_util::{serial_test, with_cleanup}; use crate::util::{conversions, Address}; diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs index c981677ab1..21a134f039 100644 --- a/src/util/heap/layout/mmapper/mod.rs +++ b/src/util/heap/layout/mmapper/mod.rs @@ -1,5 +1,5 @@ use crate::util::{ - memory::{MmapAnnotation, MmapStrategy}, + os::*, Address, }; use std::io::Result; diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 19400a485b..0302fe8c62 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -18,7 +18,7 @@ pub fn align_alloc(size: usize, align: usize) -> Address { return Address::ZERO; } let address = Address::from_mut_ptr(ptr); - crate::util::memory::zero(address, size); + crate::util::os::memory::zero(address, size); address } diff --git a/src/util/memory.rs b/src/util/memory.rs index 2e267ae17b..52f2c59e07 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -150,7 +150,7 @@ pub enum MmapAnnotation<'a> { #[macro_export] macro_rules! mmap_anno_test { () => { - &$crate::util::memory::MmapAnnotation::Test { + &$crate::util::os::memory::MmapAnnotation::Test { file: file!(), line: line!(), } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 65ce7443e1..dd8705cbf8 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -2,7 +2,7 @@ use super::*; use crate::util::constants::{BYTES_IN_PAGE, BYTES_IN_WORD, LOG_BITS_IN_BYTE}; use crate::util::conversions::raw_align_up; use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK; -use crate::util::memory::{self, MmapAnnotation}; +use crate::util::os::*; use crate::util::metadata::metadata_val_traits::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC; @@ -122,7 +122,7 @@ impl SideMetadataSpec { meta_start ); - memory::panic_if_unmapped( + OSMemory::panic_if_unmapped( meta_start, BYTES_IN_PAGE, &MmapAnnotation::Misc { @@ -180,7 +180,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - memory::zero(start, end - start); + OSMemory::zero(start, end - start); false } BitByteRange::BitsInByte { @@ -217,7 +217,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - memory::set(start, 0xff, end - start); + OSMemory::set(start, 0xff, end - start); false } BitByteRange::BitsInByte { diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 24e5366dc8..b7e0c3454d 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -4,7 +4,7 @@ use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::{BITS_IN_WORD, BYTES_IN_PAGE, LOG_BITS_IN_BYTE}; use crate::util::conversions::rshift_align_up; use crate::util::heap::layout::vm_layout::VMLayout; -use crate::util::memory::{MmapAnnotation, MmapStrategy}; +use crate::util::os::*; #[cfg(target_pointer_width = "32")] use crate::util::metadata::side_metadata::address_to_chunked_meta_address; use crate::util::Address; @@ -93,7 +93,7 @@ pub(super) fn align_metadata_address( /// Unmaps the specified metadata range, or panics. #[cfg(test)] pub(crate) fn ensure_munmap_metadata(start: Address, size: usize) { - use crate::util::memory; + use crate::util::os::memory; trace!("ensure_munmap_metadata({}, 0x{:x})", start, size); assert!(memory::munmap(start, size).is_ok()) diff --git a/src/util/options.rs b/src/util/options.rs index fd8952d1da..81f707d32f 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,6 +1,7 @@ use crate::scheduler::affinity::{get_total_num_cpus, CoreId}; use crate::util::constants::LOG_BYTES_IN_MBYTE; use crate::util::Address; +use crate::util::os::*; use std::default::Default; use std::fmt::Debug; use std::str::FromStr; @@ -950,7 +951,7 @@ options! { thread_affinity: AffinityKind [|v: &AffinityKind| v.validate()] = AffinityKind::OsDefault, /// Set the GC trigger. This defines the heap size and how MMTk triggers a GC. /// Default to a fixed heap size of 0.5x physical memory. - gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((crate::util::memory::get_system_total_memory() as f64 * 0.5f64) as usize), + gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((OSMemory::get_system_total_memory().unwrap_or(4 * 1024 * 1024 * 1024) as f64 * 0.5f64) as usize), /// Enable transparent hugepage support for MMTk spaces via madvise (only Linux is supported) /// This only affects the memory for MMTk spaces. transparent_hugepages: bool [|v: &bool| !v || cfg!(target_os = "linux")] = false, diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 9de27446cf..58d65f0be5 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -1,5 +1,5 @@ use bytemuck::NoUninit; -use std::io::{Error, Result}; +use std::io::Result; use crate::util::address::Address; @@ -13,9 +13,31 @@ pub trait Memory { } } fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
; + fn is_mmap_oom(os_errno: i32) -> bool; fn munmap(start: Address, size: usize) -> Result<()>; fn mprotect(start: Address, size: usize) -> Result<()>; fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()>; + fn panic_if_unmapped(start: Address, size: usize, annotation: &MmapAnnotation<'_>); + fn get_system_total_memory() -> Result { + use sysinfo::MemoryRefreshKind; + use sysinfo::{RefreshKind, System}; + + // TODO: Note that if we want to get system info somewhere else in the future, we should + // refactor this instance into some global struct. sysinfo recommends sharing one instance of + // `System` instead of making multiple instances. + // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info + // + // If we refactor the `System` instance to use it for other purposes, please make sure start-up + // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using + // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to + // start-up time. During start-up, MMTk core only needs the total memory to initialize the + // `Options`. If we only load memory-related components on start-up, it should only take <1ms + // to initialize the `System` instance. + let sys = System::new_with_specifics( + RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), + ); + Ok(sys.total_memory()) + } } /// Strategy for performing mmap @@ -131,7 +153,7 @@ pub enum MmapAnnotation<'a> { #[macro_export] macro_rules! mmap_anno_test_2 { () => { - &$crate::util::memory::MmapAnnotation::Test { + &$crate::util::os::memory::MmapAnnotation::Test { file: file!(), line: line!(), } diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index d0cea730d4..291717f749 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -1,5 +1,7 @@ -pub mod memory; -pub mod process; +mod memory; +pub use memory::*; +mod process; +pub use process::*; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] pub(crate) mod posix_common; @@ -10,9 +12,11 @@ pub(crate) mod linux; pub(crate) mod windows; #[cfg(target_os = "windows")] -pub use windows::Windows as OS; +pub use windows::WindowsMemoryImpl as OSMemory; +#[cfg(target_os = "windows")] +pub use windows::WindowsProcessImpl as OSProcess; -pub trait OperatingSystem { - type OSMemory: memory::Memory; - type OSProcess: process::Process; -} +// pub trait OperatingSystem { +// type OSMemory: memory::Memory; +// type OSProcess: process::Process; +// } diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index 9983121e8f..69b8326033 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -58,6 +58,29 @@ pub fn posix_mmap(start: Address, size: usize, strategy: MmapStrategy, annotatio } } +pub fn posix_panic_if_unmapped(start: Address, size: usize, anno: &MmapAnnotation) { + let flags = MMAP_FLAGS; + match mmap_fixed( + _start, + _size, + flags, + MmapStrategy { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + }, + _anno, + ) { + Ok(_) => panic!("{} of size {} is not mapped", _start, _size), + Err(e) => { + assert!( + e.kind() == std::io::ErrorKind::AlreadyExists, + "Failed to check mapped: {:?}", + e + ); + } + } +} + fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { let ret = f(); if ret == expect { diff --git a/src/util/os/process.rs b/src/util/os/process.rs index cd6f6d7719..eea183feb4 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -1,25 +1,7 @@ use std::io::Result; pub trait Process { + /// Return error if unable to get process memory maps. + /// If unimplemented, just return Ok with an string to indiate that. fn get_process_memory_maps() -> Result; - fn get_system_total_memory() -> Result { - use sysinfo::MemoryRefreshKind; - use sysinfo::{RefreshKind, System}; - - // TODO: Note that if we want to get system info somewhere else in the future, we should - // refactor this instance into some global struct. sysinfo recommends sharing one instance of - // `System` instead of making multiple instances. - // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info - // - // If we refactor the `System` instance to use it for other purposes, please make sure start-up - // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using - // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to - // start-up time. During start-up, MMTk core only needs the total memory to initialize the - // `Options`. If we only load memory-related components on start-up, it should only take <1ms - // to initialize the `System` instance. - let sys = System::new_with_specifics( - RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), - ); - Ok(sys.total_memory()) - } } diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index 1880259df8..0ebc065bfb 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -1,17 +1,11 @@ -use crate::util::os::{OperatingSystem, memory::*}; +use crate::util::os::memory::*; use crate::util::address::Address; use std::io::Result; -pub struct Windows; -impl OperatingSystem for Windows { - type OSMemory = WindowsMemoryImpl; - type OSProcess = WindowsProcessImpl; -} - pub struct WindowsMemoryImpl; impl Memory for WindowsMemoryImpl { - fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ + fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, _annotation: &MmapAnnotation<'_>) -> Result
{ use std::io; use windows_sys::Win32::System::Memory::{ VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, @@ -149,6 +143,14 @@ impl Memory for WindowsMemoryImpl { Ok(()) } } + + fn panic_if_unmapped(start: Address, size: usize, _annotation: &MmapAnnotation<'_>) { + warn!("Check if {} of size {} is mapped is ignored on Windows", start, size); + } + + fn is_mmap_oom(os_errno: i32) -> bool { + os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 || os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 + } } impl MmapProtection { diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index f02e6ad971..0020660671 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -1,9 +1,8 @@ use super::freelist::*; -use super::memory::MmapStrategy; use crate::util::address::Address; use crate::util::constants::*; use crate::util::conversions; -use crate::util::memory::MmapAnnotation; +use crate::util::os::*; /** log2 of the number of bits used by a free list entry (two entries per unit) */ const LOG_ENTRY_BITS: usize = LOG_BITS_IN_INT as _; @@ -199,7 +198,7 @@ impl RawMemoryFreeList { } fn mmap(&self, start: Address, bytes: usize) { - let res = super::memory::dzmmap_noreplace( + let res = OSMemory::dzmmap( start, bytes, self.strategy, @@ -222,7 +221,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - let _ = crate::util::memory::munmap(self.base, len); + let _ = os::memory::munmap(self.base, len); } } } diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs index 1dc6a5759e..dc78bb43a9 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs @@ -1,6 +1,6 @@ use super::mock_test_prelude::*; -use crate::util::memory; +use crate::util::os::memory; use crate::util::opaque_pointer::*; use crate::util::Address; diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs index b810c47a43..ca85532e0a 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs @@ -1,6 +1,6 @@ use super::mock_test_prelude::*; -use crate::util::memory; +use crate::util::os::memory; use crate::util::opaque_pointer::*; use crate::util::Address; From cda7b725277e189916eb8e2388aaa07311f0383b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Sun, 4 Jan 2026 21:51:27 +1300 Subject: [PATCH 06/27] Test passed on Windows --- src/policy/space.rs | 50 +-------------- src/util/heap/layout/mmapper/csm/mod.rs | 8 +-- src/util/memory.rs | 63 ++++++++++--------- src/util/metadata/side_metadata/global.rs | 2 +- src/util/metadata/side_metadata/helpers.rs | 4 +- src/util/os/memory.rs | 59 +++++++++++++++-- src/util/os/windows.rs | 5 +- src/util/raw_memory_freelist.rs | 2 +- .../mock_tests/mock_test_handle_mmap_oom.rs | 10 +-- 9 files changed, 105 insertions(+), 98 deletions(-) diff --git a/src/policy/space.rs b/src/policy/space.rs index 4bbfee0762..a28b308a87 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -202,7 +202,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { self.get_name(), )) { - handle_space_mmap_error::(mmap_error, tls, res.start, bytes); + OSMemory::handle_mmap_error::(mmap_error, tls, res.start, bytes); } }; let grow_space = || { @@ -532,54 +532,6 @@ pub(crate) fn print_vm_map( Ok(()) } -fn handle_space_mmap_error( - error: std::io::Error, - tls: VMThread, - addr: Address, - bytes: usize, -) { - use std::io::ErrorKind; - use crate::util::alloc::AllocationError; - - eprintln!("Failed to mmap {}, size {}", addr, bytes); - eprintln!("{}", OSProcess::get_process_memory_maps().unwrap()); - - let call_binding_oom = || { - // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - }; - - match error.kind() { - // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. - ErrorKind::OutOfMemory => { - call_binding_oom(); - } - // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. - // TODO: We may be able to remove this now. - ErrorKind::Other => { - // further check the error - if let Some(os_errno) = error.raw_os_error() { - if OSMemory::is_mmap_oom(os_errno) { - call_binding_oom(); - } - } - } - ErrorKind::AlreadyExists => { - panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); - } - _ => { - if let Some(os_errno) = error.raw_os_error() { - if OSMemory::is_mmap_oom(os_errno) { - call_binding_oom(); - } - } - } - } - panic!("Unexpected mmap failure: {:?}", error) -} - impl_downcast!(Space where VM: VMBinding); pub struct CommonSpace { diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index 607e17d504..fb414ca237 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -235,7 +235,7 @@ mod tests { use super::*; use crate::mmap_anno_test; use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::os::memory; + use crate::util::os::*; use crate::util::test_util::CHUNK_STATE_MMAPPER_TEST_REGION; use crate::util::test_util::{serial_test, with_cleanup}; use crate::util::{conversions, Address}; @@ -275,7 +275,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -303,7 +303,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -332,7 +332,7 @@ mod tests { } }, || { - memory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) diff --git a/src/util/memory.rs b/src/util/memory.rs index 52f2c59e07..f2ad9cfed0 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -148,9 +148,9 @@ pub enum MmapAnnotation<'a> { /// Construct an `MmapAnnotation::Test` with the current file name and line number. #[macro_export] -macro_rules! mmap_anno_test { +macro_rules! mmap_anno_test_unused { () => { - &$crate::util::os::memory::MmapAnnotation::Test { + &$crate::util::os::MmapAnnotation::Test { file: file!(), line: line!(), } @@ -158,7 +158,7 @@ macro_rules! mmap_anno_test { } // Export this to external crates -pub use mmap_anno_test; +pub use mmap_anno_test_unused; impl std::fmt::Display for MmapAnnotation<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -683,7 +683,8 @@ pub(crate) fn get_system_total_memory() -> u64 { #[cfg(test)] mod tests { - use super::*; + use crate::util::os::*; + use crate::util::Address; use crate::util::constants::BYTES_IN_PAGE; use crate::util::test_util::MEMORY_TEST_REGION; use crate::util::test_util::{serial_test, with_cleanup}; @@ -697,17 +698,17 @@ mod tests { with_cleanup( || { let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) }; assert!(res.is_ok()); // We can overwrite with dzmmap let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) }; assert!(res.is_ok()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ); }); @@ -718,18 +719,21 @@ mod tests { serial_test(|| { with_cleanup( || { - let res = dzmmap_noreplace( + let res = OSMemory::dzmmap( START, BYTES_IN_PAGE, - MmapStrategy::TEST, + MmapStrategy { + replace: false, + ..MmapStrategy::TEST + }, mmap_anno_test!(), ); assert!(res.is_ok()); - let res = munmap(START, BYTES_IN_PAGE); + let res = OSMemory::munmap(START, BYTES_IN_PAGE); assert!(res.is_ok()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ) }) @@ -743,14 +747,17 @@ mod tests { || { // Make sure we mmapped the memory let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) }; assert!(res.is_ok()); // Use dzmmap_noreplace will fail let res = dzmmap_noreplace( START, BYTES_IN_PAGE, - MmapStrategy::TEST, + MmapStrategy { + replace: false, + ..MmapStrategy::TEST + }, mmap_anno_test!(), ); assert!(res.is_err()); @@ -768,16 +775,16 @@ mod tests { with_cleanup( || { let res = - mmap_noreserve(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()); + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { reserve: false, ..MmapStrategy::TEST }, mmap_anno_test!()); assert!(res.is_ok()); // Try reserve it let res = unsafe { - dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { replace: true, ..MmapStrategy::TEST }, mmap_anno_test!()) }; assert!(res.is_ok()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ) }) @@ -791,10 +798,10 @@ mod tests { with_cleanup( || { // We expect this call to panic - panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ) }) @@ -805,17 +812,17 @@ mod tests { serial_test(|| { with_cleanup( || { - assert!(dzmmap_noreplace( + assert!(OSMemory::dzmmap( START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!() ) .is_ok()); - panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ) }) @@ -829,7 +836,7 @@ mod tests { with_cleanup( || { // map 1 page from START - assert!(dzmmap_noreplace( + assert!(OSMemory::dzmmap( START, BYTES_IN_PAGE, MmapStrategy::TEST, @@ -838,10 +845,10 @@ mod tests { .is_ok()); // check if the next page is mapped - which should panic - panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE, mmap_anno_test!()); }, || { - assert!(munmap(START, BYTES_IN_PAGE * 2).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); }, ) }) @@ -857,7 +864,7 @@ mod tests { with_cleanup( || { // map 1 page from START - assert!(dzmmap_noreplace( + assert!(OSMemory::dzmmap( START, BYTES_IN_PAGE, MmapStrategy::TEST, @@ -866,10 +873,10 @@ mod tests { .is_ok()); // check if the 2 pages from START are mapped. The second page is unmapped, so it should panic. - panic_if_unmapped(START, BYTES_IN_PAGE * 2, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE * 2, mmap_anno_test!()); }, || { - assert!(munmap(START, BYTES_IN_PAGE * 2).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); }, ) }) @@ -877,7 +884,7 @@ mod tests { #[test] fn test_get_system_total_memory() { - let total = get_system_total_memory(); + let total = OSMemory::get_system_total_memory().unwrap(); println!("Total memory: {:?}", total); } } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index dd8705cbf8..f30533fe3c 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -1662,7 +1662,7 @@ mod tests { use crate::util::heap::layout::vm_layout; use crate::util::test_util::{serial_test, with_cleanup}; - use memory::MmapStrategy; + use crate::util::os::*; use paste::paste; const TEST_LOG_BYTES_IN_REGION: usize = 12; diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index b7e0c3454d..55f6f2e58e 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -93,10 +93,10 @@ pub(super) fn align_metadata_address( /// Unmaps the specified metadata range, or panics. #[cfg(test)] pub(crate) fn ensure_munmap_metadata(start: Address, size: usize) { - use crate::util::os::memory; + use crate::util::os::*; trace!("ensure_munmap_metadata({}, 0x{:x})", start, size); - assert!(memory::munmap(start, size).is_ok()) + assert!(OSMemory::munmap(start, size).is_ok()) } /// Unmaps a metadata space (`spec`) for the specified data address range (`start` and `size`) diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 58d65f0be5..fa17a734bc 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -1,7 +1,9 @@ use bytemuck::NoUninit; use std::io::Result; -use crate::util::address::Address; +use crate::{util::{VMThread, address::Address}, vm::VMBinding}; +use crate::util::os::*; +use crate::vm::*; pub trait Memory { fn zero(start: Address, len: usize) { @@ -13,6 +15,55 @@ pub trait Memory { } } fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
; + + fn handle_mmap_error( + error: std::io::Error, + tls: VMThread, + addr: Address, + bytes: usize, + ) { + use std::io::ErrorKind; + use crate::util::alloc::AllocationError; + + eprintln!("Failed to mmap {}, size {}", addr, bytes); + eprintln!("{}", OSProcess::get_process_memory_maps().unwrap()); + + let call_binding_oom = || { + // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. + trace!("Signal MmapOutOfMemory!"); + VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); + unreachable!() + }; + + match error.kind() { + // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. + ErrorKind::OutOfMemory => { + call_binding_oom(); + } + // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. + // TODO: We may be able to remove this now. + ErrorKind::Other => { + // further check the error + if let Some(os_errno) = error.raw_os_error() { + if OSMemory::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + ErrorKind::AlreadyExists => { + panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); + } + _ => { + if let Some(os_errno) = error.raw_os_error() { + if OSMemory::is_mmap_oom(os_errno) { + call_binding_oom(); + } + } + } + } + panic!("Unexpected mmap failure: {:?}", error) + } + fn is_mmap_oom(os_errno: i32) -> bool; fn munmap(start: Address, size: usize) -> Result<()>; fn mprotect(start: Address, size: usize) -> Result<()>; @@ -151,9 +202,9 @@ pub enum MmapAnnotation<'a> { /// Construct an `MmapAnnotation::Test` with the current file name and line number. #[macro_export] -macro_rules! mmap_anno_test_2 { +macro_rules! mmap_anno_test { () => { - &$crate::util::os::memory::MmapAnnotation::Test { + &$crate::util::os::MmapAnnotation::Test { file: file!(), line: line!(), } @@ -161,7 +212,7 @@ macro_rules! mmap_anno_test_2 { } // Export this to external crates -pub use mmap_anno_test_2; +pub use mmap_anno_test; impl std::fmt::Display for MmapAnnotation<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index 0ebc065bfb..3d7d2bae5e 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -171,9 +171,6 @@ pub struct WindowsProcessImpl; impl Process for WindowsProcessImpl { fn get_process_memory_maps() -> Result { // Windows-specific implementation to get process memory maps - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "get_process_memory_maps not implemented for Windows", - )) + Ok("get_process_memory_maps not implemented for Windows".to_string()) } } diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 0020660671..649a380e45 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -221,7 +221,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - let _ = os::memory::munmap(self.base, len); + let _ = OSMemory::munmap(self.base, len); } } } diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs index ca85532e0a..d5e8acee5d 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs @@ -1,6 +1,6 @@ use super::mock_test_prelude::*; -use crate::util::os::memory; +use crate::util::os::*; use crate::util::opaque_pointer::*; use crate::util::Address; @@ -18,14 +18,14 @@ pub fn test_handle_mmap_oom() { let start = unsafe { Address::from_usize(0x100_0000) }; // mmap 1 terabyte memory - we expect this will fail due to out of memory. // If that's not the case, increase the size we mmap. - let mmap_res = memory::dzmmap_noreplace( + let mmap_res = OSMemory::dzmmap( start, LARGE_SIZE, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), + MmapStrategy::TEST, + mmap_anno_test!(), ); - memory::handle_mmap_error::( + OSMemory::handle_mmap_error::( mmap_res.err().unwrap(), VMThread::UNINITIALIZED, start, From 3bdf29504a867ccb0981a4e0483f8d7a4645dc1d Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 5 Jan 2026 22:24:53 +0000 Subject: [PATCH 07/27] Build and pass tests on Linux --- src/util/heap/layout/mmapper/csm/mod.rs | 2 +- src/util/malloc/malloc_ms_util.rs | 3 +- src/util/memory.rs | 14 +- src/util/metadata/side_metadata/global.rs | 8 +- src/util/os/linux.rs | 98 ++++++++++-- src/util/os/memory.rs | 15 +- src/util/os/mod.rs | 5 + src/util/os/posix_common.rs | 140 +++++++++--------- .../mock_test_handle_mmap_conflict.rs | 19 ++- 9 files changed, 201 insertions(+), 103 deletions(-) diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index fb414ca237..0962f7c602 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -197,7 +197,7 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - strategy.replace = false; + // strategy.replace = false; OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; Ok(Some(MapState::Mapped)) } diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 0302fe8c62..5e99211123 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -1,6 +1,7 @@ use crate::util::constants::BYTES_IN_ADDRESS; use crate::util::malloc::library::*; use crate::util::Address; +use crate::util::os::*; use crate::vm::VMBinding; /// Allocate with alignment. This also guarantees the memory is zero initialized. @@ -18,7 +19,7 @@ pub fn align_alloc(size: usize, align: usize) -> Address { return Address::ZERO; } let address = Address::from_mut_ptr(ptr); - crate::util::os::memory::zero(address, size); + OSMemory::zero(address, size); address } diff --git a/src/util/memory.rs b/src/util/memory.rs index f2ad9cfed0..706577181f 100644 --- a/src/util/memory.rs +++ b/src/util/memory.rs @@ -703,7 +703,7 @@ mod tests { assert!(res.is_ok()); // We can overwrite with dzmmap let res = unsafe { - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) + OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { replace: true, ..MmapStrategy::TEST }, mmap_anno_test!()) }; assert!(res.is_ok()); }, @@ -751,7 +751,7 @@ mod tests { }; assert!(res.is_ok()); // Use dzmmap_noreplace will fail - let res = dzmmap_noreplace( + let res = OSMemory::dzmmap( START, BYTES_IN_PAGE, MmapStrategy { @@ -763,7 +763,7 @@ mod tests { assert!(res.is_err()); }, || { - assert!(munmap(START, BYTES_IN_PAGE).is_ok()); + assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); }, ) }); @@ -798,7 +798,7 @@ mod tests { with_cleanup( || { // We expect this call to panic - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE); }, || { assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); @@ -819,7 +819,7 @@ mod tests { mmap_anno_test!() ) .is_ok()); - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE); }, || { assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); @@ -845,7 +845,7 @@ mod tests { .is_ok()); // check if the next page is mapped - which should panic - OSMemory::panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE); }, || { assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); @@ -873,7 +873,7 @@ mod tests { .is_ok()); // check if the 2 pages from START are mapped. The second page is unmapped, so it should panic. - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE * 2, mmap_anno_test!()); + OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE * 2); }, || { assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index f30533fe3c..0ccf1d7cf6 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -124,10 +124,7 @@ impl SideMetadataSpec { OSMemory::panic_if_unmapped( meta_start, - BYTES_IN_PAGE, - &MmapAnnotation::Misc { - name: "assert_metadata_mapped", - }, + BYTES_IN_PAGE ); } @@ -1696,7 +1693,8 @@ mod tests { || { let mmap_result = context.try_map_metadata_space(data_addr, BYTES_IN_PAGE, "test_space"); - assert!(mmap_result.is_ok()); + let _ = mmap_result.unwrap(); + // assert!(mmap_result.is_ok()); f(&spec, data_addr, meta_addr); }, diff --git a/src/util/os/linux.rs b/src/util/os/linux.rs index d9d6474f40..becdca9b35 100644 --- a/src/util/os/linux.rs +++ b/src/util/os/linux.rs @@ -1,27 +1,101 @@ -use crate::util::os::memory::*; +use crate::util::os::*; +use crate::util::os::posix_common; use crate::util::address::Address; use std::io::Result; -pub struct LinuxMemory; +pub struct LinuxMemoryImpl; -impl Memory for LinuxMemory { +impl Memory for LinuxMemoryImpl { fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ - // Windows-specific implementation of dzmmap - unimplemented!() + // println!("Mmap with strategy: {:?}", strategy); + let addr = posix_common::mmap(start, size, strategy)?; + // println!("Mmap done"); + + if !cfg!(feature = "no_mmap_annotation") { + posix_common::set_vma_name(addr, size, annotation); + // println!("Set annotation done"); + } + + Self::set_hugepage(addr, size, strategy.huge_page)?; + // println!("Set huge page done"); + + // Zero memory if needed + Ok(addr) } fn munmap(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of munmap - unimplemented!() + posix_common::munmap(start, size) } fn mprotect(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of mprotect - unimplemented!() + posix_common::mprotect(start, size) + } + + fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + posix_common::munprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + posix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(start: Address, size: usize) { + let strategy = MmapStrategy { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + match posix_common::mmap( + start, + size, + strategy, + ) { + Ok(_) => panic!("{} of size {} is not mapped", start, size), + Err(e) => { + assert!( + e.kind() == std::io::ErrorKind::AlreadyExists, + "Failed to check mapped: {:?}", + e + ); + } + } } +} + +impl LinuxMemoryImpl { + pub fn set_hugepage(start: Address, size: usize, options: HugePageSupport) -> Result<()> { + match options { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => { + posix_common::wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ) + } + } + } +} + +impl MmapStrategy { + pub fn get_mmap_flags(&self) -> i32 { + let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; + if self.replace { + flags |= libc::MAP_FIXED; + } else { + flags |= libc::MAP_FIXED_NOREPLACE + } + if !self.reserve { + flags |= libc::MAP_NORESERVE; + } + flags + } +} + +pub struct LinuxProcessImpl; - fn munprotect(start: Address, size: usize) -> Result<()> { - // Windows-specific implementation of munprotect - unimplemented!() +impl Process for LinuxProcessImpl { + fn get_process_memory_maps() -> Result { + posix_common::get_process_memory_maps() } } diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index fa17a734bc..b27bb5045c 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -68,7 +68,7 @@ pub trait Memory { fn munmap(start: Address, size: usize) -> Result<()>; fn mprotect(start: Address, size: usize) -> Result<()>; fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()>; - fn panic_if_unmapped(start: Address, size: usize, annotation: &MmapAnnotation<'_>); + fn panic_if_unmapped(start: Address, size: usize); fn get_system_total_memory() -> Result { use sysinfo::MemoryRefreshKind; use sysinfo::{RefreshKind, System}; @@ -118,6 +118,9 @@ impl MmapStrategy { } /// The strategy for MMTk's own internal memory + #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings. + pub const INTERNAL_MEMORY: Self = Self::TEST; + #[cfg(not(test))] pub const INTERNAL_MEMORY: Self = Self { huge_page: HugePageSupport::No, prot: MmapProtection::ReadWrite, @@ -126,11 +129,19 @@ impl MmapStrategy { }; /// The strategy for MMTk side metadata + #[cfg(test)] + pub const SIDE_METADATA: Self = Self::TEST; + #[cfg(not(test))] pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; /// The strategy for MMTk's test memory #[cfg(test)] - pub const TEST: Self = Self::INTERNAL_MEMORY; + pub const TEST: Self = Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: true, + reserve: true, + }; } /// The protection flags for Mmap diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index 291717f749..c467882879 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -16,6 +16,11 @@ pub use windows::WindowsMemoryImpl as OSMemory; #[cfg(target_os = "windows")] pub use windows::WindowsProcessImpl as OSProcess; +#[cfg(target_os = "linux")] +pub use linux::LinuxMemoryImpl as OSMemory; +#[cfg(target_os = "linux")] +pub use linux::LinuxProcessImpl as OSProcess; + // pub trait OperatingSystem { // type OSMemory: memory::Memory; // type OSProcess: process::Process; diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index 69b8326033..620a07890e 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -2,86 +2,92 @@ use crate::util::os::memory::*; use crate::util::address::Address; use std::io::Result; -pub fn posix_mmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ +impl MmapProtection { + fn into_native_flags(&self) -> i32 { + use libc::{PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC}; + match self { + Self::ReadWrite => PROT_READ | PROT_WRITE, + Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, + Self::NoAccess => PROT_NONE, + } + } +} + +pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ let ptr = start.to_mut_ptr(); let prot = strategy.prot.into_native_flags(); + let flags = strategy.get_mmap_flags(); wrap_libc_call( &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, ptr, )?; + Ok(start) +} - #[cfg(all( - any(target_os = "linux", target_os = "android"), - not(feature = "no_mmap_annotation") - ))] - { - // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that - // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not - // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` - // for other reasons, too. That includes `start` being an invalid address, and the - // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But - // since this prctl is used for debugging, we log the error instead of panicking. - let anno_str = _anno.to_string(); - let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( - &|| unsafe { - libc::prctl( - libc::PR_SET_VMA, - libc::PR_SET_VMA_ANON_NAME, - start.to_ptr::(), - size, - anno_cstr.as_ptr(), - ) - }, - 0, - ); - if let Err(e) = result { - debug!("Error while calling prctl: {e}"); - } - } - - match strategy.huge_page { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - #[cfg(target_os = "linux")] - { - wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) - } - // Setting the transparent hugepage option to true will not pass - // the validation on non-Linux OSes - #[cfg(not(target_os = "linux"))] - unreachable!() - } - } +pub fn is_mmap_oom(os_errno: i32) -> bool { + os_errno == libc::ENOMEM } -pub fn posix_panic_if_unmapped(start: Address, size: usize, anno: &MmapAnnotation) { - let flags = MMAP_FLAGS; - match mmap_fixed( - _start, - _size, - flags, - MmapStrategy { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn set_vma_name(start: Address, size: usize, annotation: &MmapAnnotation) { + // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that + // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not + // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` + // for other reasons, too. That includes `start` being an invalid address, and the + // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But + // since this prctl is used for debugging, we log the error instead of panicking. + let anno_str = annotation.to_string(); + let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); + let result = wrap_libc_call( + &|| unsafe { + libc::prctl( + libc::PR_SET_VMA, + libc::PR_SET_VMA_ANON_NAME, + start.to_ptr::(), + size, + anno_cstr.as_ptr(), + ) }, - _anno, - ) { - Ok(_) => panic!("{} of size {} is not mapped", _start, _size), - Err(e) => { - assert!( - e.kind() == std::io::ErrorKind::AlreadyExists, - "Failed to check mapped: {:?}", - e - ); - } + 0, + ); + if let Err(e) = result { + debug!("Error while calling prctl: {e}"); } } -fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { +/// Get the memory maps for the process. The returned string is a multi-line string. +/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn get_process_memory_maps() -> Result { + // print map + use std::fs::File; + use std::io::Read; + let mut data = String::new(); + let mut f = File::open("/proc/self/maps")?; + f.read_to_string(&mut data)?; + Ok(data) +} + +pub fn munmap(start: Address, size: usize) -> Result<()> { + return wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0); +} + +pub fn mprotect(start: Address, size: usize) -> Result<()> { + let prot = libc::PROT_NONE; + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, + 0, + ) +} + +pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.into_native_flags()) }, + 0, + ) +} + +pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { let ret = f(); if ret == expect { Ok(()) diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs index dc78bb43a9..f8dcd85e74 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs @@ -1,6 +1,6 @@ use super::mock_test_prelude::*; -use crate::util::os::memory; +use crate::util::os::*; use crate::util::opaque_pointer::*; use crate::util::Address; @@ -11,23 +11,26 @@ pub fn test_handle_mmap_conflict() { || { let start = unsafe { Address::from_usize(0x100_0000) }; let one_megabyte = 1000000; - let mmap1_res = memory::dzmmap_noreplace( + let mmap1_res = OSMemory::dzmmap( start, one_megabyte, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), + MmapStrategy::TEST, + mmap_anno_test!(), ); assert!(mmap1_res.is_ok()); let panic_res = std::panic::catch_unwind(|| { - let mmap2_res = memory::dzmmap_noreplace( + let mmap2_res = OSMemory::dzmmap( start, one_megabyte, - memory::MmapStrategy::TEST, - memory::mmap_anno_test!(), + MmapStrategy { + replace: false, + ..MmapStrategy::TEST + }, + mmap_anno_test!(), ); assert!(mmap2_res.is_err()); - memory::handle_mmap_error::( + OSMemory::handle_mmap_error::( mmap2_res.err().unwrap(), VMThread::UNINITIALIZED, start, From 88ccd39f43805a33b49c1f2b2e2e1593e1f2ccfe Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 6 Jan 2026 03:18:41 +0000 Subject: [PATCH 08/27] Refactor MmapStrategy to builder style --- src/policy/largeobjectspace.rs | 2 +- src/policy/lockfreeimmortalspace.rs | 8 +--- src/policy/space.rs | 22 ++++------- src/util/heap/layout/mmapper/csm/mod.rs | 24 +++++------ src/util/heap/layout/mmapper/mod.rs | 5 ++- src/util/metadata/side_metadata/global.rs | 2 +- src/util/metadata/side_metadata/helpers.rs | 5 ++- src/util/os/memory.rs | 46 +++++++++++++++++++--- 8 files changed, 69 insertions(+), 45 deletions(-) diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index e30c12e18a..6960a93721 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -292,7 +292,7 @@ impl LargeObjectSpace { FreeListPageResource::new_contiguous(common.start, common.extent, vm_map) }; pr.protect_memory_on_release = if protect_memory_on_release { - Some(common.mmap_strategy().prot) + Some(common.mmap_protection()) } else { None }; diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index f6dcd98879..e8a3f6a625 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -254,12 +254,8 @@ impl LockFreeImmortalSpace { }; // Eagerly memory map the entire heap (also zero all the memory) - let strategy = MmapStrategy::new( - *args.options.transparent_hugepages, - crate::util::os::MmapProtection::ReadWrite, - false, - true, - ); + let strategy = MmapStrategy::default().transparent_hugepages(*args.options.transparent_hugepages).prot( + crate::util::os::MmapProtection::ReadWrite).replace(false).reserve(true); crate::util::os::OSMemory::dzmmap( start, aligned_total_bytes, diff --git a/src/policy/space.rs b/src/policy/space.rs index a28b308a87..eddc81adba 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -191,7 +191,8 @@ pub trait Space: 'static + SFT + Sync + Downcast { .ensure_mapped( res.start, res.pages, - self.common().mmap_strategy(), + if *self.common().options.transparent_hugepages { HugePageSupport::TransparentHugePages } else { HugePageSupport::No }, + self.common().mmap_protection(), &MmapAnnotation::Space { name: self.get_name(), }, @@ -755,20 +756,11 @@ impl CommonSpace { self.vm_map } - pub fn mmap_strategy(&self) -> MmapStrategy { - MmapStrategy { - huge_page: if *self.options.transparent_hugepages { - HugePageSupport::TransparentHugePages - } else { - HugePageSupport::No - }, - prot: if self.permission_exec || cfg!(feature = "exec_permission_on_all_spaces") { - MmapProtection::ReadWriteExec - } else { - MmapProtection::ReadWrite - }, - replace: false, - reserve: true, + pub fn mmap_protection(&self) -> MmapProtection { + if self.permission_exec || cfg!(feature = "exec_permission_on_all_spaces") { + MmapProtection::ReadWriteExec + } else { + MmapProtection::ReadWrite } } diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index 0962f7c602..2a7d25ca2a 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -146,7 +146,7 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - mut strategy: MmapStrategy, + huge_page_option: HugePageSupport, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -162,8 +162,8 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { trace!("Trying to quarantine {group_range}"); - strategy.reserve = false; - OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; + let mmap_strategy = MmapStrategy::default().huge_page(huge_page_option).prot(MmapProtection::NoAccess).reserve(false).replace(false); + OSMemory::dzmmap(group_start, group_bytes, mmap_strategy, anno)?; Ok(Some(MapState::Quarantined)) } MapState::Quarantined => { @@ -182,7 +182,8 @@ impl Mmapper for ChunkStateMmapper { &self, start: Address, pages: usize, - mut strategy: MmapStrategy, + huge_page_option: HugePageSupport, + prot: MmapProtection, anno: &MmapAnnotation, ) -> Result<()> { let _guard = self.transition_lock.lock().unwrap(); @@ -190,6 +191,8 @@ impl Mmapper for ChunkStateMmapper { let bytes = pages << LOG_BYTES_IN_PAGE; let range = ChunkRange::new_unaligned(start, bytes); + let mmap_strategy = MmapStrategy::default().huge_page(huge_page_option).prot(prot).reserve(true); + self.storage .bulk_transition_state(range, |group_range, state| { let group_start: Address = group_range.start; @@ -197,14 +200,11 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - // strategy.replace = false; - OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; + OSMemory::dzmmap(group_start, group_bytes, mmap_strategy.replace(false), anno)?; Ok(Some(MapState::Mapped)) } MapState::Quarantined => { - strategy.replace = true; - strategy.reserve = true; - OSMemory::dzmmap(group_start, group_bytes, strategy, anno)?; + OSMemory::dzmmap(group_start, group_bytes, mmap_strategy.replace(true), anno)?; Ok(Some(MapState::Mapped)) } MapState::Mapped => Ok(None), @@ -260,7 +260,7 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -288,7 +288,7 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -317,7 +317,7 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) .unwrap(); let chunks = pages_to_chunks_up(pages); diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs index 21a134f039..786b231a55 100644 --- a/src/util/heap/layout/mmapper/mod.rs +++ b/src/util/heap/layout/mmapper/mod.rs @@ -77,7 +77,7 @@ pub trait Mmapper: Sync { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, anno: &MmapAnnotation, ) -> Result<()>; @@ -97,7 +97,8 @@ pub trait Mmapper: Sync { &self, start: Address, pages: usize, - strategy: MmapStrategy, + huge_page_option: HugePageSupport, + prot: MmapProtection, anno: &MmapAnnotation, ) -> Result<()>; diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 0ccf1d7cf6..d55d53f5ff 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -1686,7 +1686,7 @@ mod tests { let data_addr = vm_layout::vm_layout().heap_start; // Make sure the address is mapped. crate::MMAPPER - .ensure_mapped(data_addr, 1, MmapStrategy::TEST, mmap_anno_test!()) + .ensure_mapped(data_addr, 1, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) .unwrap(); let meta_addr = address_to_meta_address(&spec, data_addr); with_cleanup( diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 55f6f2e58e..ad9ae20415 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -143,14 +143,15 @@ pub(super) fn try_mmap_contiguous_metadata_space( MMAPPER.ensure_mapped( mmap_start, mmap_size >> LOG_BYTES_IN_PAGE, - MmapStrategy::SIDE_METADATA, + HugePageSupport::No, + MmapProtection::ReadWrite, anno, ) } else { MMAPPER.quarantine_address_range( mmap_start, mmap_size >> LOG_BYTES_IN_PAGE, - MmapStrategy::SIDE_METADATA, + HugePageSupport::No, anno, ) } diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index b27bb5045c..a2b7288f6b 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -102,21 +102,55 @@ pub struct MmapStrategy { pub reserve: bool, } +impl std::default::Default for MmapStrategy { + fn default() -> Self { + Self { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + } + } +} + impl MmapStrategy { /// Create a new strategy - pub fn new(transparent_hugepages: bool, prot: MmapProtection, replace: bool, reserve: bool) -> Self { + pub fn new(huge_page: HugePageSupport, prot: MmapProtection, replace: bool, reserve: bool) -> Self { Self { - huge_page: if transparent_hugepages { - HugePageSupport::TransparentHugePages - } else { - HugePageSupport::No - }, + huge_page, prot, replace, reserve, } } + // Builder methods + + pub fn huge_page(self, huge_page: HugePageSupport) -> Self { + Self { huge_page, ..self } + } + + pub fn transparent_hugepages(self, enable: bool) -> Self { + let huge_page = if enable { + HugePageSupport::TransparentHugePages + } else { + HugePageSupport::No + }; + Self { huge_page, ..self } + } + + pub fn prot(self, prot: MmapProtection) -> Self { + Self { prot, ..self } + } + + pub fn replace(self, replace: bool) -> Self { + Self { replace, ..self } + } + + pub fn reserve(self, reserve: bool) -> Self { + Self { reserve, ..self } + } + /// The strategy for MMTk's own internal memory #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings. pub const INTERNAL_MEMORY: Self = Self::TEST; From 582b885ba1d497218c08e1219636af8bf2599690 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 6 Jan 2026 17:15:31 +1300 Subject: [PATCH 09/27] Get macOS to work --- src/util/os/linux.rs | 5 ++- src/util/os/macos.rs | 80 +++++++++++++++++++++++++++++++++++++ src/util/os/mod.rs | 8 +++- src/util/os/posix_common.rs | 2 +- 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 src/util/os/macos.rs diff --git a/src/util/os/linux.rs b/src/util/os/linux.rs index becdca9b35..833b65438d 100644 --- a/src/util/os/linux.rs +++ b/src/util/os/linux.rs @@ -19,7 +19,8 @@ impl Memory for LinuxMemoryImpl { Self::set_hugepage(addr, size, strategy.huge_page)?; // println!("Set huge page done"); - // Zero memory if needed + // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) + Ok(addr) } @@ -78,7 +79,7 @@ impl LinuxMemoryImpl { } impl MmapStrategy { - pub fn get_mmap_flags(&self) -> i32 { + pub fn get_posix_mmap_flags(&self) -> i32 { let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; if self.replace { flags |= libc::MAP_FIXED; diff --git a/src/util/os/macos.rs b/src/util/os/macos.rs new file mode 100644 index 0000000000..aa4c811b62 --- /dev/null +++ b/src/util/os/macos.rs @@ -0,0 +1,80 @@ +use crate::util::os::*; +use crate::util::os::posix_common; +use crate::util::address::Address; +use std::io::Result; + +pub struct MacOSMemoryImpl; + +impl Memory for MacOSMemoryImpl { + fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, _annotation: &MmapAnnotation<'_>) -> Result
{ + let addr = posix_common::mmap(start, size, strategy)?; + + // Annotation is ignored on macOS + // Huge page is ignored on macOS + + // Zero memory if we actually reserve the memory + if strategy.reserve { + Self::zero(start, size); + } + Ok(addr) + } + + fn munmap(start: Address, size: usize) -> Result<()> { + posix_common::munmap(start, size) + } + + fn mprotect(start: Address, size: usize) -> Result<()> { + posix_common::mprotect(start, size) + } + + fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + posix_common::munprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + posix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(start: Address, size: usize) { + // Do nothing for now + } +} + +impl MmapStrategy { + pub fn get_posix_mmap_flags(&self) -> i32 { + let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED; + // replace is isgnored on macOS + if !self.reserve { + flags |= libc::MAP_NORESERVE; + } + flags + } +} + +pub struct MacOSProcessImpl; + +impl Process for MacOSProcessImpl { + fn get_process_memory_maps() -> Result { + // Get the current process ID (replace this with a specific PID if needed) + let pid = std::process::id(); + + // Execute the vmmap command + let output = std::process::Command::new("vmmap") + .arg(pid.to_string()) // Pass the PID as an argument + .output() // Capture the output + .expect("Failed to execute vmmap command"); + + // Check if the command was successful + if output.status.success() { + // Convert the command output to a string + let output_str = + std::str::from_utf8(&output.stdout).expect("Failed to convert output to string"); + Ok(output_str.to_string()) + } else { + // Handle the error case + let error_message = + std::str::from_utf8(&output.stderr).expect("Failed to convert error message to string"); + Err(std::io::Error::other(format!("Failed to get process memory map: {}", error_message))) + } + } +} diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index c467882879..3ed6ec1ac9 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -7,7 +7,8 @@ pub use process::*; pub(crate) mod posix_common; #[cfg(target_os = "linux")] pub(crate) mod linux; - +#[cfg(target_os = "macos")] +pub(crate) mod macos; #[cfg(target_os = "windows")] pub(crate) mod windows; @@ -21,6 +22,11 @@ pub use linux::LinuxMemoryImpl as OSMemory; #[cfg(target_os = "linux")] pub use linux::LinuxProcessImpl as OSProcess; +#[cfg(target_os = "macos")] +pub use macos::MacOSMemoryImpl as OSMemory; +#[cfg(target_os = "macos")] +pub use macos::MacOSProcessImpl as OSProcess; + // pub trait OperatingSystem { // type OSMemory: memory::Memory; // type OSProcess: process::Process; diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index 620a07890e..45e64bbfaa 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -16,7 +16,7 @@ impl MmapProtection { pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ let ptr = start.to_mut_ptr(); let prot = strategy.prot.into_native_flags(); - let flags = strategy.get_mmap_flags(); + let flags = strategy.get_posix_mmap_flags(); wrap_libc_call( &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, ptr, From c09128d993099f52cbaf1fb0d963f850fe19bd6b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 6 Jan 2026 23:22:13 +0000 Subject: [PATCH 10/27] Add CPU affinity methods into Process --- src/scheduler/affinity.rs | 104 ++---------------------------------- src/util/options.rs | 5 +- src/util/os/linux.rs | 21 ++++++++ src/util/os/macos.rs | 20 +++++++ src/util/os/posix_common.rs | 45 +++++++++++++++- src/util/os/process.rs | 13 +++++ src/util/os/windows.rs | 40 ++++++++++++++ src/util/rust_util/mod.rs | 22 +------- 8 files changed, 145 insertions(+), 125 deletions(-) diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index 819489c128..6d4ab858b3 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -1,40 +1,6 @@ use super::worker::ThreadId; +use crate::util::os::*; use crate::util::options::AffinityKind; -#[cfg(target_os = "linux")] -use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; - -/// Represents the ID of a logical CPU on a system. -pub type CoreId = u16; - -// XXX: Maybe in the future we can use a library such as https://github.com/Elzair/core_affinity_rs -// to have an OS agnostic way of setting thread affinity. -#[cfg(target_os = "linux")] -/// Return the total number of cores allocated to the program. -pub fn get_total_num_cpus() -> u16 { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - sched_getaffinity(0, std::mem::size_of::(), &mut cs); - CPU_COUNT(&cs) as u16 - } -} - -#[cfg(target_os = "windows")] -/// Return the total number of cores allocated to the program. -pub fn get_total_num_cpus() -> u16 { - unsafe { - windows_sys::Win32::System::Threading::GetActiveProcessorCount( - windows_sys::Win32::System::Threading::ALL_PROCESSOR_GROUPS, - ) as u16 - } -} - -#[cfg(not(any(target_os = "linux", target_os = "windows")))] -/// Return the total number of cores allocated to the program. -pub fn get_total_num_cpus() -> u16 { - unimplemented!() -} impl AffinityKind { /// Resolve affinity of GC thread. Has a side-effect of calling into the kernel to set the @@ -46,77 +12,13 @@ impl AffinityKind { AffinityKind::AllInSet(cpuset) => { // Bind the current thread to all the cores in the set debug!("Set affinity for thread {} to cpuset {:?}", thread, cpuset); - bind_current_thread_to_cpuset(cpuset.as_slice()); + OSProcess::bind_current_thread_to_cpuset(cpuset.as_slice()); } AffinityKind::RoundRobin(cpuset) => { let cpu = cpuset[thread % cpuset.len()]; debug!("Set affinity for thread {} to core {}", thread, cpu); - bind_current_thread_to_core(cpu); + OSProcess::bind_current_thread_to_core(cpu); } } } } - -#[cfg(target_os = "linux")] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_core(cpu: CoreId) { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - CPU_SET(cpu as usize, &mut cs); - sched_setaffinity(0, std::mem::size_of::(), &cs); - } -} - -#[cfg(target_os = "windows")] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_core(cpu: CoreId) { - unsafe { - windows_sys::Win32::System::Threading::SetThreadAffinityMask( - windows_sys::Win32::System::Threading::GetCurrentThread(), - 1 << cpu, - ); - } -} - -#[cfg(not(any(target_os = "linux", target_os = "windows")))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_core(_cpu: CoreId) { - unimplemented!() -} - -#[cfg(any(target_os = "linux", target_os = "android"))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { - use std::mem::MaybeUninit; - unsafe { - let mut cs = MaybeUninit::zeroed().assume_init(); - CPU_ZERO(&mut cs); - for cpu in cpuset { - CPU_SET(*cpu as usize, &mut cs); - } - sched_setaffinity(0, std::mem::size_of::(), &cs); - } -} - -#[cfg(target_os = "windows")] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { - let mut mask = 0; - for cpu in cpuset { - mask |= 1 << cpu; - } - unsafe { - windows_sys::Win32::System::Threading::SetThreadAffinityMask( - windows_sys::Win32::System::Threading::GetCurrentThread(), - mask, - ); - } -} - -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "windows")))] -/// Bind the current thread to the specified core. -fn bind_current_thread_to_cpuset(_cpuset: &[CoreId]) { - unimplemented!() -} diff --git a/src/util/options.rs b/src/util/options.rs index 81f707d32f..dd8c870fdd 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,4 +1,3 @@ -use crate::scheduler::affinity::{get_total_num_cpus, CoreId}; use crate::util::constants::LOG_BYTES_IN_MBYTE; use crate::util::Address; use crate::util::os::*; @@ -456,7 +455,7 @@ impl AffinityKind { /// maximum number of cores allocated to the program. Assumes core ids on the system are /// 0-indexed. pub fn validate(&self) -> bool { - let num_cpu = get_total_num_cpus(); + let num_cpu = OSProcess::get_total_num_cpus(); if let AffinityKind::RoundRobin(cpuset) = self { for cpu in cpuset { @@ -1211,7 +1210,7 @@ mod tests { || { let mut vec = vec![0_u16]; let mut cpu_list = String::new(); - let num_cpus = get_total_num_cpus(); + let num_cpus = OSProcess::get_total_num_cpus(); cpu_list.push('0'); for cpu in 1..num_cpus { diff --git a/src/util/os/linux.rs b/src/util/os/linux.rs index 833b65438d..52c8e2a4f3 100644 --- a/src/util/os/linux.rs +++ b/src/util/os/linux.rs @@ -1,6 +1,7 @@ use crate::util::os::*; use crate::util::os::posix_common; use crate::util::address::Address; + use std::io::Result; pub struct LinuxMemoryImpl; @@ -99,4 +100,24 @@ impl Process for LinuxProcessImpl { fn get_process_memory_maps() -> Result { posix_common::get_process_memory_maps() } + + fn get_process_id() -> Result { + posix_common::get_process_id() + } + + fn get_thread_id() -> Result { + posix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + posix_common::get_total_num_cpus() + } + + fn bind_current_thread_to_core(core_id: CoreId) { + posix_common::bind_current_thread_to_core(core_id) + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + posix_common::bind_current_thread_to_cpuset(core_ids) + } } diff --git a/src/util/os/macos.rs b/src/util/os/macos.rs index aa4c811b62..458929f817 100644 --- a/src/util/os/macos.rs +++ b/src/util/os/macos.rs @@ -77,4 +77,24 @@ impl Process for MacOSProcessImpl { Err(std::io::Error::other(format!("Failed to get process memory map: {}", error_message))) } } + + fn get_process_id() -> Result { + posix_common::get_process_id() + } + + fn get_thread_id() -> Result { + posix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + posix_common::get_total_num_cpus() + } + + fn bind_current_thread_to_core(core_id: CoreId) { + posix_common::bind_current_thread_to_core(core_id) + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + posix_common::bind_current_thread_to_cpuset(core_ids) + } } diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index 45e64bbfaa..64e539238b 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -1,6 +1,8 @@ -use crate::util::os::memory::*; +use crate::util::os::*; use crate::util::address::Address; use std::io::Result; +use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; +use std::mem::MaybeUninit; impl MmapProtection { fn into_native_flags(&self) -> i32 { @@ -87,6 +89,47 @@ pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<( ) } +pub fn get_process_id() -> Result { + let pid = unsafe { libc::getpid() }; + Ok(format!("{}", pid)) +} + +pub fn get_thread_id() -> Result { + let tid = unsafe { libc::gettid() }; + Ok(format!("{}", tid)) +} + +pub fn get_total_num_cpus() -> CoreNum { + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + sched_getaffinity(0, std::mem::size_of::(), &mut cs); + CPU_COUNT(&cs) as u16 + } +} + +pub fn bind_current_thread_to_core(core_id: CoreId) { + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + CPU_SET(core_id as usize, &mut cs); + sched_setaffinity(0, std::mem::size_of::(), &cs); + } +} + +/// Bind the current thread to the specified core. +pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { + use std::mem::MaybeUninit; + unsafe { + let mut cs = MaybeUninit::zeroed().assume_init(); + CPU_ZERO(&mut cs); + for cpu in cpuset { + CPU_SET(*cpu as usize, &mut cs); + } + sched_setaffinity(0, std::mem::size_of::(), &cs); + } +} + pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { let ret = f(); if ret == expect { diff --git a/src/util/os/process.rs b/src/util/os/process.rs index eea183feb4..5f3f79e4c0 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -1,7 +1,20 @@ use std::io::Result; +pub type CoreId = u16; +pub type CoreNum = u16; + pub trait Process { /// Return error if unable to get process memory maps. /// If unimplemented, just return Ok with an string to indiate that. fn get_process_memory_maps() -> Result; + + fn get_process_id() -> Result; + + fn get_thread_id() -> Result; + + fn get_total_num_cpus() -> u16; + + fn bind_current_thread_to_core(core_id: CoreId); + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]); } diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index 3d7d2bae5e..48e3505a50 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -173,4 +173,44 @@ impl Process for WindowsProcessImpl { // Windows-specific implementation to get process memory maps Ok("get_process_memory_maps not implemented for Windows".to_string()) } + + fn get_process_id() -> Result { + use windows_sys::Win32::System::Threading::GetCurrentProcessId; + Ok(format!("{}", unsafe { GetCurrentProcessId() })) + } + + fn get_thread_id() -> Result { + use windows_sys::Win32::System::Threading::GetCurrentThreadId; + Ok(format!("{}", unsafe { GetCurrentThreadId() })) + } + + fn get_total_num_cpus() -> CoreNum { + unsafe { + windows_sys::Win32::System::Threading::GetActiveProcessorCount( + windows_sys::Win32::System::Threading::ALL_PROCESSOR_GROUPS, + ) as CoreNum + } + } + + fn bind_current_thread_to_core(core_id: CoreId) { + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + 1 << cpu, + ); + } + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + let mut mask = 0; + for cpu in cpuset { + mask |= 1 << cpu; + } + unsafe { + windows_sys::Win32::System::Threading::SetThreadAffinityMask( + windows_sys::Win32::System::Threading::GetCurrentThread(), + mask, + ); + } + } } diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index 10f57c560b..ade9baf21d 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -108,26 +108,8 @@ unsafe impl Sync for InitializeOnce {} /// Create a formatted string that makes the best effort idenfying the current process and thread. pub fn debug_process_thread_id() -> String { - #[cfg(target_os = "linux")] - { - let pid = unsafe { libc::getpid() }; - // `gettid()` is Linux-specific. - let tid = unsafe { libc::gettid() }; - format!("PID: {}, TID: {}", pid, tid) - } - #[cfg(target_os = "windows")] - { - let pid = unsafe { windows_sys::Win32::System::Threading::GetCurrentProcessId() }; - let tid = unsafe { windows_sys::Win32::System::Threading::GetCurrentThreadId() }; - format!("PID: {}, TID: {}", pid, tid) - } - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - { - let pid = unsafe { libc::getpid() }; - // TODO: When we support other platforms, use platform-specific methods to get thread - // identifiers. - format!("PID: {}", pid) - } + use crate::util::os::*; + format!("PID: {}, TID: {}", OSProcess::get_process_id().unwrap_or_default(), OSProcess::get_thread_id().unwrap_or_default()) } #[cfg(test)] From 1be26c373c78614f0aaab0b64b3b1c1667993c93 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 7 Jan 2026 00:42:12 +0000 Subject: [PATCH 11/27] Tidy up for Linux --- benches/mock_bench/mmapper.rs | 14 +- src/plan/generational/gc_work.rs | 2 +- src/policy/copyspace.rs | 2 +- src/policy/lockfreeimmortalspace.rs | 9 +- src/policy/markcompactspace.rs | 2 +- src/policy/space.rs | 6 +- src/scheduler/affinity.rs | 2 +- src/util/alloc/free_list_allocator.rs | 4 +- src/util/alloc/immix_allocator.rs | 2 +- src/util/heap/freelistpageresource.rs | 2 +- src/util/heap/layout/mmapper/csm/mod.rs | 50 +- src/util/heap/layout/mmapper/mod.rs | 5 +- src/util/malloc/malloc_ms_util.rs | 16 +- src/util/memory.rs | 890 ------------------ src/util/metadata/side_metadata/global.rs | 19 +- src/util/metadata/side_metadata/helpers.rs | 2 +- src/util/mod.rs | 2 - src/util/options.rs | 2 +- src/util/os/linux.rs | 33 +- src/util/os/macos.rs | 23 +- src/util/os/memory.rs | 66 +- src/util/os/mod.rs | 17 +- src/util/os/posix_common.rs | 16 +- src/util/os/process.rs | 16 +- src/util/os/windows.rs | 37 +- src/util/rust_util/mod.rs | 6 +- .../mock_test_handle_mmap_conflict.rs | 10 +- .../mock_tests/mock_test_handle_mmap_oom.rs | 10 +- 28 files changed, 255 insertions(+), 1010 deletions(-) delete mode 100644 src/util/memory.rs diff --git a/benches/mock_bench/mmapper.rs b/benches/mock_bench/mmapper.rs index ba0cfbb4e1..3bf0165f0e 100644 --- a/benches/mock_bench/mmapper.rs +++ b/benches/mock_bench/mmapper.rs @@ -3,8 +3,7 @@ pub use criterion::Criterion; use mmtk::{ memory_manager, mmap_anno_test, util::{ - constants::BYTES_IN_PAGE, memory::MmapStrategy, test_private::MMAPPER, - test_util::fixtures::*, Address, + constants::BYTES_IN_PAGE, os::*, test_private::MMAPPER, test_util::fixtures::*, Address, }, }; @@ -76,10 +75,17 @@ pub fn bench(c: &mut Criterion) { c.bench_function("ensure_mapped_regular", |b| { let start = regular.align_down(BYTES_IN_PAGE); assert!(start.is_mapped()); - let strategy = MmapStrategy::new(false, mmtk::util::memory::MmapProtection::ReadWrite); let anno = mmap_anno_test!(); b.iter(|| { - MMAPPER.ensure_mapped(start, 1, strategy, anno).unwrap(); + MMAPPER + .ensure_mapped( + start, + 1, + HugePageSupport::No, + MmapProtection::ReadWrite, + anno, + ) + .unwrap(); }) }); } diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index 96a7f60c38..f833796ffd 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -4,10 +4,10 @@ use crate::plan::PlanTraceObject; use crate::plan::VectorObjectQueue; use crate::policy::gc_work::TraceKind; use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage}; +use crate::util::os::*; use crate::util::ObjectReference; use crate::vm::slot::{MemorySlice, Slot}; use crate::vm::*; -use crate::util::os::*; use crate::MMTK; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 8cf1b30c4e..9a8fa7b0d7 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -10,9 +10,9 @@ use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, MetadataSpec}; use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; +use crate::util::os::*; use crate::util::{copy::*, object_enum}; use crate::util::{Address, ObjectReference}; -use crate::util::os::*; use crate::vm::*; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index e8a3f6a625..bf63855ca8 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -14,11 +14,11 @@ use crate::util::heap::gc_trigger::GCTrigger; use crate::util::heap::layout::vm_layout::vm_layout; use crate::util::heap::PageResource; use crate::util::heap::VMRequest; -use crate::util::os::*; use crate::util::metadata::side_metadata::SideMetadataContext; use crate::util::metadata::side_metadata::SideMetadataSanity; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::ObjectReference; use crate::vm::VMBinding; @@ -254,8 +254,11 @@ impl LockFreeImmortalSpace { }; // Eagerly memory map the entire heap (also zero all the memory) - let strategy = MmapStrategy::default().transparent_hugepages(*args.options.transparent_hugepages).prot( - crate::util::os::MmapProtection::ReadWrite).replace(false).reserve(true); + let strategy = MmapStrategy::default() + .transparent_hugepages(*args.options.transparent_hugepages) + .prot(crate::util::os::MmapProtection::ReadWrite) + .replace(false) + .reserve(true); crate::util::os::OSMemory::dzmmap( start, aligned_total_bytes, diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index fc78a14be5..1b928efced 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -12,8 +12,8 @@ use crate::util::copy::CopySemantics; use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, vo_bit}; use crate::util::object_enum::{self, ObjectEnumerator}; -use crate::util::{Address, ObjectReference}; use crate::util::os::*; +use crate::util::{Address, ObjectReference}; use crate::{vm::*, ObjectQueue}; use atomic::Ordering; diff --git a/src/policy/space.rs b/src/policy/space.rs index eddc81adba..6710270d84 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -191,7 +191,11 @@ pub trait Space: 'static + SFT + Sync + Downcast { .ensure_mapped( res.start, res.pages, - if *self.common().options.transparent_hugepages { HugePageSupport::TransparentHugePages } else { HugePageSupport::No }, + if *self.common().options.transparent_hugepages { + HugePageSupport::TransparentHugePages + } else { + HugePageSupport::No + }, self.common().mmap_protection(), &MmapAnnotation::Space { name: self.get_name(), diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index 6d4ab858b3..a16f909034 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -1,6 +1,6 @@ use super::worker::ThreadId; -use crate::util::os::*; use crate::util::options::AffinityKind; +use crate::util::os::*; impl AffinityKind { /// Resolve affinity of GC thread. Has a side-effect of calling into the kernel to set the diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index 6e637c8314..9485fbc1f7 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -2,15 +2,15 @@ use std::sync::Arc; +use super::allocator::AllocatorContext; use crate::policy::marksweepspace::native_ms::*; use crate::util::alloc::allocator; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; +use crate::util::os::*; use crate::util::Address; use crate::util::VMThread; use crate::vm::VMBinding; -use crate::util::os::*; -use super::allocator::AllocatorContext; /// A MiMalloc free list allocator #[repr(C)] diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index c47dbd6b26..9a8494c203 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -10,9 +10,9 @@ use crate::util::alloc::allocator::get_maximum_aligned_size; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; use crate::util::opaque_pointer::VMThread; +use crate::util::os::*; use crate::util::rust_util::unlikely; use crate::util::Address; -use crate::util::os::*; use crate::vm::*; /// Immix allocator diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index 00a75b54d1..dab5eae56e 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -14,8 +14,8 @@ use crate::util::heap::layout::vm_layout::*; use crate::util::heap::layout::CreateFreeListResult; use crate::util::heap::pageresource::CommonPageResource; use crate::util::heap::space_descriptor::SpaceDescriptor; -use crate::util::os::*; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::vm::*; use std::marker::PhantomData; diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index 2a7d25ca2a..84b15ef134 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -162,7 +162,11 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { trace!("Trying to quarantine {group_range}"); - let mmap_strategy = MmapStrategy::default().huge_page(huge_page_option).prot(MmapProtection::NoAccess).reserve(false).replace(false); + let mmap_strategy = MmapStrategy::default() + .huge_page(huge_page_option) + .prot(MmapProtection::NoAccess) + .reserve(false) + .replace(false); OSMemory::dzmmap(group_start, group_bytes, mmap_strategy, anno)?; Ok(Some(MapState::Quarantined)) } @@ -191,7 +195,10 @@ impl Mmapper for ChunkStateMmapper { let bytes = pages << LOG_BYTES_IN_PAGE; let range = ChunkRange::new_unaligned(start, bytes); - let mmap_strategy = MmapStrategy::default().huge_page(huge_page_option).prot(prot).reserve(true); + let mmap_strategy = MmapStrategy::default() + .huge_page(huge_page_option) + .prot(prot) + .reserve(true); self.storage .bulk_transition_state(range, |group_range, state| { @@ -200,11 +207,21 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - OSMemory::dzmmap(group_start, group_bytes, mmap_strategy.replace(false), anno)?; + OSMemory::dzmmap( + group_start, + group_bytes, + mmap_strategy.replace(false), + anno, + )?; Ok(Some(MapState::Mapped)) } MapState::Quarantined => { - OSMemory::dzmmap(group_start, group_bytes, mmap_strategy.replace(true), anno)?; + OSMemory::dzmmap( + group_start, + group_bytes, + mmap_strategy.replace(true), + anno, + )?; Ok(Some(MapState::Mapped)) } MapState::Mapped => Ok(None), @@ -235,7 +252,6 @@ mod tests { use super::*; use crate::mmap_anno_test; use crate::util::constants::LOG_BYTES_IN_PAGE; - use crate::util::os::*; use crate::util::test_util::CHUNK_STATE_MMAPPER_TEST_REGION; use crate::util::test_util::{serial_test, with_cleanup}; use crate::util::{conversions, Address}; @@ -260,7 +276,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -288,7 +310,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); @@ -317,7 +345,13 @@ mod tests { || { let mmapper = ChunkStateMmapper::new(); mmapper - .ensure_mapped(FIXED_ADDRESS, pages, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) + .ensure_mapped( + FIXED_ADDRESS, + pages, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let chunks = pages_to_chunks_up(pages); diff --git a/src/util/heap/layout/mmapper/mod.rs b/src/util/heap/layout/mmapper/mod.rs index 786b231a55..580b2e26b1 100644 --- a/src/util/heap/layout/mmapper/mod.rs +++ b/src/util/heap/layout/mmapper/mod.rs @@ -1,7 +1,4 @@ -use crate::util::{ - os::*, - Address, -}; +use crate::util::{os::*, Address}; use std::io::Result; #[allow(unused)] // Used in doc comment. diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 5e99211123..1feb495c7f 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -1,17 +1,18 @@ use crate::util::constants::BYTES_IN_ADDRESS; use crate::util::malloc::library::*; use crate::util::Address; -use crate::util::os::*; use crate::vm::VMBinding; /// Allocate with alignment. This also guarantees the memory is zero initialized. /// This uses posix_memalign, which is not available on Windows. /// This would somehow affect `MallocMarkSweep` performance on Windows. -#[cfg(all( - not(target_os = "windows"), - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -))] +// #[cfg(all( +// not(target_os = "windows"), +// not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) +// ))] +#[cfg(not(target_os = "windows"))] pub fn align_alloc(size: usize, align: usize) -> Address { + use crate::util::os::*; let mut ptr = std::ptr::null_mut::(); let ptr_ptr = std::ptr::addr_of_mut!(ptr); let result = unsafe { posix_memalign(ptr_ptr, align, size) }; @@ -85,10 +86,7 @@ pub fn alloc(size: usize, align: usize, offset: usize) -> (Addres address = Address::from_mut_ptr(raw); debug_assert!(address.is_aligned_to(align)); } - #[cfg(all( - not(target_os = "windows"), - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) - ))] + #[cfg(not(target_os = "windows"))] // On non-Windows platforms with posix_memalign, we can use align_alloc for alignments > 16 // However, on Windows, there is no equivalent function. // The memory alloc by `align_alloc` may not be freed correctly by `free`. diff --git a/src/util/memory.rs b/src/util/memory.rs deleted file mode 100644 index 706577181f..0000000000 --- a/src/util/memory.rs +++ /dev/null @@ -1,890 +0,0 @@ -use crate::util::alloc::AllocationError; -use crate::util::opaque_pointer::*; -use crate::util::Address; -use crate::vm::{Collection, VMBinding}; -use bytemuck::NoUninit; -#[cfg(not(target_os = "windows"))] -use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; -use std::io::{Error, Result}; -use sysinfo::MemoryRefreshKind; -use sysinfo::{RefreshKind, System}; - -#[cfg(target_os = "linux")] -// MAP_FIXED_NOREPLACE returns EEXIST if already mapped -const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED_NOREPLACE; -#[cfg(target_os = "macos")] -// MAP_FIXED is used instead of MAP_FIXED_NOREPLACE (which is not available on macOS). We are at the risk of overwriting pre-existing mappings. -const MMAP_FLAGS: libc::c_int = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; -#[cfg(target_os = "windows")] -const MMAP_FLAGS: libc::c_int = 0; // Not used on Windows -#[cfg(target_os = "windows")] -const MAP_NORESERVE: libc::c_int = 0x4000; // Custom flag for Windows emulation - -/// Strategy for performing mmap -#[derive(Debug, Copy, Clone)] -pub struct MmapStrategy { - /// Do we support huge pages? - pub huge_page: HugePageSupport, - /// The protection flags for mmap - pub prot: MmapProtection, -} - -impl MmapStrategy { - /// Create a new strategy - pub fn new(transparent_hugepages: bool, prot: MmapProtection) -> Self { - Self { - huge_page: if transparent_hugepages { - HugePageSupport::TransparentHugePages - } else { - HugePageSupport::No - }, - prot, - } - } - - /// The strategy for MMTk's own internal memory - pub const INTERNAL_MEMORY: Self = Self { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - }; - - /// The strategy for MMTk side metadata - pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; - - /// The strategy for MMTk's test memory - #[cfg(test)] - pub const TEST: Self = Self::INTERNAL_MEMORY; -} - -/// The protection flags for Mmap -#[repr(i32)] -#[derive(Debug, Copy, Clone)] -pub enum MmapProtection { - /// Allow read + write - ReadWrite, - /// Allow read + write + code execution - ReadWriteExec, - /// Do not allow any access - NoAccess, -} - -impl MmapProtection { - /// Turn the protection enum into the native flags on non-Windows platforms - #[cfg(not(target_os = "windows"))] - pub fn into_native_flags(self) -> libc::c_int { - match self { - Self::ReadWrite => PROT_READ | PROT_WRITE, - Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, - Self::NoAccess => PROT_NONE, - } - } - - /// Turn the protection enum into the native flags on Windows platforms - #[cfg(target_os = "windows")] - pub fn into_native_flags(self) -> u32 { - use windows_sys::Win32::System::Memory::*; - match self { - Self::ReadWrite => PAGE_READWRITE, - Self::ReadWriteExec => PAGE_EXECUTE_READWRITE, - Self::NoAccess => PAGE_NOACCESS, - } - } -} - -/// Support for huge pages -#[repr(u8)] -#[derive(Debug, Copy, Clone, NoUninit)] -pub enum HugePageSupport { - /// No support for huge page - No, - /// Enable transparent huge pages for the pages that are mapped. This option is only for linux. - TransparentHugePages, -} - -/// Annotation for an mmap entry. -/// -/// Invocations of `mmap_fixed` and other functions that may transitively call `mmap_fixed` -/// require an annotation that indicates the purpose of the memory mapping. -/// -/// This is for debugging. On Linux, mmtk-core will use `prctl` with `PR_SET_VMA` to set the -/// human-readable name for the given mmap region. The annotation is ignored on other platforms. -/// -/// Note that when using `Map32` (even when running on 64-bit architectures), the discontiguous -/// memory range is shared between different spaces. Spaces may use `mmap` to map new chunks, but -/// the same chunk may later be reused by other spaces. The annotation only applies when `mmap` is -/// called for a chunk for the first time, which reflects which space first attempted the mmap, not -/// which space is currently using the chunk. Use `crate::policy::space::print_vm_map` to print a -/// more accurate mapping between address ranges and spaces. -/// -/// On 32-bit architecture, side metadata are allocated in a chunked fasion. One single `mmap` -/// region will contain many different metadata. In that case, we simply annotate the whole region -/// with a `MmapAnnotation::SideMeta` where `meta` is `"all"`. -pub enum MmapAnnotation<'a> { - /// The mmap is for a space. - Space { - /// The name of the space. - name: &'a str, - }, - /// The mmap is for a side metadata. - SideMeta { - /// The name of the space. - space: &'a str, - /// The name of the side metadata. - meta: &'a str, - }, - /// The mmap is for a test case. Usually constructed using the [`mmap_anno_test!`] macro. - Test { - /// The source file. - file: &'a str, - /// The line number. - line: u32, - }, - /// For all other use cases. - Misc { - /// A human-readable descriptive name. - name: &'a str, - }, -} - -/// Construct an `MmapAnnotation::Test` with the current file name and line number. -#[macro_export] -macro_rules! mmap_anno_test_unused { - () => { - &$crate::util::os::MmapAnnotation::Test { - file: file!(), - line: line!(), - } - }; -} - -// Export this to external crates -pub use mmap_anno_test_unused; - -impl std::fmt::Display for MmapAnnotation<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - MmapAnnotation::Space { name } => write!(f, "mmtk:space:{name}"), - MmapAnnotation::SideMeta { space, meta } => write!(f, "mmtk:sidemeta:{space}:{meta}"), - MmapAnnotation::Test { file, line } => write!(f, "mmtk:test:{file}:{line}"), - MmapAnnotation::Misc { name } => write!(f, "mmtk:misc:{name}"), - } - } -} - -/// Check the result from an mmap function in this module. -/// Return true if the mmap has failed due to an existing conflicting mapping. -pub(crate) fn result_is_mapped(result: Result<()>) -> bool { - #[cfg(not(target_os = "windows"))] - match result { - Ok(_) => false, - Err(err) => err.raw_os_error().unwrap() == libc::EEXIST, - } - #[cfg(target_os = "windows")] - match result { - Ok(_) => false, - Err(err) => { - // ERROR_INVALID_ADDRESS may be returned if the address is already mapped or invalid - err.raw_os_error().unwrap() - == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 - } - } -} - -/// Set a range of memory to 0. -pub fn zero(start: Address, len: usize) { - set(start, 0, len); -} - -/// Set a range of memory to the given value. Similar to memset. -pub fn set(start: Address, val: u8, len: usize) { - unsafe { - std::ptr::write_bytes::(start.to_mut_ptr(), val, len); - } -} - -/// Demand-zero mmap: -/// This function mmaps the memory and guarantees to zero all mapped memory. -/// This function WILL overwrite existing memory mapping. The user of this function -/// needs to be aware of this, and use it cautiously. -/// -/// # Safety -/// This function WILL overwrite existing memory mapping if there is any. So only use this function if you know -/// the memory has been reserved by mmtk (e.g. after the use of mmap_noreserve()). Otherwise using this function -/// may corrupt others' data. -#[allow(clippy::let_and_return)] // Zeroing is not neceesary for some OS/s -pub unsafe fn dzmmap( - start: Address, - size: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - #[cfg(not(target_os = "windows"))] - let flags = libc::MAP_ANON | libc::MAP_PRIVATE | libc::MAP_FIXED; - #[cfg(target_os = "windows")] - let flags = 0; // Not used - let ret = mmap_fixed(start, size, flags, strategy, anno); - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - // On Windows, MEM_COMMIT guarantees zero-initialized pages. - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - if ret.is_ok() { - zero(start, size) - } - ret -} -/// Demand-zero mmap (no replace): -/// This function mmaps the memory and guarantees to zero all mapped memory. -/// This function will not overwrite existing memory mapping, and it will result Err if there is an existing mapping. -#[allow(clippy::let_and_return)] // Zeroing is not neceesary for some OS/s -pub fn dzmmap_noreplace( - start: Address, - size: usize, - strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - #[cfg(not(target_os = "windows"))] - let flags = MMAP_FLAGS; - #[cfg(target_os = "windows")] - let flags = 0; // Not used - - let ret = mmap_fixed(start, size, flags, strategy, anno); - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - // On Windows, MEM_COMMIT guarantees zero-initialized pages. - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - if ret.is_ok() { - zero(start, size) - } - ret -} - -/// mmap with no swap space reserve: -/// This function does not reserve swap space for this mapping, which means there is no guarantee that writes to the -/// mapping can always be successful. In case of out of physical memory, one may get a segfault for writing to the mapping. -/// We can use this to reserve the address range, and then later overwrites the mapping with dzmmap(). -pub fn mmap_noreserve( - start: Address, - size: usize, - mut strategy: MmapStrategy, - anno: &MmapAnnotation, -) -> Result<()> { - strategy.prot = MmapProtection::NoAccess; - #[cfg(not(target_os = "windows"))] - let flags = MMAP_FLAGS | libc::MAP_NORESERVE; - #[cfg(target_os = "windows")] - let flags = MAP_NORESERVE; - mmap_fixed(start, size, flags, strategy, anno) -} - -fn mmap_fixed( - start: Address, - size: usize, - flags: libc::c_int, - strategy: MmapStrategy, - _anno: &MmapAnnotation, -) -> Result<()> { - #[cfg(not(target_os = "windows"))] - { - let ptr = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, - ptr, - )?; - - #[cfg(all( - any(target_os = "linux", target_os = "android"), - not(feature = "no_mmap_annotation") - ))] - { - // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that - // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not - // recognize this attribute and will return `EINVAL`. However, `prctl` may return `EINVAL` - // for other reasons, too. That includes `start` being an invalid address, and the - // formatted `anno_cstr` being longer than 80 bytes including the trailing `'\0'`. But - // since this prctl is used for debugging, we log the error instead of panicking. - let anno_str = _anno.to_string(); - let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( - &|| unsafe { - libc::prctl( - libc::PR_SET_VMA, - libc::PR_SET_VMA_ANON_NAME, - start.to_ptr::(), - size, - anno_cstr.as_ptr(), - ) - }, - 0, - ); - if let Err(e) = result { - debug!("Error while calling prctl: {e}"); - } - } - - match strategy.huge_page { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - #[cfg(target_os = "linux")] - { - wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) - } - // Setting the transparent hugepage option to true will not pass - // the validation on non-Linux OSes - #[cfg(not(target_os = "linux"))] - unreachable!() - } - } - } - - #[cfg(target_os = "windows")] - { - use std::io; - use windows_sys::Win32::System::Memory::{ - VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, - }; - - let ptr: *mut u8 = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); - - // Has to COMMIT inmediately if: - // - not MAP_NORESERVE - // - and protection is not NoAccess - let commit = - (flags & MAP_NORESERVE) == 0 && !matches!(strategy.prot, MmapProtection::NoAccess); - - // Scan the region [ptr, ptr + size) to understand its current state - unsafe { - let mut addr = ptr; - let end = ptr.add(size); - - let mut saw_free = false; - let mut saw_reserved = false; - let mut saw_committed = false; - - while addr < end { - let mut mbi: MEMORY_BASIC_INFORMATION = std::mem::zeroed(); - let q = VirtualQuery( - addr as *const _, - &mut mbi, - std::mem::size_of::(), - ); - if q == 0 { - return Err(io::Error::last_os_error()); - } - - let region_base = mbi.BaseAddress as *mut u8; - let region_size = mbi.RegionSize; - let region_end = region_base.add(region_size); - - // Calculate the intersection of [addr, end) and [region_base, region_end) - let _sub_begin = if addr > region_base { - addr - } else { - region_base - }; - let _sub_end = if end < region_end { end } else { region_end }; - - match mbi.State { - MEM_FREE => saw_free = true, - MEM_RESERVE => saw_reserved = true, - MEM_COMMIT => saw_committed = true, - _ => { - return Err(io::Error::other("Unexpected memory state in mmap_fixed")); - } - } - - // Jump to the next region (VirtualQuery always returns "continuous regions with the same attributes") - addr = region_end; - } - - // 1. All FREE: make a new mapping in the region - // 2. All RESERVE/COMMIT: treat as an existing mapping, can just COMMIT or succeed directly - // 3. MIX of FREE + others: not allowed (semantically similar to MAP_FIXED_NOREPLACE) - if saw_free && (saw_reserved || saw_committed) { - return Err(io::Error::from_raw_os_error( - windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32, - )); - } - - if saw_free && !saw_reserved && !saw_committed { - // All FREE: make a new mapping in the region - let mut allocation_type = MEM_RESERVE; - if commit { - allocation_type |= MEM_COMMIT; - } - - let res = VirtualAlloc(ptr as *mut _, size, allocation_type, prot); - if res.is_null() { - return Err(io::Error::last_os_error()); - } - - Ok(()) - } else { - // This behavior is similar to mmap with MAP_FIXED on Linux. - // If the region is already mapped, we just ensure the required commitment. - // If commit is not needed, we just return Ok. - if commit { - let res = VirtualAlloc(ptr as *mut _, size, MEM_COMMIT, prot); - if res.is_null() { - return Err(io::Error::last_os_error()); - } - } - Ok(()) - } - } - } -} - -/// Unmap the given memory (in page granularity). This wraps the unsafe libc munmap call. -pub fn munmap(start: Address, size: usize) -> Result<()> { - #[cfg(not(target_os = "windows"))] - return wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0); - - #[cfg(target_os = "windows")] - { - use windows_sys::Win32::System::Memory::*; - // Using MEM_DECOMMIT will decommit the memory but leave the address space reserved. - // This is the safest way to emulate munmap on Windows, as MEM_RELEASE would free - // the entire allocation, which could be larger than the requested size. - let res = unsafe { VirtualFree(start.to_mut_ptr(), size, MEM_DECOMMIT) }; - if res == 0 { - // If decommit fails, we try to release the memory. This might happen if the memory was - // only reserved. - let res_release = unsafe { VirtualFree(start.to_mut_ptr(), 0, MEM_RELEASE) }; - if res_release == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } else { - Ok(()) - } - } -} - -/// Properly handle errors from a mmap Result, including invoking the binding code in the case of -/// an OOM error. -pub fn handle_mmap_error( - error: Error, - tls: VMThread, - addr: Address, - bytes: usize, -) -> ! { - use std::io::ErrorKind; - - eprintln!("Failed to mmap {}, size {}", addr, bytes); - eprintln!("{}", get_process_memory_maps()); - - match error.kind() { - // From Rust nightly 2021-05-12, we started to see Rust added this ErrorKind. - ErrorKind::OutOfMemory => { - // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - // Before Rust had ErrorKind::OutOfMemory, this is how we capture OOM from OS calls. - // TODO: We may be able to remove this now. - ErrorKind::Other => { - // further check the error - if let Some(os_errno) = error.raw_os_error() { - // If it is OOM, we invoke out_of_memory() through the VM interface. - #[cfg(not(target_os = "windows"))] - if os_errno == libc::ENOMEM { - // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - #[cfg(target_os = "windows")] - if os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 { - // ERROR_NOT_ENOUGH_MEMORY - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - } - } - ErrorKind::AlreadyExists => { - panic!("Failed to mmap, the address is already mapped. Should MMTk quarantine the address range first?"); - } - _ => { - #[cfg(target_os = "windows")] - if let Some(os_errno) = error.raw_os_error() { - // If it is invalid address, we provide a more specific panic message. - if os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 { - // ERROR_INVALID_ADDRESS - trace!("Signal MmapOutOfMemory!"); - VM::VMCollection::out_of_memory(tls, AllocationError::MmapOutOfMemory); - unreachable!() - } - } - } - } - panic!("Unexpected mmap failure: {:?}", error) -} - -/// Checks if the memory has already been mapped. If not, we panic. -/// -/// Note that the checking has a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped. -/// Be very careful about using this function. -/// -/// This function is currently left empty for non-linux, and should be implemented in the future. -/// As the function is only used for assertions, MMTk will still run even if we never panic. -pub(crate) fn panic_if_unmapped(_start: Address, _size: usize, _anno: &MmapAnnotation) { - #[cfg(target_os = "linux")] - { - let flags = MMAP_FLAGS; - match mmap_fixed( - _start, - _size, - flags, - MmapStrategy { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - }, - _anno, - ) { - Ok(_) => panic!("{} of size {} is not mapped", _start, _size), - Err(e) => { - assert!( - e.kind() == std::io::ErrorKind::AlreadyExists, - "Failed to check mapped: {:?}", - e - ); - } - } - } -} - -/// Unprotect the given memory (in page granularity) to allow access (PROT_READ/WRITE/EXEC). -pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - #[cfg(not(target_os = "windows"))] - { - let prot = prot.into_native_flags(); - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) - } - #[cfg(target_os = "windows")] - { - use windows_sys::Win32::System::Memory::*; - let prot = prot.into_native_flags(); - let mut old_protect = 0; - let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; - if res == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } -} - -/// Protect the given memory (in page granularity) to forbid any access (PROT_NONE). -pub fn mprotect(start: Address, size: usize) -> Result<()> { - #[cfg(not(target_os = "windows"))] - { - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, PROT_NONE) }, - 0, - ) - } - #[cfg(target_os = "windows")] - { - use windows_sys::Win32::System::Memory::*; - let mut old_protect = 0; - let res = - unsafe { VirtualProtect(start.to_mut_ptr(), size, PAGE_NOACCESS, &mut old_protect) }; - if res == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } -} - -fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { - let ret = f(); - if ret == expect { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn get_process_memory_maps() -> String { - // print map - use std::fs::File; - use std::io::Read; - let mut data = String::new(); - let mut f = File::open("/proc/self/maps").unwrap(); - f.read_to_string(&mut data).unwrap(); - data -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(target_os = "macos")] -pub fn get_process_memory_maps() -> String { - // Get the current process ID (replace this with a specific PID if needed) - let pid = std::process::id(); - - // Execute the vmmap command - let output = std::process::Command::new("vmmap") - .arg(pid.to_string()) // Pass the PID as an argument - .output() // Capture the output - .expect("Failed to execute vmmap command"); - - // Check if the command was successful - if output.status.success() { - // Convert the command output to a string - let output_str = - std::str::from_utf8(&output.stdout).expect("Failed to convert output to string"); - output_str.to_string() - } else { - // Handle the error case - let error_message = - std::str::from_utf8(&output.stderr).expect("Failed to convert error message to string"); - panic!("Failed to get process memory map: {}", error_message) - } -} - -/// Get the memory maps for the process. The returned string is a multi-line string. -/// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(not(any(target_os = "linux", target_os = "android", target_os = "macos")))] -pub fn get_process_memory_maps() -> String { - "(process map unavailable)".to_string() -} - -/// Returns the total physical memory for the system in bytes. -pub(crate) fn get_system_total_memory() -> u64 { - // TODO: Note that if we want to get system info somewhere else in the future, we should - // refactor this instance into some global struct. sysinfo recommends sharing one instance of - // `System` instead of making multiple instances. - // See https://docs.rs/sysinfo/0.29.0/sysinfo/index.html#usage for more info - // - // If we refactor the `System` instance to use it for other purposes, please make sure start-up - // time is not affected. It takes a long time to load all components in sysinfo (e.g. by using - // `System::new_all()`). Some applications, especially short-running scripts, are sensitive to - // start-up time. During start-up, MMTk core only needs the total memory to initialize the - // `Options`. If we only load memory-related components on start-up, it should only take <1ms - // to initialize the `System` instance. - let sys = System::new_with_specifics( - RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram()), - ); - sys.total_memory() -} - -#[cfg(test)] -mod tests { - use crate::util::os::*; - use crate::util::Address; - use crate::util::constants::BYTES_IN_PAGE; - use crate::util::test_util::MEMORY_TEST_REGION; - use crate::util::test_util::{serial_test, with_cleanup}; - - // In the tests, we will mmap this address. This address should not be in our heap (in case we mess up with other tests) - const START: Address = MEMORY_TEST_REGION.start; - - #[test] - fn test_mmap() { - serial_test(|| { - with_cleanup( - || { - let res = unsafe { - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - // We can overwrite with dzmmap - let res = unsafe { - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { replace: true, ..MmapStrategy::TEST }, mmap_anno_test!()) - }; - assert!(res.is_ok()); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ); - }); - } - - #[test] - fn test_munmap() { - serial_test(|| { - with_cleanup( - || { - let res = OSMemory::dzmmap( - START, - BYTES_IN_PAGE, - MmapStrategy { - replace: false, - ..MmapStrategy::TEST - }, - mmap_anno_test!(), - ); - assert!(res.is_ok()); - let res = OSMemory::munmap(START, BYTES_IN_PAGE); - assert!(res.is_ok()); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - fn test_mmap_noreplace() { - serial_test(|| { - with_cleanup( - || { - // Make sure we mmapped the memory - let res = unsafe { - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy::TEST, mmap_anno_test!()) - }; - assert!(res.is_ok()); - // Use dzmmap_noreplace will fail - let res = OSMemory::dzmmap( - START, - BYTES_IN_PAGE, - MmapStrategy { - replace: false, - ..MmapStrategy::TEST - }, - mmap_anno_test!(), - ); - assert!(res.is_err()); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }); - } - - #[test] - fn test_mmap_noreserve() { - serial_test(|| { - with_cleanup( - || { - let res = - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { reserve: false, ..MmapStrategy::TEST }, mmap_anno_test!()); - assert!(res.is_ok()); - // Try reserve it - let res = unsafe { - OSMemory::dzmmap(START, BYTES_IN_PAGE, MmapStrategy { replace: true, ..MmapStrategy::TEST }, mmap_anno_test!()) - }; - assert!(res.is_ok()); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - #[should_panic] - fn test_check_is_mmapped_for_unmapped() { - serial_test(|| { - with_cleanup( - || { - // We expect this call to panic - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[test] - fn test_check_is_mmapped_for_mapped() { - serial_test(|| { - with_cleanup( - || { - assert!(OSMemory::dzmmap( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!() - ) - .is_ok()); - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE).is_ok()); - }, - ) - }) - } - - #[cfg(target_os = "linux")] - #[test] - #[should_panic] - fn test_check_is_mmapped_for_unmapped_next_to_mapped() { - serial_test(|| { - with_cleanup( - || { - // map 1 page from START - assert!(OSMemory::dzmmap( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!(), - ) - .is_ok()); - - // check if the next page is mapped - which should panic - OSMemory::panic_if_unmapped(START + BYTES_IN_PAGE, BYTES_IN_PAGE); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); - }, - ) - }) - } - - #[test] - #[should_panic] - // This is a bug we need to fix. We need to figure out a way to properly check if a piece of memory is mapped or not. - // Alternatively, we should remove the code that calls the function. - #[ignore] - fn test_check_is_mmapped_for_partial_mapped() { - serial_test(|| { - with_cleanup( - || { - // map 1 page from START - assert!(OSMemory::dzmmap( - START, - BYTES_IN_PAGE, - MmapStrategy::TEST, - mmap_anno_test!() - ) - .is_ok()); - - // check if the 2 pages from START are mapped. The second page is unmapped, so it should panic. - OSMemory::panic_if_unmapped(START, BYTES_IN_PAGE * 2); - }, - || { - assert!(OSMemory::munmap(START, BYTES_IN_PAGE * 2).is_ok()); - }, - ) - }) - } - - #[test] - fn test_get_system_total_memory() { - let total = OSMemory::get_system_total_memory().unwrap(); - println!("Total memory: {:?}", total); - } -} diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index d55d53f5ff..dcf6756a6b 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -2,10 +2,10 @@ use super::*; use crate::util::constants::{BYTES_IN_PAGE, BYTES_IN_WORD, LOG_BITS_IN_BYTE}; use crate::util::conversions::raw_align_up; use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK; -use crate::util::os::*; use crate::util::metadata::metadata_val_traits::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_SPEC; +use crate::util::os::*; use crate::util::Address; use num_traits::FromPrimitive; use ranges::BitByteRange; @@ -122,10 +122,7 @@ impl SideMetadataSpec { meta_start ); - OSMemory::panic_if_unmapped( - meta_start, - BYTES_IN_PAGE - ); + OSMemory::panic_if_unmapped(meta_start, BYTES_IN_PAGE); } #[cfg(debug_assertions)] @@ -1659,7 +1656,6 @@ mod tests { use crate::util::heap::layout::vm_layout; use crate::util::test_util::{serial_test, with_cleanup}; - use crate::util::os::*; use paste::paste; const TEST_LOG_BYTES_IN_REGION: usize = 12; @@ -1686,15 +1682,20 @@ mod tests { let data_addr = vm_layout::vm_layout().heap_start; // Make sure the address is mapped. crate::MMAPPER - .ensure_mapped(data_addr, 1, HugePageSupport::No, MmapProtection::ReadWrite, mmap_anno_test!()) + .ensure_mapped( + data_addr, + 1, + HugePageSupport::No, + MmapProtection::ReadWrite, + mmap_anno_test!(), + ) .unwrap(); let meta_addr = address_to_meta_address(&spec, data_addr); with_cleanup( || { let mmap_result = context.try_map_metadata_space(data_addr, BYTES_IN_PAGE, "test_space"); - let _ = mmap_result.unwrap(); - // assert!(mmap_result.is_ok()); + assert!(mmap_result.is_ok(), "{:?}", mmap_result); f(&spec, data_addr, meta_addr); }, diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index ad9ae20415..d14eee01d9 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -4,9 +4,9 @@ use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::{BITS_IN_WORD, BYTES_IN_PAGE, LOG_BITS_IN_BYTE}; use crate::util::conversions::rshift_align_up; use crate::util::heap::layout::vm_layout::VMLayout; -use crate::util::os::*; #[cfg(target_pointer_width = "32")] use crate::util::metadata::side_metadata::address_to_chunked_meta_address; +use crate::util::os::*; use crate::util::Address; use crate::MMAPPER; use std::io::Result; diff --git a/src/util/mod.rs b/src/util/mod.rs index a12a8ff6c7..30f6269fc0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -28,8 +28,6 @@ pub mod is_mmtk_object; pub mod linear_scan; /// Various malloc implementations (conditionally compiled by features) pub mod malloc; -/// Wrapper functions for memory syscalls such as mmap, mprotect, etc. -pub mod memory; /// Metadata (OnSide or InHeader) implementation. pub mod metadata; /// Opaque pointers used in MMTk, e.g. VMThread. diff --git a/src/util/options.rs b/src/util/options.rs index dd8c870fdd..40b3add07e 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,6 +1,6 @@ use crate::util::constants::LOG_BYTES_IN_MBYTE; -use crate::util::Address; use crate::util::os::*; +use crate::util::Address; use std::default::Default; use std::fmt::Debug; use std::str::FromStr; diff --git a/src/util/os/linux.rs b/src/util/os/linux.rs index 52c8e2a4f3..a90aaf925a 100644 --- a/src/util/os/linux.rs +++ b/src/util/os/linux.rs @@ -1,19 +1,25 @@ -use crate::util::os::*; -use crate::util::os::posix_common; use crate::util::address::Address; +use crate::util::os::posix_common; +use crate::util::os::*; use std::io::Result; +/// Linux implementation of the `Memory` trait. pub struct LinuxMemoryImpl; impl Memory for LinuxMemoryImpl { - fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
{ + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
{ // println!("Mmap with strategy: {:?}", strategy); let addr = posix_common::mmap(start, size, strategy)?; // println!("Mmap done"); if !cfg!(feature = "no_mmap_annotation") { - posix_common::set_vma_name(addr, size, annotation); + posix_common::set_vma_name(addr, size, annotation); // println!("Set annotation done"); } @@ -48,11 +54,7 @@ impl Memory for LinuxMemoryImpl { replace: false, reserve: true, }; - match posix_common::mmap( - start, - size, - strategy, - ) { + match posix_common::mmap(start, size, strategy) { Ok(_) => panic!("{} of size {} is not mapped", start, size), Err(e) => { assert!( @@ -66,20 +68,20 @@ impl Memory for LinuxMemoryImpl { } impl LinuxMemoryImpl { + /// Set huge page option for the given memory. pub fn set_hugepage(start: Address, size: usize, options: HugePageSupport) -> Result<()> { match options { HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => { - posix_common::wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ) - } + HugePageSupport::TransparentHugePages => posix_common::wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ), } } } impl MmapStrategy { + /// get the flags for POSIX mmap. pub fn get_posix_mmap_flags(&self) -> i32 { let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; if self.replace { @@ -94,6 +96,7 @@ impl MmapStrategy { } } +/// Linux implementation of the `Process` trait. pub struct LinuxProcessImpl; impl Process for LinuxProcessImpl { diff --git a/src/util/os/macos.rs b/src/util/os/macos.rs index 458929f817..a0e18e24b1 100644 --- a/src/util/os/macos.rs +++ b/src/util/os/macos.rs @@ -1,12 +1,18 @@ -use crate::util::os::*; -use crate::util::os::posix_common; use crate::util::address::Address; +use crate::util::os::posix_common; +use crate::util::os::*; use std::io::Result; +/// MacOS implementation of the `Memory` trait. pub struct MacOSMemoryImpl; impl Memory for MacOSMemoryImpl { - fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, _annotation: &MmapAnnotation<'_>) -> Result
{ + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + _annotation: &MmapAnnotation<'_>, + ) -> Result
{ let addr = posix_common::mmap(start, size, strategy)?; // Annotation is ignored on macOS @@ -41,6 +47,7 @@ impl Memory for MacOSMemoryImpl { } impl MmapStrategy { + /// get the flags for POSIX mmap. pub fn get_posix_mmap_flags(&self) -> i32 { let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED; // replace is isgnored on macOS @@ -51,6 +58,7 @@ impl MmapStrategy { } } +/// MacOS implementation of the `Process` trait. pub struct MacOSProcessImpl; impl Process for MacOSProcessImpl { @@ -72,9 +80,12 @@ impl Process for MacOSProcessImpl { Ok(output_str.to_string()) } else { // Handle the error case - let error_message = - std::str::from_utf8(&output.stderr).expect("Failed to convert error message to string"); - Err(std::io::Error::other(format!("Failed to get process memory map: {}", error_message))) + let error_message = std::str::from_utf8(&output.stderr) + .expect("Failed to convert error message to string"); + Err(std::io::Error::other(format!( + "Failed to get process memory map: {}", + error_message + ))) } } diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index a2b7288f6b..9ed3884467 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -1,29 +1,46 @@ use bytemuck::NoUninit; use std::io::Result; -use crate::{util::{VMThread, address::Address}, vm::VMBinding}; use crate::util::os::*; use crate::vm::*; +use crate::{ + util::{address::Address, VMThread}, + vm::VMBinding, +}; +/// Abstraction for OS memory operations. pub trait Memory { + /// Set a memory region to zero. fn zero(start: Address, len: usize) { Self::set(start, 0, len); } + + /// Set a memory region to a specific value. fn set(start: Address, val: u8, len: usize) { unsafe { std::ptr::write_bytes::(start.to_mut_ptr(), val, len); } } - fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, annotation: &MmapAnnotation<'_>) -> Result
; + /// Perform a demand-zero mmap. + /// + /// Falback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
; + + /// Handle mmap errors, possibly by signaling the VM about an out-of-memory condition. fn handle_mmap_error( error: std::io::Error, tls: VMThread, addr: Address, bytes: usize, ) { - use std::io::ErrorKind; use crate::util::alloc::AllocationError; + use std::io::ErrorKind; eprintln!("Failed to mmap {}, size {}", addr, bytes); eprintln!("{}", OSProcess::get_process_memory_maps().unwrap()); @@ -64,11 +81,27 @@ pub trait Memory { panic!("Unexpected mmap failure: {:?}", error) } + /// Check whether the given OS error number indicates an out-of-memory condition. fn is_mmap_oom(os_errno: i32) -> bool; + + /// Unmap a memory region. fn munmap(start: Address, size: usize) -> Result<()>; + + /// Change the protection of a memory region to no access to forbit any access to the memory region. fn mprotect(start: Address, size: usize) -> Result<()>; + + /// Change the protection of a memory region to the specified protection. fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()>; + + /// Checks if the memory has already been mapped. If not, we panic. + /// + /// Note that the checking may have a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped. + /// Be very careful about using this function. + /// + /// Fallback: As the function is only used for assertions, it can be a no-op, and MMTk will still run and never panics in this function. fn panic_if_unmapped(start: Address, size: usize); + + /// Get the total memory of the system in bytes. fn get_system_total_memory() -> Result { use sysinfo::MemoryRefreshKind; use sysinfo::{RefreshKind, System}; @@ -94,11 +127,16 @@ pub trait Memory { /// Strategy for performing mmap #[derive(Debug, Copy, Clone)] pub struct MmapStrategy { - /// Do we support huge pages? + /// Whether we should use huge page for this mmapping. + /// Fallback: for platforms that do not support huge pages, this option can be ignored. pub huge_page: HugePageSupport, - /// The protection flags for mmap + /// The protection flags for mmap. pub prot: MmapProtection, + /// Whether this mmap allows replacing existing mappings. + /// Fallback: for platforms that cannot replace existing mappings, or always replace existing mappings, this option can be ignored. pub replace: bool, + /// Whether this mmap allows reserve/commit physical memory. + /// This has to be implemented properly for a platform. Otherwise, we will see huge unrealistic memory consumption. pub reserve: bool, } @@ -115,7 +153,12 @@ impl std::default::Default for MmapStrategy { impl MmapStrategy { /// Create a new strategy - pub fn new(huge_page: HugePageSupport, prot: MmapProtection, replace: bool, reserve: bool) -> Self { + pub fn new( + huge_page: HugePageSupport, + prot: MmapProtection, + replace: bool, + reserve: bool, + ) -> Self { Self { huge_page, prot, @@ -126,10 +169,12 @@ impl MmapStrategy { // Builder methods + /// Set huge page option. pub fn huge_page(self, huge_page: HugePageSupport) -> Self { Self { huge_page, ..self } } + /// Set huge page option by a boolean flag. pub fn transparent_hugepages(self, enable: bool) -> Self { let huge_page = if enable { HugePageSupport::TransparentHugePages @@ -139,22 +184,26 @@ impl MmapStrategy { Self { huge_page, ..self } } + /// Set protection option. pub fn prot(self, prot: MmapProtection) -> Self { Self { prot, ..self } } + /// Set the replace flag. pub fn replace(self, replace: bool) -> Self { Self { replace, ..self } } + /// Set the reserve flag. pub fn reserve(self, reserve: bool) -> Self { Self { reserve, ..self } } - /// The strategy for MMTk's own internal memory #[cfg(test)] // In test mode, we use test settings which allows replacing existing mappings. + /// The strategy for MMTk's own internal memory (test) pub const INTERNAL_MEMORY: Self = Self::TEST; #[cfg(not(test))] + /// The strategy for MMTk's own internal memory pub const INTERNAL_MEMORY: Self = Self { huge_page: HugePageSupport::No, prot: MmapProtection::ReadWrite, @@ -162,10 +211,11 @@ impl MmapStrategy { reserve: true, }; - /// The strategy for MMTk side metadata #[cfg(test)] + /// The strategy for MMTk side metadata (test) pub const SIDE_METADATA: Self = Self::TEST; #[cfg(not(test))] + /// The strategy for MMTk side metadata pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; /// The strategy for MMTk's test memory diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index 3ed6ec1ac9..b2d6bc9b86 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -1,14 +1,22 @@ +//! Operating System abstractions for MMTk. + +// Note: +// 1. For functions that return `Result`, an error value should only be used for exceptional cases. If a function returns +// a placeholder value, that should not be considered as 'exceptional cases', and should return Ok. +// 2. Some functions or arguments (e.g. [`crate::util::os::memory::MmapStrategy`]) allow fallback behaviors for platforms where certain features +// are not supported, or unimplemented. + mod memory; pub use memory::*; mod process; pub use process::*; -#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] -pub(crate) mod posix_common; #[cfg(target_os = "linux")] pub(crate) mod linux; #[cfg(target_os = "macos")] pub(crate) mod macos; +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] +pub(crate) mod posix_common; #[cfg(target_os = "windows")] pub(crate) mod windows; @@ -26,8 +34,3 @@ pub use linux::LinuxProcessImpl as OSProcess; pub use macos::MacOSMemoryImpl as OSMemory; #[cfg(target_os = "macos")] pub use macos::MacOSProcessImpl as OSProcess; - -// pub trait OperatingSystem { -// type OSMemory: memory::Memory; -// type OSProcess: process::Process; -// } diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index 64e539238b..c0c038ece3 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -1,12 +1,12 @@ -use crate::util::os::*; use crate::util::address::Address; -use std::io::Result; +use crate::util::os::*; use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; +use std::io::Result; use std::mem::MaybeUninit; impl MmapProtection { - fn into_native_flags(&self) -> i32 { - use libc::{PROT_NONE, PROT_READ, PROT_WRITE, PROT_EXEC}; + fn get_native_flags(&self) -> i32 { + use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; match self { Self::ReadWrite => PROT_READ | PROT_WRITE, Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, @@ -17,7 +17,7 @@ impl MmapProtection { pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ let ptr = start.to_mut_ptr(); - let prot = strategy.prot.into_native_flags(); + let prot = strategy.prot.get_native_flags(); let flags = strategy.get_posix_mmap_flags(); wrap_libc_call( &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, @@ -71,7 +71,7 @@ pub fn get_process_memory_maps() -> Result { } pub fn munmap(start: Address, size: usize) -> Result<()> { - return wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0); + wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) } pub fn mprotect(start: Address, size: usize) -> Result<()> { @@ -84,7 +84,7 @@ pub fn mprotect(start: Address, size: usize) -> Result<()> { pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.into_native_flags()) }, + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.get_native_flags()) }, 0, ) } @@ -137,4 +137,4 @@ pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> } else { Err(std::io::Error::last_os_error()) } -} \ No newline at end of file +} diff --git a/src/util/os/process.rs b/src/util/os/process.rs index 5f3f79e4c0..ba56f86dd7 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -1,20 +1,30 @@ use std::io::Result; +/// Representation of a CPU core identifier. pub type CoreId = u16; +/// Representation of number of CPU cores. pub type CoreNum = u16; +/// Abstraction for OS process operations. pub trait Process { - /// Return error if unable to get process memory maps. - /// If unimplemented, just return Ok with an string to indiate that. + /// Get the memory maps for the process. The returned string is a multi-line string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. fn get_process_memory_maps() -> Result; + /// Get the process ID as a string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. fn get_process_id() -> Result; + //// Get the thread ID as a string. + /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. fn get_thread_id() -> Result; - fn get_total_num_cpus() -> u16; + /// Return the total number of cores allocated to the program. + fn get_total_num_cpus() -> CoreNum; + /// Bind the current thread to the specified core. fn bind_current_thread_to_core(core_id: CoreId); + /// Bind the current thread to the specified core set. fn bind_current_thread_to_cpuset(core_ids: &[CoreId]); } diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index 48e3505a50..e0f36f3d54 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -1,11 +1,17 @@ -use crate::util::os::memory::*; use crate::util::address::Address; +use crate::util::os::memory::*; use std::io::Result; +/// Windows implementation of the `Memory` trait. pub struct WindowsMemoryImpl; impl Memory for WindowsMemoryImpl { - fn dzmmap(start: Address, size: usize, strategy: MmapStrategy, _annotation: &MmapAnnotation<'_>) -> Result
{ + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + _annotation: &MmapAnnotation<'_>, + ) -> Result
{ use std::io; use windows_sys::Win32::System::Memory::{ VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, @@ -79,7 +85,12 @@ impl Memory for WindowsMemoryImpl { allocation_type |= MEM_COMMIT; } - let res = VirtualAlloc(ptr as *mut _, size, allocation_type, strategy.prot.into_native_flags()); + let res = VirtualAlloc( + ptr as *mut _, + size, + allocation_type, + strategy.prot.get_native_flags(), + ); if res.is_null() { return Err(io::Error::last_os_error()); } @@ -90,7 +101,12 @@ impl Memory for WindowsMemoryImpl { // If the region is already mapped, we just ensure the required commitment. // If commit is not needed, we just return Ok. if commit { - let res = VirtualAlloc(ptr as *mut _, size, MEM_COMMIT, strategy.prot.into_native_flags()); + let res = VirtualAlloc( + ptr as *mut _, + size, + MEM_COMMIT, + strategy.prot.get_native_flags(), + ); if res.is_null() { return Err(io::Error::last_os_error()); } @@ -134,7 +150,7 @@ impl Memory for WindowsMemoryImpl { fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { use windows_sys::Win32::System::Memory::*; - let prot = prot.into_native_flags(); + let prot = prot.get_native_flags(); let mut old_protect = 0; let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; if res == 0 { @@ -145,16 +161,20 @@ impl Memory for WindowsMemoryImpl { } fn panic_if_unmapped(start: Address, size: usize, _annotation: &MmapAnnotation<'_>) { - warn!("Check if {} of size {} is mapped is ignored on Windows", start, size); + warn!( + "Check if {} of size {} is mapped is ignored on Windows", + start, size + ); } fn is_mmap_oom(os_errno: i32) -> bool { - os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 || os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 + os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 + || os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 } } impl MmapProtection { - fn into_native_flags(&self) -> u32 { + fn get_native_flags(&self) -> u32 { use windows_sys::Win32::System::Memory::*; match self { Self::ReadWrite => PAGE_READWRITE, @@ -166,6 +186,7 @@ impl MmapProtection { use crate::util::os::process::*; +/// Windows implementation of the `Process` trait. pub struct WindowsProcessImpl; impl Process for WindowsProcessImpl { diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index ade9baf21d..6a0cb2b436 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -109,7 +109,11 @@ unsafe impl Sync for InitializeOnce {} /// Create a formatted string that makes the best effort idenfying the current process and thread. pub fn debug_process_thread_id() -> String { use crate::util::os::*; - format!("PID: {}, TID: {}", OSProcess::get_process_id().unwrap_or_default(), OSProcess::get_thread_id().unwrap_or_default()) + format!( + "PID: {}, TID: {}", + OSProcess::get_process_id().unwrap_or_default(), + OSProcess::get_thread_id().unwrap_or_default() + ) } #[cfg(test)] diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs index f8dcd85e74..3d78b9035f 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs @@ -1,7 +1,7 @@ use super::mock_test_prelude::*; -use crate::util::os::*; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::Address; #[test] @@ -11,12 +11,8 @@ pub fn test_handle_mmap_conflict() { || { let start = unsafe { Address::from_usize(0x100_0000) }; let one_megabyte = 1000000; - let mmap1_res = OSMemory::dzmmap( - start, - one_megabyte, - MmapStrategy::TEST, - mmap_anno_test!(), - ); + let mmap1_res = + OSMemory::dzmmap(start, one_megabyte, MmapStrategy::TEST, mmap_anno_test!()); assert!(mmap1_res.is_ok()); let panic_res = std::panic::catch_unwind(|| { diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs index d5e8acee5d..6f2cc06eca 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs @@ -1,7 +1,7 @@ use super::mock_test_prelude::*; -use crate::util::os::*; use crate::util::opaque_pointer::*; +use crate::util::os::*; use crate::util::Address; #[cfg(target_pointer_width = "32")] @@ -18,12 +18,8 @@ pub fn test_handle_mmap_oom() { let start = unsafe { Address::from_usize(0x100_0000) }; // mmap 1 terabyte memory - we expect this will fail due to out of memory. // If that's not the case, increase the size we mmap. - let mmap_res = OSMemory::dzmmap( - start, - LARGE_SIZE, - MmapStrategy::TEST, - mmap_anno_test!(), - ); + let mmap_res = + OSMemory::dzmmap(start, LARGE_SIZE, MmapStrategy::TEST, mmap_anno_test!()); OSMemory::handle_mmap_error::( mmap_res.err().unwrap(), From f781f9a8825566754feabcd19e8eb6ae94cf9eff Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 7 Jan 2026 16:12:48 +1300 Subject: [PATCH 12/27] Tidy up for macOS --- src/util/os/macos.rs | 14 +++++++------- src/util/os/memory.rs | 4 ++-- src/util/os/mod.rs | 2 +- src/util/os/posix_common.rs | 13 +++++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/util/os/macos.rs b/src/util/os/macos.rs index a0e18e24b1..1570cf414f 100644 --- a/src/util/os/macos.rs +++ b/src/util/os/macos.rs @@ -41,8 +41,8 @@ impl Memory for MacOSMemoryImpl { posix_common::is_mmap_oom(os_errno) } - fn panic_if_unmapped(start: Address, size: usize) { - // Do nothing for now + fn panic_if_unmapped(_start: Address, _size: usize) { + // Unimplemented for now } } @@ -98,14 +98,14 @@ impl Process for MacOSProcessImpl { } fn get_total_num_cpus() -> CoreNum { - posix_common::get_total_num_cpus() + unimplemented!() } - fn bind_current_thread_to_core(core_id: CoreId) { - posix_common::bind_current_thread_to_core(core_id) + fn bind_current_thread_to_core(_core_id: CoreId) { + unimplemented!() } - fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { - posix_common::bind_current_thread_to_cpuset(core_ids) + fn bind_current_thread_to_cpuset(_core_ids: &[CoreId]) { + unimplemented!() } } diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 9ed3884467..a745954904 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -23,7 +23,7 @@ pub trait Memory { } /// Perform a demand-zero mmap. - /// + /// /// Falback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. fn dzmmap( start: Address, @@ -97,7 +97,7 @@ pub trait Memory { /// /// Note that the checking may have a side effect that it will map the memory if it was unmapped. So we panic if it was unmapped. /// Be very careful about using this function. - /// + /// /// Fallback: As the function is only used for assertions, it can be a no-op, and MMTk will still run and never panics in this function. fn panic_if_unmapped(start: Address, size: usize); diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index b2d6bc9b86..1b7c8c1ce2 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -1,6 +1,6 @@ //! Operating System abstractions for MMTk. -// Note: +// Note: // 1. For functions that return `Result`, an error value should only be used for exceptional cases. If a function returns // a placeholder value, that should not be considered as 'exceptional cases', and should return Ok. // 2. Some functions or arguments (e.g. [`crate::util::os::memory::MmapStrategy`]) allow fallback behaviors for platforms where certain features diff --git a/src/util/os/posix_common.rs b/src/util/os/posix_common.rs index c0c038ece3..ab76e330a0 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/posix_common.rs @@ -1,8 +1,6 @@ use crate::util::address::Address; use crate::util::os::*; -use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; use std::io::Result; -use std::mem::MaybeUninit; impl MmapProtection { fn get_native_flags(&self) -> i32 { @@ -95,11 +93,16 @@ pub fn get_process_id() -> Result { } pub fn get_thread_id() -> Result { - let tid = unsafe { libc::gettid() }; + let tid = unsafe { libc::pthread_self() }; Ok(format!("{}", tid)) } +#[cfg(any(target_os = "linux", target_os = "android"))] +use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; + +#[cfg(any(target_os = "linux", target_os = "android"))] pub fn get_total_num_cpus() -> CoreNum { + use std::mem::MaybeUninit; unsafe { let mut cs = MaybeUninit::zeroed().assume_init(); CPU_ZERO(&mut cs); @@ -108,7 +111,9 @@ pub fn get_total_num_cpus() -> CoreNum { } } +#[cfg(any(target_os = "linux", target_os = "android"))] pub fn bind_current_thread_to_core(core_id: CoreId) { + use std::mem::MaybeUninit; unsafe { let mut cs = MaybeUninit::zeroed().assume_init(); CPU_ZERO(&mut cs); @@ -117,7 +122,7 @@ pub fn bind_current_thread_to_core(core_id: CoreId) { } } -/// Bind the current thread to the specified core. +#[cfg(any(target_os = "linux", target_os = "android"))] pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { use std::mem::MaybeUninit; unsafe { From 86baa5b4d6fc563f27eaf72d32cee408508caa35 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 7 Jan 2026 16:45:46 +1300 Subject: [PATCH 13/27] Tidy up for Windows --- src/util/malloc/library.rs | 1 + src/util/os/windows.rs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/util/malloc/library.rs b/src/util/malloc/library.rs index e716502a04..b25c677faa 100644 --- a/src/util/malloc/library.rs +++ b/src/util/malloc/library.rs @@ -45,6 +45,7 @@ mod mimalloc { mi_calloc as calloc, mi_free as free, mi_malloc as malloc, mi_realloc as realloc, }; // Posix + #[cfg_attr(target_os = "windows", allow(unused_imports))] pub use mimalloc_sys::mi_posix_memalign as posix_memalign; // GNU pub use mimalloc_sys::mi_malloc_usable_size as malloc_usable_size; diff --git a/src/util/os/windows.rs b/src/util/os/windows.rs index e0f36f3d54..1079475c79 100644 --- a/src/util/os/windows.rs +++ b/src/util/os/windows.rs @@ -160,7 +160,7 @@ impl Memory for WindowsMemoryImpl { } } - fn panic_if_unmapped(start: Address, size: usize, _annotation: &MmapAnnotation<'_>) { + fn panic_if_unmapped(start: Address, size: usize) { warn!( "Check if {} of size {} is mapped is ignored on Windows", start, size @@ -217,14 +217,14 @@ impl Process for WindowsProcessImpl { unsafe { windows_sys::Win32::System::Threading::SetThreadAffinityMask( windows_sys::Win32::System::Threading::GetCurrentThread(), - 1 << cpu, + 1 << core_id, ); } } fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { let mut mask = 0; - for cpu in cpuset { + for cpu in core_ids { mask |= 1 << cpu; } unsafe { From a36639c3f1168501dd51afc763bdda5b4e718ab8 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 7 Jan 2026 04:03:04 +0000 Subject: [PATCH 14/27] Fix for 32 bits linux --- src/util/metadata/side_metadata/helpers_32.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/util/metadata/side_metadata/helpers_32.rs b/src/util/metadata/side_metadata/helpers_32.rs index 1372d5916c..3e4e2d88d4 100644 --- a/src/util/metadata/side_metadata/helpers_32.rs +++ b/src/util/metadata/side_metadata/helpers_32.rs @@ -2,7 +2,7 @@ use super::SideMetadataSpec; use crate::util::{ constants::{self, LOG_BITS_IN_BYTE}, heap::layout::vm_layout::{BYTES_IN_CHUNK, CHUNK_MASK, LOG_BYTES_IN_CHUNK}, - memory::{self, MmapAnnotation}, + os::*, Address, }; use std::io::Result; @@ -102,7 +102,7 @@ pub(super) const fn metadata_bytes_per_chunk( pub(crate) fn ensure_munmap_metadata_chunk(start: Address, local_per_chunk: usize) { if local_per_chunk != 0 { let policy_meta_start = address_to_meta_chunk_addr(start); - assert!(memory::munmap(policy_meta_start, local_per_chunk).is_ok()) + assert!(OSMemory::munmap(policy_meta_start, local_per_chunk).is_ok()) } } @@ -156,7 +156,8 @@ pub(super) fn try_map_per_chunk_metadata_space( } if munmap_first_chunk.is_none() { // if first chunk is newly mapped, it needs munmap on failure - munmap_first_chunk = Some(memory::result_is_mapped(res)); + let map_exists = res.is_err_and(|e| e.kind() == std::io::ErrorKind::AlreadyExists); + munmap_first_chunk = Some(map_exists); } aligned_start += BYTES_IN_CHUNK; total_mapped += local_per_chunk; @@ -187,14 +188,15 @@ pub(super) fn try_mmap_metadata_chunk( MMAPPER.ensure_mapped( policy_meta_start, pages, - memory::MmapStrategy::SIDE_METADATA, + HugePageSupport::No, + MmapProtection::ReadWrite, anno, ) } else { MMAPPER.quarantine_address_range( policy_meta_start, pages, - memory::MmapStrategy::SIDE_METADATA, + HugePageSupport::No, anno, ) } From 04ea5aa1f703378db55fc8e6608dfe407954ec99 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 7 Jan 2026 04:26:50 +0000 Subject: [PATCH 15/27] Restructure os imp --- src/util/metadata/side_metadata/helpers_32.rs | 7 +- src/util/os/imp/mod.rs | 25 +++++ .../os/imp/unix_like/linux_like/android.rs | 94 ++++++++++++++++ .../{ => imp/unix_like/linux_like}/linux.rs | 64 +++-------- .../unix_like/linux_like/linux_common.rs} | 103 +++++------------- src/util/os/imp/unix_like/linux_like/mod.rs | 7 ++ src/util/os/{ => imp/unix_like}/macos.rs | 16 +-- src/util/os/imp/unix_like/mod.rs | 7 ++ src/util/os/imp/unix_like/unix_common.rs | 67 ++++++++++++ src/util/os/{ => imp}/windows.rs | 0 src/util/os/mod.rs | 26 +---- 11 files changed, 258 insertions(+), 158 deletions(-) create mode 100644 src/util/os/imp/mod.rs create mode 100644 src/util/os/imp/unix_like/linux_like/android.rs rename src/util/os/{ => imp/unix_like/linux_like}/linux.rs (50%) rename src/util/os/{posix_common.rs => imp/unix_like/linux_like/linux_common.rs} (55%) create mode 100644 src/util/os/imp/unix_like/linux_like/mod.rs rename src/util/os/{ => imp/unix_like}/macos.rs (88%) create mode 100644 src/util/os/imp/unix_like/mod.rs create mode 100644 src/util/os/imp/unix_like/unix_common.rs rename src/util/os/{ => imp}/windows.rs (100%) diff --git a/src/util/metadata/side_metadata/helpers_32.rs b/src/util/metadata/side_metadata/helpers_32.rs index 3e4e2d88d4..241904d963 100644 --- a/src/util/metadata/side_metadata/helpers_32.rs +++ b/src/util/metadata/side_metadata/helpers_32.rs @@ -193,11 +193,6 @@ pub(super) fn try_mmap_metadata_chunk( anno, ) } else { - MMAPPER.quarantine_address_range( - policy_meta_start, - pages, - HugePageSupport::No, - anno, - ) + MMAPPER.quarantine_address_range(policy_meta_start, pages, HugePageSupport::No, anno) } } diff --git a/src/util/os/imp/mod.rs b/src/util/os/imp/mod.rs new file mode 100644 index 0000000000..0a30e5f71e --- /dev/null +++ b/src/util/os/imp/mod.rs @@ -0,0 +1,25 @@ +#[cfg(target_os = "windows")] +pub(crate) mod windows; + +#[cfg(target_os = "windows")] +pub use windows::WindowsMemoryImpl as OSMemory; +#[cfg(target_os = "windows")] +pub use windows::WindowsProcessImpl as OSProcess; + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] +pub(crate) mod unix_like; + +#[cfg(target_os = "linux")] +pub use unix_like::linux_like::linux::LinuxMemoryImpl as OSMemory; +#[cfg(target_os = "linux")] +pub use unix_like::linux_like::linux::LinuxProcessImpl as OSProcess; + +#[cfg(target_os = "android")] +pub use unix_like::linux_like::android::AndroidMemoryImpl as OSMemory; +#[cfg(target_os = "android")] +pub use unix_like::linux_like::android::AndroidProcessImpl as OSProcess; + +#[cfg(target_os = "macos")] +pub use unix_like::macos::MacOSMemoryImpl as OSMemory; +#[cfg(target_os = "macos")] +pub use unix_like::macos::MacOSProcessImpl as OSProcess; diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs new file mode 100644 index 0000000000..3ff4219e49 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -0,0 +1,94 @@ +use crate::util::address::Address; +use crate::util::os::imp::unix_like::linux_like::linux_common; +use crate::util::os::imp::unix_like::unix_common; +use crate::util::os::*; + +use std::io::Result; + +/// Android implementation of the `Memory` trait. +pub struct AndroidMemoryImpl; + +impl Memory for AndroidMemoryImpl { + fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, + ) -> Result
{ + let addr = unix_common::mmap(start, size, strategy)?; + + if !cfg!(feature = "no_mmap_annotation") { + linux_common::set_vma_name(addr, size, annotation); + } + + linux_common::set_hugepage(addr, size, strategy.huge_page)?; + + // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) + + Ok(addr) + } + + fn munmap(start: Address, size: usize) -> Result<()> { + unix_common::munmap(start, size) + } + + fn mprotect(start: Address, size: usize) -> Result<()> { + unix_common::mprotect(start, size) + } + + fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::munprotect(start, size, prot) + } + + fn is_mmap_oom(os_errno: i32) -> bool { + unix_common::is_mmap_oom(os_errno) + } + + fn panic_if_unmapped(start: Address, size: usize) { + let strategy = MmapStrategy { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + match unix_common::mmap(start, size, strategy) { + Ok(_) => panic!("{} of size {} is not mapped", start, size), + Err(e) => { + assert!( + e.kind() == std::io::ErrorKind::AlreadyExists, + "Failed to check mapped: {:?}", + e + ); + } + } + } +} + +/// Android implementation of the `Process` trait. +pub struct AndroidProcessImpl; + +impl Process for AndroidProcessImpl { + fn get_process_memory_maps() -> Result { + linux_common::get_process_memory_maps() + } + + fn get_process_id() -> Result { + unix_common::get_process_id() + } + + fn get_thread_id() -> Result { + unix_common::get_thread_id() + } + + fn get_total_num_cpus() -> CoreNum { + linux_common::get_total_num_cpus() + } + + fn bind_current_thread_to_core(core_id: CoreId) { + linux_common::bind_current_thread_to_core(core_id) + } + + fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { + linux_common::bind_current_thread_to_cpuset(core_ids) + } +} diff --git a/src/util/os/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs similarity index 50% rename from src/util/os/linux.rs rename to src/util/os/imp/unix_like/linux_like/linux.rs index a90aaf925a..735f6bc208 100644 --- a/src/util/os/linux.rs +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -1,5 +1,6 @@ use crate::util::address::Address; -use crate::util::os::posix_common; +use crate::util::os::imp::unix_like::linux_like::linux_common; +use crate::util::os::imp::unix_like::unix_common; use crate::util::os::*; use std::io::Result; @@ -14,17 +15,13 @@ impl Memory for LinuxMemoryImpl { strategy: MmapStrategy, annotation: &MmapAnnotation<'_>, ) -> Result
{ - // println!("Mmap with strategy: {:?}", strategy); - let addr = posix_common::mmap(start, size, strategy)?; - // println!("Mmap done"); + let addr = unix_common::mmap(start, size, strategy)?; if !cfg!(feature = "no_mmap_annotation") { - posix_common::set_vma_name(addr, size, annotation); - // println!("Set annotation done"); + linux_common::set_vma_name(addr, size, annotation); } - Self::set_hugepage(addr, size, strategy.huge_page)?; - // println!("Set huge page done"); + linux_common::set_hugepage(addr, size, strategy.huge_page)?; // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) @@ -32,19 +29,19 @@ impl Memory for LinuxMemoryImpl { } fn munmap(start: Address, size: usize) -> Result<()> { - posix_common::munmap(start, size) + unix_common::munmap(start, size) } fn mprotect(start: Address, size: usize) -> Result<()> { - posix_common::mprotect(start, size) + unix_common::mprotect(start, size) } fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - posix_common::munprotect(start, size, prot) + unix_common::munprotect(start, size, prot) } fn is_mmap_oom(os_errno: i32) -> bool { - posix_common::is_mmap_oom(os_errno) + unix_common::is_mmap_oom(os_errno) } fn panic_if_unmapped(start: Address, size: usize) { @@ -54,7 +51,7 @@ impl Memory for LinuxMemoryImpl { replace: false, reserve: true, }; - match posix_common::mmap(start, size, strategy) { + match unix_common::mmap(start, size, strategy) { Ok(_) => panic!("{} of size {} is not mapped", start, size), Err(e) => { assert!( @@ -67,60 +64,31 @@ impl Memory for LinuxMemoryImpl { } } -impl LinuxMemoryImpl { - /// Set huge page option for the given memory. - pub fn set_hugepage(start: Address, size: usize, options: HugePageSupport) -> Result<()> { - match options { - HugePageSupport::No => Ok(()), - HugePageSupport::TransparentHugePages => posix_common::wrap_libc_call( - &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, - 0, - ), - } - } -} - -impl MmapStrategy { - /// get the flags for POSIX mmap. - pub fn get_posix_mmap_flags(&self) -> i32 { - let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; - if self.replace { - flags |= libc::MAP_FIXED; - } else { - flags |= libc::MAP_FIXED_NOREPLACE - } - if !self.reserve { - flags |= libc::MAP_NORESERVE; - } - flags - } -} - /// Linux implementation of the `Process` trait. pub struct LinuxProcessImpl; impl Process for LinuxProcessImpl { fn get_process_memory_maps() -> Result { - posix_common::get_process_memory_maps() + linux_common::get_process_memory_maps() } fn get_process_id() -> Result { - posix_common::get_process_id() + unix_common::get_process_id() } fn get_thread_id() -> Result { - posix_common::get_thread_id() + unix_common::get_thread_id() } fn get_total_num_cpus() -> CoreNum { - posix_common::get_total_num_cpus() + linux_common::get_total_num_cpus() } fn bind_current_thread_to_core(core_id: CoreId) { - posix_common::bind_current_thread_to_core(core_id) + linux_common::bind_current_thread_to_core(core_id) } fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { - posix_common::bind_current_thread_to_cpuset(core_ids) + linux_common::bind_current_thread_to_cpuset(core_ids) } } diff --git a/src/util/os/posix_common.rs b/src/util/os/imp/unix_like/linux_like/linux_common.rs similarity index 55% rename from src/util/os/posix_common.rs rename to src/util/os/imp/unix_like/linux_like/linux_common.rs index ab76e330a0..2ca3044095 100644 --- a/src/util/os/posix_common.rs +++ b/src/util/os/imp/unix_like/linux_like/linux_common.rs @@ -1,34 +1,9 @@ use crate::util::address::Address; +use crate::util::os::imp::unix_like::unix_common; use crate::util::os::*; +use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; use std::io::Result; -impl MmapProtection { - fn get_native_flags(&self) -> i32 { - use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; - match self { - Self::ReadWrite => PROT_READ | PROT_WRITE, - Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, - Self::NoAccess => PROT_NONE, - } - } -} - -pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ - let ptr = start.to_mut_ptr(); - let prot = strategy.prot.get_native_flags(); - let flags = strategy.get_posix_mmap_flags(); - wrap_libc_call( - &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, - ptr, - )?; - Ok(start) -} - -pub fn is_mmap_oom(os_errno: i32) -> bool { - os_errno == libc::ENOMEM -} - -#[cfg(any(target_os = "linux", target_os = "android"))] pub fn set_vma_name(start: Address, size: usize, annotation: &MmapAnnotation) { // `PR_SET_VMA` is new in Linux 5.17. We compile against a version of the `libc` crate that // has the `PR_SET_VMA_ANON_NAME` constant. When runnning on an older kernel, it will not @@ -38,7 +13,7 @@ pub fn set_vma_name(start: Address, size: usize, annotation: &MmapAnnotation) { // since this prctl is used for debugging, we log the error instead of panicking. let anno_str = annotation.to_string(); let anno_cstr = std::ffi::CString::new(anno_str).unwrap(); - let result = wrap_libc_call( + let result = unix_common::wrap_libc_call( &|| unsafe { libc::prctl( libc::PR_SET_VMA, @@ -55,9 +30,35 @@ pub fn set_vma_name(start: Address, size: usize, annotation: &MmapAnnotation) { } } +/// Set huge page option for the given memory. +pub fn set_hugepage(start: Address, size: usize, options: HugePageSupport) -> Result<()> { + match options { + HugePageSupport::No => Ok(()), + HugePageSupport::TransparentHugePages => unix_common::wrap_libc_call( + &|| unsafe { libc::madvise(start.to_mut_ptr(), size, libc::MADV_HUGEPAGE) }, + 0, + ), + } +} + +impl MmapStrategy { + /// get the flags for POSIX mmap. + pub fn get_posix_mmap_flags(&self) -> i32 { + let mut flags = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS; + if self.replace { + flags |= libc::MAP_FIXED; + } else { + flags |= libc::MAP_FIXED_NOREPLACE + } + if !self.reserve { + flags |= libc::MAP_NORESERVE; + } + flags + } +} + /// Get the memory maps for the process. The returned string is a multi-line string. /// This is only meant to be used for debugging. For example, log process memory maps after detecting a clash. -#[cfg(any(target_os = "linux", target_os = "android"))] pub fn get_process_memory_maps() -> Result { // print map use std::fs::File; @@ -68,39 +69,6 @@ pub fn get_process_memory_maps() -> Result { Ok(data) } -pub fn munmap(start: Address, size: usize) -> Result<()> { - wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) -} - -pub fn mprotect(start: Address, size: usize) -> Result<()> { - let prot = libc::PROT_NONE; - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) -} - -pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.get_native_flags()) }, - 0, - ) -} - -pub fn get_process_id() -> Result { - let pid = unsafe { libc::getpid() }; - Ok(format!("{}", pid)) -} - -pub fn get_thread_id() -> Result { - let tid = unsafe { libc::pthread_self() }; - Ok(format!("{}", tid)) -} - -#[cfg(any(target_os = "linux", target_os = "android"))] -use libc::{cpu_set_t, sched_getaffinity, sched_setaffinity, CPU_COUNT, CPU_SET, CPU_ZERO}; - -#[cfg(any(target_os = "linux", target_os = "android"))] pub fn get_total_num_cpus() -> CoreNum { use std::mem::MaybeUninit; unsafe { @@ -111,7 +79,6 @@ pub fn get_total_num_cpus() -> CoreNum { } } -#[cfg(any(target_os = "linux", target_os = "android"))] pub fn bind_current_thread_to_core(core_id: CoreId) { use std::mem::MaybeUninit; unsafe { @@ -122,7 +89,6 @@ pub fn bind_current_thread_to_core(core_id: CoreId) { } } -#[cfg(any(target_os = "linux", target_os = "android"))] pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { use std::mem::MaybeUninit; unsafe { @@ -134,12 +100,3 @@ pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { sched_setaffinity(0, std::mem::size_of::(), &cs); } } - -pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { - let ret = f(); - if ret == expect { - Ok(()) - } else { - Err(std::io::Error::last_os_error()) - } -} diff --git a/src/util/os/imp/unix_like/linux_like/mod.rs b/src/util/os/imp/unix_like/linux_like/mod.rs new file mode 100644 index 0000000000..59f3bbb521 --- /dev/null +++ b/src/util/os/imp/unix_like/linux_like/mod.rs @@ -0,0 +1,7 @@ +pub mod linux_common; + +#[cfg(target_os = "linux")] +pub mod linux; + +#[cfg(target_os = "android")] +pub mod android; diff --git a/src/util/os/macos.rs b/src/util/os/imp/unix_like/macos.rs similarity index 88% rename from src/util/os/macos.rs rename to src/util/os/imp/unix_like/macos.rs index 1570cf414f..3fb99d890e 100644 --- a/src/util/os/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -1,5 +1,5 @@ use crate::util::address::Address; -use crate::util::os::posix_common; +use crate::util::os::imp::unix_like::unix_common; use crate::util::os::*; use std::io::Result; @@ -13,7 +13,7 @@ impl Memory for MacOSMemoryImpl { strategy: MmapStrategy, _annotation: &MmapAnnotation<'_>, ) -> Result
{ - let addr = posix_common::mmap(start, size, strategy)?; + let addr = unix_common::mmap(start, size, strategy)?; // Annotation is ignored on macOS // Huge page is ignored on macOS @@ -26,19 +26,19 @@ impl Memory for MacOSMemoryImpl { } fn munmap(start: Address, size: usize) -> Result<()> { - posix_common::munmap(start, size) + unix_common::munmap(start, size) } fn mprotect(start: Address, size: usize) -> Result<()> { - posix_common::mprotect(start, size) + unix_common::mprotect(start, size) } fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - posix_common::munprotect(start, size, prot) + unix_common::munprotect(start, size, prot) } fn is_mmap_oom(os_errno: i32) -> bool { - posix_common::is_mmap_oom(os_errno) + unix_common::is_mmap_oom(os_errno) } fn panic_if_unmapped(_start: Address, _size: usize) { @@ -90,11 +90,11 @@ impl Process for MacOSProcessImpl { } fn get_process_id() -> Result { - posix_common::get_process_id() + unix_common::get_process_id() } fn get_thread_id() -> Result { - posix_common::get_thread_id() + unix_common::get_thread_id() } fn get_total_num_cpus() -> CoreNum { diff --git a/src/util/os/imp/unix_like/mod.rs b/src/util/os/imp/unix_like/mod.rs new file mode 100644 index 0000000000..ba1c7f9c7d --- /dev/null +++ b/src/util/os/imp/unix_like/mod.rs @@ -0,0 +1,7 @@ +pub mod unix_common; + +#[cfg(target_os = "macos")] +pub mod macos; + +#[cfg(any(target_os = "linux", target_os = "android"))] +pub mod linux_like; diff --git a/src/util/os/imp/unix_like/unix_common.rs b/src/util/os/imp/unix_like/unix_common.rs new file mode 100644 index 0000000000..8a32e215c6 --- /dev/null +++ b/src/util/os/imp/unix_like/unix_common.rs @@ -0,0 +1,67 @@ +use crate::util::address::Address; +use crate::util::os::*; +use std::io::Result; + +impl MmapProtection { + fn get_native_flags(&self) -> i32 { + use libc::{PROT_EXEC, PROT_NONE, PROT_READ, PROT_WRITE}; + match self { + Self::ReadWrite => PROT_READ | PROT_WRITE, + Self::ReadWriteExec => PROT_READ | PROT_WRITE | PROT_EXEC, + Self::NoAccess => PROT_NONE, + } + } +} + +pub fn mmap(start: Address, size: usize, strategy: MmapStrategy) -> Result
{ + let ptr = start.to_mut_ptr(); + let prot = strategy.prot.get_native_flags(); + let flags = strategy.get_posix_mmap_flags(); + wrap_libc_call( + &|| unsafe { libc::mmap(start.to_mut_ptr(), size, prot, flags, -1, 0) }, + ptr, + )?; + Ok(start) +} + +pub fn is_mmap_oom(os_errno: i32) -> bool { + os_errno == libc::ENOMEM +} + +pub fn munmap(start: Address, size: usize) -> Result<()> { + wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) +} + +pub fn mprotect(start: Address, size: usize) -> Result<()> { + let prot = libc::PROT_NONE; + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, + 0, + ) +} + +pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + wrap_libc_call( + &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.get_native_flags()) }, + 0, + ) +} + +pub fn get_process_id() -> Result { + let pid = unsafe { libc::getpid() }; + Ok(format!("{}", pid)) +} + +pub fn get_thread_id() -> Result { + let tid = unsafe { libc::pthread_self() }; + Ok(format!("{}", tid)) +} + +pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { + let ret = f(); + if ret == expect { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} diff --git a/src/util/os/windows.rs b/src/util/os/imp/windows.rs similarity index 100% rename from src/util/os/windows.rs rename to src/util/os/imp/windows.rs diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index 1b7c8c1ce2..24608a3a11 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -11,26 +11,6 @@ pub use memory::*; mod process; pub use process::*; -#[cfg(target_os = "linux")] -pub(crate) mod linux; -#[cfg(target_os = "macos")] -pub(crate) mod macos; -#[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] -pub(crate) mod posix_common; -#[cfg(target_os = "windows")] -pub(crate) mod windows; - -#[cfg(target_os = "windows")] -pub use windows::WindowsMemoryImpl as OSMemory; -#[cfg(target_os = "windows")] -pub use windows::WindowsProcessImpl as OSProcess; - -#[cfg(target_os = "linux")] -pub use linux::LinuxMemoryImpl as OSMemory; -#[cfg(target_os = "linux")] -pub use linux::LinuxProcessImpl as OSProcess; - -#[cfg(target_os = "macos")] -pub use macos::MacOSMemoryImpl as OSMemory; -#[cfg(target_os = "macos")] -pub use macos::MacOSProcessImpl as OSProcess; +mod imp; +pub use imp::OSMemory; +pub use imp::OSProcess; From 72d98ea0af174d7aee832e659332bf0857a070e5 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 8 Jan 2026 02:25:59 +0000 Subject: [PATCH 16/27] Introduce trait OS. Rename set/zero to memset/zero. --- src/plan/generational/gc_work.rs | 2 +- src/policy/copyspace.rs | 4 ++-- src/policy/immix/block.rs | 2 +- src/policy/lockfreeimmortalspace.rs | 4 ++-- src/policy/markcompactspace.rs | 2 +- src/policy/space.rs | 4 ++-- src/scheduler/affinity.rs | 4 ++-- src/util/alloc/free_list_allocator.rs | 2 +- src/util/alloc/immix_allocator.rs | 2 +- src/util/heap/freelistpageresource.rs | 4 ++-- src/util/heap/layout/mmapper/csm/mod.rs | 22 +++++-------------- src/util/malloc/malloc_ms_util.rs | 2 +- src/util/metadata/side_metadata/global.rs | 6 ++--- src/util/metadata/side_metadata/helpers.rs | 2 +- src/util/metadata/side_metadata/helpers_32.rs | 2 +- src/util/options.rs | 6 ++--- src/util/os/imp/mod.rs | 16 ++++---------- .../os/imp/unix_like/linux_like/android.rs | 11 ++++------ src/util/os/imp/unix_like/linux_like/linux.rs | 11 ++++------ src/util/os/imp/unix_like/macos.rs | 11 ++++------ src/util/os/imp/windows.rs | 11 ++++------ src/util/os/memory.rs | 14 ++++++------ src/util/os/mod.rs | 5 +++-- src/util/os/process.rs | 2 +- src/util/raw_memory_freelist.rs | 4 ++-- src/util/rust_util/mod.rs | 4 ++-- .../mock_test_handle_mmap_conflict.rs | 7 +++--- .../mock_tests/mock_test_handle_mmap_oom.rs | 5 ++--- 28 files changed, 70 insertions(+), 101 deletions(-) diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index f833796ffd..af3dd502da 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -118,7 +118,7 @@ impl GCWork for ProcessModBuf { !gen.is_object_in_nursery(*obj), "{} was logged but is not mature. Dumping process memory maps:\n{}", *obj, - OSProcess::get_process_memory_maps().unwrap(), + OS::get_process_memory_maps().unwrap(), ); ::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::( *obj, diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 9a8fa7b0d7..685231c0d5 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -293,7 +293,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::os::OSMemory::mprotect(start, extent) { + if let Err(e) = crate::util::os::OS::mprotect(start, extent) { panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); @@ -308,7 +308,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::os::OSMemory::munprotect( + if let Err(e) = crate::util::os::OS::munprotect( start, extent, crate::util::os::MmapProtection::ReadWriteExec, diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index 6853db1916..7d7b2fcea8 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -255,7 +255,7 @@ impl Block { #[cfg(feature = "immix_zero_on_release")] { use crate::util::os::*; - OSMemory::zero(line.start(), Line::BYTES); + OS::memzero(line.start(), Line::BYTES); } // We need to clear the pin bit if it is on the side, as this line can be reused diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index bf63855ca8..5e04e5c6eb 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -157,7 +157,7 @@ impl Space for LockFreeImmortalSpace { } } if self.slow_path_zeroing { - crate::util::os::OSMemory::zero(start, bytes); + crate::util::os::OS::memzero(start, bytes); } start } @@ -259,7 +259,7 @@ impl LockFreeImmortalSpace { .prot(crate::util::os::MmapProtection::ReadWrite) .replace(false) .reserve(true); - crate::util::os::OSMemory::dzmmap( + crate::util::os::OS::dzmmap( start, aligned_total_bytes, strategy, diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 1b928efced..e6dd01134b 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -226,7 +226,7 @@ impl MarkCompactSpace { // Clear header forwarding pointer for an object fn clear_header_forwarding_pointer(object: ObjectReference) { - OSMemory::zero( + OS::memzero( Self::header_forwarding_pointer_address(object), GC_EXTRA_HEADER_BYTES, ); diff --git a/src/policy/space.rs b/src/policy/space.rs index 6710270d84..36aa50ca61 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -207,7 +207,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { self.get_name(), )) { - OSMemory::handle_mmap_error::(mmap_error, tls, res.start, bytes); + OS::handle_mmap_error::(mmap_error, tls, res.start, bytes); } }; let grow_space = || { @@ -232,7 +232,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { // TODO: Concurrent zeroing if self.common().zeroed { - OSMemory::zero(res.start, bytes); + OS::memzero(res.start, bytes); } // Some assertions diff --git a/src/scheduler/affinity.rs b/src/scheduler/affinity.rs index a16f909034..8655805f20 100644 --- a/src/scheduler/affinity.rs +++ b/src/scheduler/affinity.rs @@ -12,12 +12,12 @@ impl AffinityKind { AffinityKind::AllInSet(cpuset) => { // Bind the current thread to all the cores in the set debug!("Set affinity for thread {} to cpuset {:?}", thread, cpuset); - OSProcess::bind_current_thread_to_cpuset(cpuset.as_slice()); + OS::bind_current_thread_to_cpuset(cpuset.as_slice()); } AffinityKind::RoundRobin(cpuset) => { let cpu = cpuset[thread % cpuset.len()]; debug!("Set affinity for thread {} to core {}", thread, cpu); - OSProcess::bind_current_thread_to_core(cpu); + OS::bind_current_thread_to_core(cpu); } } } diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index 9485fbc1f7..9bb98c465a 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -166,7 +166,7 @@ impl FreeListAllocator { // Zeroing memory right before we return it. // If we move the zeroing to somewhere else, we need to clear the list link here: cell.store::
(Address::ZERO) let cell_size = block.load_block_cell_size(); - crate::util::os::OSMemory::zero(cell, cell_size); + crate::util::os::OS::memzero(cell, cell_size); // Make sure the memory is zeroed. This looks silly as we zero the cell right before this check. // But we would need to move the zeroing to somewhere so we can do zeroing at a coarser grainularity. diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index 9a8494c203..7dd9d5b201 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -251,7 +251,7 @@ impl ImmixAllocator { end_line, self.tls ); - OSMemory::zero( + OS::memzero( self.bump_pointer.cursor, self.bump_pointer.limit - self.bump_pointer.cursor, ); diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index dab5eae56e..c4a248fbe7 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -219,7 +219,7 @@ impl FreeListPageResource { // > (e.g., read versus read/write protection) exceeding the // > allowed maximum. assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = OSMemory::mprotect(start, conversions::pages_to_bytes(pages)) { + if let Err(e) = OS::mprotect(start, conversions::pages_to_bytes(pages)) { panic!( "Failed at protecting memory (starting at {}): {:?}", start, e @@ -230,7 +230,7 @@ impl FreeListPageResource { /// Unprotect the memory fn munprotect(&self, start: Address, pages: usize) { assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = OSMemory::munprotect( + if let Err(e) = OS::munprotect( start, conversions::pages_to_bytes(pages), self.protect_memory_on_release.unwrap(), diff --git a/src/util/heap/layout/mmapper/csm/mod.rs b/src/util/heap/layout/mmapper/csm/mod.rs index 84b15ef134..04cf041b7a 100644 --- a/src/util/heap/layout/mmapper/csm/mod.rs +++ b/src/util/heap/layout/mmapper/csm/mod.rs @@ -167,7 +167,7 @@ impl Mmapper for ChunkStateMmapper { .prot(MmapProtection::NoAccess) .reserve(false) .replace(false); - OSMemory::dzmmap(group_start, group_bytes, mmap_strategy, anno)?; + OS::dzmmap(group_start, group_bytes, mmap_strategy, anno)?; Ok(Some(MapState::Quarantined)) } MapState::Quarantined => { @@ -207,21 +207,11 @@ impl Mmapper for ChunkStateMmapper { match state { MapState::Unmapped => { - OSMemory::dzmmap( - group_start, - group_bytes, - mmap_strategy.replace(false), - anno, - )?; + OS::dzmmap(group_start, group_bytes, mmap_strategy.replace(false), anno)?; Ok(Some(MapState::Mapped)) } MapState::Quarantined => { - OSMemory::dzmmap( - group_start, - group_bytes, - mmap_strategy.replace(true), - anno, - )?; + OS::dzmmap(group_start, group_bytes, mmap_strategy.replace(true), anno)?; Ok(Some(MapState::Mapped)) } MapState::Mapped => Ok(None), @@ -297,7 +287,7 @@ mod tests { } }, || { - OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -331,7 +321,7 @@ mod tests { } }, || { - OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) @@ -366,7 +356,7 @@ mod tests { } }, || { - OSMemory::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); + OS::munmap(FIXED_ADDRESS, MAX_BYTES).unwrap(); }, ) }) diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 1feb495c7f..5a9b2be44e 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -20,7 +20,7 @@ pub fn align_alloc(size: usize, align: usize) -> Address { return Address::ZERO; } let address = Address::from_mut_ptr(ptr); - OSMemory::zero(address, size); + OS::memzero(address, size); address } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index dcf6756a6b..a60211bfbb 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -122,7 +122,7 @@ impl SideMetadataSpec { meta_start ); - OSMemory::panic_if_unmapped(meta_start, BYTES_IN_PAGE); + OS::panic_if_unmapped(meta_start, BYTES_IN_PAGE); } #[cfg(debug_assertions)] @@ -174,7 +174,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - OSMemory::zero(start, end - start); + OS::memzero(start, end - start); false } BitByteRange::BitsInByte { @@ -211,7 +211,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - OSMemory::set(start, 0xff, end - start); + OS::memset(start, 0xff, end - start); false } BitByteRange::BitsInByte { diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index d14eee01d9..859e37ddcc 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -96,7 +96,7 @@ pub(crate) fn ensure_munmap_metadata(start: Address, size: usize) { use crate::util::os::*; trace!("ensure_munmap_metadata({}, 0x{:x})", start, size); - assert!(OSMemory::munmap(start, size).is_ok()) + assert!(OS::munmap(start, size).is_ok()) } /// Unmaps a metadata space (`spec`) for the specified data address range (`start` and `size`) diff --git a/src/util/metadata/side_metadata/helpers_32.rs b/src/util/metadata/side_metadata/helpers_32.rs index 241904d963..87ad6ee35a 100644 --- a/src/util/metadata/side_metadata/helpers_32.rs +++ b/src/util/metadata/side_metadata/helpers_32.rs @@ -102,7 +102,7 @@ pub(super) const fn metadata_bytes_per_chunk( pub(crate) fn ensure_munmap_metadata_chunk(start: Address, local_per_chunk: usize) { if local_per_chunk != 0 { let policy_meta_start = address_to_meta_chunk_addr(start); - assert!(OSMemory::munmap(policy_meta_start, local_per_chunk).is_ok()) + assert!(OS::munmap(policy_meta_start, local_per_chunk).is_ok()) } } diff --git a/src/util/options.rs b/src/util/options.rs index 40b3add07e..216b162d61 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -455,7 +455,7 @@ impl AffinityKind { /// maximum number of cores allocated to the program. Assumes core ids on the system are /// 0-indexed. pub fn validate(&self) -> bool { - let num_cpu = OSProcess::get_total_num_cpus(); + let num_cpu = OS::get_total_num_cpus(); if let AffinityKind::RoundRobin(cpuset) = self { for cpu in cpuset { @@ -950,7 +950,7 @@ options! { thread_affinity: AffinityKind [|v: &AffinityKind| v.validate()] = AffinityKind::OsDefault, /// Set the GC trigger. This defines the heap size and how MMTk triggers a GC. /// Default to a fixed heap size of 0.5x physical memory. - gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((OSMemory::get_system_total_memory().unwrap_or(4 * 1024 * 1024 * 1024) as f64 * 0.5f64) as usize), + gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((OS::get_system_total_memory().unwrap_or(4 * 1024 * 1024 * 1024) as f64 * 0.5f64) as usize), /// Enable transparent hugepage support for MMTk spaces via madvise (only Linux is supported) /// This only affects the memory for MMTk spaces. transparent_hugepages: bool [|v: &bool| !v || cfg!(target_os = "linux")] = false, @@ -1210,7 +1210,7 @@ mod tests { || { let mut vec = vec![0_u16]; let mut cpu_list = String::new(); - let num_cpus = OSProcess::get_total_num_cpus(); + let num_cpus = OS::get_total_num_cpus(); cpu_list.push('0'); for cpu in 1..num_cpus { diff --git a/src/util/os/imp/mod.rs b/src/util/os/imp/mod.rs index 0a30e5f71e..c4060421df 100644 --- a/src/util/os/imp/mod.rs +++ b/src/util/os/imp/mod.rs @@ -2,24 +2,16 @@ pub(crate) mod windows; #[cfg(target_os = "windows")] -pub use windows::WindowsMemoryImpl as OSMemory; -#[cfg(target_os = "windows")] -pub use windows::WindowsProcessImpl as OSProcess; +pub use windows::Windows as OS; #[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] pub(crate) mod unix_like; #[cfg(target_os = "linux")] -pub use unix_like::linux_like::linux::LinuxMemoryImpl as OSMemory; -#[cfg(target_os = "linux")] -pub use unix_like::linux_like::linux::LinuxProcessImpl as OSProcess; +pub use unix_like::linux_like::linux::Linux as OS; #[cfg(target_os = "android")] -pub use unix_like::linux_like::android::AndroidMemoryImpl as OSMemory; -#[cfg(target_os = "android")] -pub use unix_like::linux_like::android::AndroidProcessImpl as OSProcess; +pub use unix_like::linux_like::android::Android as OS; #[cfg(target_os = "macos")] -pub use unix_like::macos::MacOSMemoryImpl as OSMemory; -#[cfg(target_os = "macos")] -pub use unix_like::macos::MacOSProcessImpl as OSProcess; +pub use unix_like::macos::MacOS as OS; diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs index 3ff4219e49..9a39b56813 100644 --- a/src/util/os/imp/unix_like/linux_like/android.rs +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -5,10 +5,10 @@ use crate::util::os::*; use std::io::Result; -/// Android implementation of the `Memory` trait. -pub struct AndroidMemoryImpl; +/// Android implementation of the `OS` trait. +pub struct Android; -impl Memory for AndroidMemoryImpl { +impl OSMemory for Android { fn dzmmap( start: Address, size: usize, @@ -64,10 +64,7 @@ impl Memory for AndroidMemoryImpl { } } -/// Android implementation of the `Process` trait. -pub struct AndroidProcessImpl; - -impl Process for AndroidProcessImpl { +impl OSProcess for Android { fn get_process_memory_maps() -> Result { linux_common::get_process_memory_maps() } diff --git a/src/util/os/imp/unix_like/linux_like/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs index 735f6bc208..9e1f9fc1b4 100644 --- a/src/util/os/imp/unix_like/linux_like/linux.rs +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -5,10 +5,10 @@ use crate::util::os::*; use std::io::Result; -/// Linux implementation of the `Memory` trait. -pub struct LinuxMemoryImpl; +/// Linux implementation of the `OS` trait. +pub struct Linux; -impl Memory for LinuxMemoryImpl { +impl OSMemory for Linux { fn dzmmap( start: Address, size: usize, @@ -64,10 +64,7 @@ impl Memory for LinuxMemoryImpl { } } -/// Linux implementation of the `Process` trait. -pub struct LinuxProcessImpl; - -impl Process for LinuxProcessImpl { +impl OSProcess for Linux { fn get_process_memory_maps() -> Result { linux_common::get_process_memory_maps() } diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs index 3fb99d890e..d90c7e2d9f 100644 --- a/src/util/os/imp/unix_like/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -3,10 +3,10 @@ use crate::util::os::imp::unix_like::unix_common; use crate::util::os::*; use std::io::Result; -/// MacOS implementation of the `Memory` trait. -pub struct MacOSMemoryImpl; +/// MacOS implementation of the `OS` trait. +pub struct MacOS; -impl Memory for MacOSMemoryImpl { +impl OSMemory for MacOS { fn dzmmap( start: Address, size: usize, @@ -58,10 +58,7 @@ impl MmapStrategy { } } -/// MacOS implementation of the `Process` trait. -pub struct MacOSProcessImpl; - -impl Process for MacOSProcessImpl { +impl OSProcess for MacOS { fn get_process_memory_maps() -> Result { // Get the current process ID (replace this with a specific PID if needed) let pid = std::process::id(); diff --git a/src/util/os/imp/windows.rs b/src/util/os/imp/windows.rs index 1079475c79..7af3073363 100644 --- a/src/util/os/imp/windows.rs +++ b/src/util/os/imp/windows.rs @@ -2,10 +2,10 @@ use crate::util::address::Address; use crate::util::os::memory::*; use std::io::Result; -/// Windows implementation of the `Memory` trait. -pub struct WindowsMemoryImpl; +/// Windows implementation of the `OS` trait. +pub struct Windows; -impl Memory for WindowsMemoryImpl { +impl OSMemory for Windows { fn dzmmap( start: Address, size: usize, @@ -186,10 +186,7 @@ impl MmapProtection { use crate::util::os::process::*; -/// Windows implementation of the `Process` trait. -pub struct WindowsProcessImpl; - -impl Process for WindowsProcessImpl { +impl OSProcess for Windows { fn get_process_memory_maps() -> Result { // Windows-specific implementation to get process memory maps Ok("get_process_memory_maps not implemented for Windows".to_string()) diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index a745954904..ec149a70d8 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -9,14 +9,14 @@ use crate::{ }; /// Abstraction for OS memory operations. -pub trait Memory { +pub trait OSMemory { /// Set a memory region to zero. - fn zero(start: Address, len: usize) { - Self::set(start, 0, len); + fn memzero(start: Address, len: usize) { + Self::memset(start, 0, len); } /// Set a memory region to a specific value. - fn set(start: Address, val: u8, len: usize) { + fn memset(start: Address, val: u8, len: usize) { unsafe { std::ptr::write_bytes::(start.to_mut_ptr(), val, len); } @@ -43,7 +43,7 @@ pub trait Memory { use std::io::ErrorKind; eprintln!("Failed to mmap {}, size {}", addr, bytes); - eprintln!("{}", OSProcess::get_process_memory_maps().unwrap()); + eprintln!("{}", OS::get_process_memory_maps().unwrap()); let call_binding_oom = || { // Signal `MmapOutOfMemory`. Expect the VM to abort immediately. @@ -62,7 +62,7 @@ pub trait Memory { ErrorKind::Other => { // further check the error if let Some(os_errno) = error.raw_os_error() { - if OSMemory::is_mmap_oom(os_errno) { + if OS::is_mmap_oom(os_errno) { call_binding_oom(); } } @@ -72,7 +72,7 @@ pub trait Memory { } _ => { if let Some(os_errno) = error.raw_os_error() { - if OSMemory::is_mmap_oom(os_errno) { + if OS::is_mmap_oom(os_errno) { call_binding_oom(); } } diff --git a/src/util/os/mod.rs b/src/util/os/mod.rs index 24608a3a11..328186d788 100644 --- a/src/util/os/mod.rs +++ b/src/util/os/mod.rs @@ -12,5 +12,6 @@ mod process; pub use process::*; mod imp; -pub use imp::OSMemory; -pub use imp::OSProcess; +pub use imp::OS; + +trait OperatingSystem: OSMemory + OSProcess {} diff --git a/src/util/os/process.rs b/src/util/os/process.rs index ba56f86dd7..7eff4f3c03 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -6,7 +6,7 @@ pub type CoreId = u16; pub type CoreNum = u16; /// Abstraction for OS process operations. -pub trait Process { +pub trait OSProcess { /// Get the memory maps for the process. The returned string is a multi-line string. /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. fn get_process_memory_maps() -> Result; diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 649a380e45..b0d51803e7 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -198,7 +198,7 @@ impl RawMemoryFreeList { } fn mmap(&self, start: Address, bytes: usize) { - let res = OSMemory::dzmmap( + let res = OS::dzmmap( start, bytes, self.strategy, @@ -221,7 +221,7 @@ impl Drop for RawMemoryFreeList { fn drop(&mut self) { let len = self.high_water - self.base; if len != 0 { - let _ = OSMemory::munmap(self.base, len); + let _ = OS::munmap(self.base, len); } } } diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index 6a0cb2b436..91aa50e610 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -111,8 +111,8 @@ pub fn debug_process_thread_id() -> String { use crate::util::os::*; format!( "PID: {}, TID: {}", - OSProcess::get_process_id().unwrap_or_default(), - OSProcess::get_thread_id().unwrap_or_default() + OS::get_process_id().unwrap_or_default(), + OS::get_thread_id().unwrap_or_default() ) } diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs index 3d78b9035f..f343a8d9e2 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_conflict.rs @@ -11,12 +11,11 @@ pub fn test_handle_mmap_conflict() { || { let start = unsafe { Address::from_usize(0x100_0000) }; let one_megabyte = 1000000; - let mmap1_res = - OSMemory::dzmmap(start, one_megabyte, MmapStrategy::TEST, mmap_anno_test!()); + let mmap1_res = OS::dzmmap(start, one_megabyte, MmapStrategy::TEST, mmap_anno_test!()); assert!(mmap1_res.is_ok()); let panic_res = std::panic::catch_unwind(|| { - let mmap2_res = OSMemory::dzmmap( + let mmap2_res = OS::dzmmap( start, one_megabyte, MmapStrategy { @@ -26,7 +25,7 @@ pub fn test_handle_mmap_conflict() { mmap_anno_test!(), ); assert!(mmap2_res.is_err()); - OSMemory::handle_mmap_error::( + OS::handle_mmap_error::( mmap2_res.err().unwrap(), VMThread::UNINITIALIZED, start, diff --git a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs index 6f2cc06eca..e47b2a44b2 100644 --- a/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs +++ b/src/vm/tests/mock_tests/mock_test_handle_mmap_oom.rs @@ -18,10 +18,9 @@ pub fn test_handle_mmap_oom() { let start = unsafe { Address::from_usize(0x100_0000) }; // mmap 1 terabyte memory - we expect this will fail due to out of memory. // If that's not the case, increase the size we mmap. - let mmap_res = - OSMemory::dzmmap(start, LARGE_SIZE, MmapStrategy::TEST, mmap_anno_test!()); + let mmap_res = OS::dzmmap(start, LARGE_SIZE, MmapStrategy::TEST, mmap_anno_test!()); - OSMemory::handle_mmap_error::( + OS::handle_mmap_error::( mmap_res.err().unwrap(), VMThread::UNINITIALIZED, start, From 4ab4f528f820dda4680e717518e0256d7ba0d351 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 10 Feb 2026 03:52:38 +0000 Subject: [PATCH 17/27] Revert windows-related changes. --- .github/scripts/ci-common.sh | 10 - .github/workflows/minimal-tests-core.yml | 2 - Cargo.toml | 9 - src/util/malloc/library.rs | 55 +--- src/util/malloc/malloc_ms_util.rs | 62 ++--- src/util/os/imp/mod.rs | 6 - src/util/os/imp/windows.rs | 234 ------------------ src/util/test_util/mod.rs | 3 - .../tests/mock_tests/mock_test_malloc_ms.rs | 24 +- 9 files changed, 27 insertions(+), 378 deletions(-) delete mode 100644 src/util/os/imp/windows.rs diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index 4aa3f13e7e..892ef56901 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -83,10 +83,6 @@ init_non_exclusive_features() { if [[ ! -z "$feature" ]]; then # Trim whitespaces feature_name=$(trim_ws "$feature") - # jemalloc does not support Windows - if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then - continue - fi features[i]=$feature_name let "i++" fi @@ -94,8 +90,6 @@ init_non_exclusive_features() { done < $cargo_toml non_exclusive_features=$(IFS=$','; echo "${features[*]}") - - echo "Non exclusive features: $non_exclusive_features" } # Get exclusive features @@ -142,10 +136,6 @@ init_exclusive_features() { if [[ ! -z "$feature" ]]; then # Trim whitespaces feature_name=$(trim_ws "$feature") - # jemalloc does not support Windows - if [[ $os == "windows" && $feature_name == "malloc_jemalloc" ]]; then - continue - fi features[i]=$feature_name let "i++" fi diff --git a/.github/workflows/minimal-tests-core.yml b/.github/workflows/minimal-tests-core.yml index 69edf67362..88e9ae4b60 100644 --- a/.github/workflows/minimal-tests-core.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -37,7 +37,6 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } - - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }} @@ -93,7 +92,6 @@ jobs: - { os: ubuntu-22.04, triple: x86_64-unknown-linux-gnu } - { os: ubuntu-22.04, triple: i686-unknown-linux-gnu } - { os: macos-15, triple: x86_64-apple-darwin } - - { os: windows-latest, triple: x86_64-pc-windows-msvc } rust: ${{ fromJson(needs.setup-test-matrix.outputs.rust )}} name: style-check/${{ matrix.target.triple }}/${{ matrix.rust }} diff --git a/Cargo.toml b/Cargo.toml index e244adb175..5ca7a0358e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,15 +54,6 @@ strum = "0.27.1" strum_macros = "0.27.1" sysinfo = "0.33.1" -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = [ - "Win32_Foundation", - "Win32_System_Memory", - "Win32_System_SystemInformation", - "Win32_System_Threading", - "Win32_System_Diagnostics_Debug", -] } - [dev-dependencies] paste = "1.0.8" rand = "0.9.0" diff --git a/src/util/malloc/library.rs b/src/util/malloc/library.rs index b25c677faa..2177a06144 100644 --- a/src/util/malloc/library.rs +++ b/src/util/malloc/library.rs @@ -2,18 +2,10 @@ #[cfg(feature = "malloc_jemalloc")] pub use self::jemalloc::*; -#[cfg(all( - not(target_os = "windows"), - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -))] +#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] pub use self::libc_malloc::*; #[cfg(feature = "malloc_mimalloc")] pub use self::mimalloc::*; -#[cfg(all( - target_os = "windows", - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -))] -pub use self::win_malloc::*; /// When we count page usage of library malloc, we assume they allocate in pages. For some malloc implementations, /// they may use a larger page (e.g. mimalloc's 64K page). For libraries that we are not sure, we assume they use @@ -45,17 +37,13 @@ mod mimalloc { mi_calloc as calloc, mi_free as free, mi_malloc as malloc, mi_realloc as realloc, }; // Posix - #[cfg_attr(target_os = "windows", allow(unused_imports))] pub use mimalloc_sys::mi_posix_memalign as posix_memalign; // GNU pub use mimalloc_sys::mi_malloc_usable_size as malloc_usable_size; } /// If no malloc lib is specified, use the libc implementation -#[cfg(all( - not(target_os = "windows"), - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -))] +#[cfg(not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc",)))] mod libc_malloc { // Normal 4K page pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; @@ -73,42 +61,3 @@ mod libc_malloc { #[cfg(target_os = "macos")] pub use self::malloc_size as malloc_usable_size; } - -/// Windows malloc implementation using HeapAlloc -#[cfg(all( - target_os = "windows", - not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -))] -mod win_malloc { - // Normal 4K page - pub const LOG_BYTES_IN_MALLOC_PAGE: u8 = crate::util::constants::LOG_BYTES_IN_PAGE; - - use std::ffi::c_void; - use windows_sys::Win32::System::Memory::*; - - pub unsafe fn malloc(size: usize) -> *mut c_void { - HeapAlloc(GetProcessHeap(), 0, size) - } - - pub unsafe fn free(ptr: *mut c_void) { - if !ptr.is_null() { - HeapFree(GetProcessHeap(), 0, ptr); - } - } - - pub unsafe fn calloc(nmemb: usize, size: usize) -> *mut c_void { - let total = nmemb * size; - HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, total) - } - - pub unsafe fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void { - if ptr.is_null() { - return malloc(size); - } - HeapReAlloc(GetProcessHeap(), 0, ptr, size) - } - - pub unsafe fn malloc_usable_size(ptr: *const c_void) -> usize { - HeapSize(GetProcessHeap(), 0, ptr) - } -} diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 5a9b2be44e..45076cf07d 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -3,14 +3,6 @@ use crate::util::malloc::library::*; use crate::util::Address; use crate::vm::VMBinding; -/// Allocate with alignment. This also guarantees the memory is zero initialized. -/// This uses posix_memalign, which is not available on Windows. -/// This would somehow affect `MallocMarkSweep` performance on Windows. -// #[cfg(all( -// not(target_os = "windows"), -// not(any(feature = "malloc_jemalloc", feature = "malloc_mimalloc")) -// ))] -#[cfg(not(target_os = "windows"))] pub fn align_alloc(size: usize, align: usize) -> Address { use crate::util::os::*; let mut ptr = std::ptr::null_mut::(); @@ -79,38 +71,28 @@ pub fn alloc(size: usize, align: usize, offset: usize) -> (Addres let mut is_offset_malloc = false; // malloc returns 16 bytes aligned address. // So if the alignment is smaller than 16 bytes, we do not need to align. - - match (align, offset) { - (a, 0) if a <= 16 => { - let raw = unsafe { calloc(1, size) }; - address = Address::from_mut_ptr(raw); - debug_assert!(address.is_aligned_to(align)); - } - #[cfg(not(target_os = "windows"))] - // On non-Windows platforms with posix_memalign, we can use align_alloc for alignments > 16 - // However, on Windows, there is no equivalent function. - // The memory alloc by `align_alloc` may not be freed correctly by `free`. - // So we use offset allocation for all alignments > 16 on Windows. - (a, 0) if a > 16 => { - address = align_alloc(size, align); - debug_assert!( - address.is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {}", - address, - align - ); - } - _ => { - address = align_offset_alloc::(size, align, offset); - is_offset_malloc = true; - debug_assert!( - (address + offset).is_aligned_to(align), - "Address: {:x} is not aligned to the given alignment: {} at offset: {}", - address, - align, - offset - ); - } + if align <= 16 && offset == 0 { + let raw = unsafe { calloc(1, size) }; + address = Address::from_mut_ptr(raw); + debug_assert!(address.is_aligned_to(align)); + } else if align > 16 && offset == 0 { + address = align_alloc(size, align); + debug_assert!( + address.is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {}", + address, + align + ); + } else { + address = align_offset_alloc::(size, align, offset); + is_offset_malloc = true; + debug_assert!( + (address + offset).is_aligned_to(align), + "Address: {:x} is not aligned to the given alignment: {} at offset: {}", + address, + align, + offset + ); } (address, is_offset_malloc) } diff --git a/src/util/os/imp/mod.rs b/src/util/os/imp/mod.rs index c4060421df..5e1fdc9680 100644 --- a/src/util/os/imp/mod.rs +++ b/src/util/os/imp/mod.rs @@ -1,9 +1,3 @@ -#[cfg(target_os = "windows")] -pub(crate) mod windows; - -#[cfg(target_os = "windows")] -pub use windows::Windows as OS; - #[cfg(any(target_os = "linux", target_os = "macos", target_os = "android"))] pub(crate) mod unix_like; diff --git a/src/util/os/imp/windows.rs b/src/util/os/imp/windows.rs deleted file mode 100644 index 7af3073363..0000000000 --- a/src/util/os/imp/windows.rs +++ /dev/null @@ -1,234 +0,0 @@ -use crate::util::address::Address; -use crate::util::os::memory::*; -use std::io::Result; - -/// Windows implementation of the `OS` trait. -pub struct Windows; - -impl OSMemory for Windows { - fn dzmmap( - start: Address, - size: usize, - strategy: MmapStrategy, - _annotation: &MmapAnnotation<'_>, - ) -> Result
{ - use std::io; - use windows_sys::Win32::System::Memory::{ - VirtualAlloc, VirtualQuery, MEMORY_BASIC_INFORMATION, MEM_COMMIT, MEM_FREE, MEM_RESERVE, - }; - - let ptr: *mut u8 = start.to_mut_ptr(); - - // Has to COMMIT immediately if: - // - not MAP_NORESERVE - // - and protection is not NoAccess - let commit = strategy.reserve && !matches!(strategy.prot, MmapProtection::NoAccess); - - // Scan the region [ptr, ptr + size) to understand its current state - unsafe { - let mut addr = ptr; - let end = ptr.add(size); - - let mut saw_free = false; - let mut saw_reserved = false; - let mut saw_committed = false; - - while addr < end { - let mut mbi: MEMORY_BASIC_INFORMATION = std::mem::zeroed(); - let q = VirtualQuery( - addr as *const _, - &mut mbi, - std::mem::size_of::(), - ); - if q == 0 { - return Err(io::Error::last_os_error()); - } - - let region_base = mbi.BaseAddress as *mut u8; - let region_size = mbi.RegionSize; - let region_end = region_base.add(region_size); - - // Calculate the intersection of [addr, end) and [region_base, region_end) - let _sub_begin = if addr > region_base { - addr - } else { - region_base - }; - let _sub_end = if end < region_end { end } else { region_end }; - - match mbi.State { - MEM_FREE => saw_free = true, - MEM_RESERVE => saw_reserved = true, - MEM_COMMIT => saw_committed = true, - _ => { - return Err(io::Error::other("Unexpected memory state in mmap_fixed")); - } - } - - // Jump to the next region (VirtualQuery always returns "continuous regions with the same attributes") - addr = region_end; - } - - // 1. All FREE: make a new mapping in the region - // 2. All RESERVE/COMMIT: treat as an existing mapping, can just COMMIT or succeed directly - // 3. MIX of FREE + others: not allowed (semantically similar to MAP_FIXED_NOREPLACE) - if saw_free && (saw_reserved || saw_committed) { - return Err(io::Error::from_raw_os_error( - windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32, - )); - } - - if saw_free && !saw_reserved && !saw_committed { - // All FREE: make a new mapping in the region - let mut allocation_type = MEM_RESERVE; - if commit { - allocation_type |= MEM_COMMIT; - } - - let res = VirtualAlloc( - ptr as *mut _, - size, - allocation_type, - strategy.prot.get_native_flags(), - ); - if res.is_null() { - return Err(io::Error::last_os_error()); - } - - Ok(start) - } else { - // This behavior is similar to mmap with MAP_FIXED on Linux. - // If the region is already mapped, we just ensure the required commitment. - // If commit is not needed, we just return Ok. - if commit { - let res = VirtualAlloc( - ptr as *mut _, - size, - MEM_COMMIT, - strategy.prot.get_native_flags(), - ); - if res.is_null() { - return Err(io::Error::last_os_error()); - } - } - Ok(start) - } - } - } - - fn munmap(start: Address, size: usize) -> Result<()> { - use windows_sys::Win32::System::Memory::*; - // Using MEM_DECOMMIT will decommit the memory but leave the address space reserved. - // This is the safest way to emulate munmap on Windows, as MEM_RELEASE would free - // the entire allocation, which could be larger than the requested size. - let res = unsafe { VirtualFree(start.to_mut_ptr(), size, MEM_DECOMMIT) }; - if res == 0 { - // If decommit fails, we try to release the memory. This might happen if the memory was - // only reserved. - let res_release = unsafe { VirtualFree(start.to_mut_ptr(), 0, MEM_RELEASE) }; - if res_release == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } else { - Ok(()) - } - } - - fn mprotect(start: Address, size: usize) -> Result<()> { - use windows_sys::Win32::System::Memory::*; - let mut old_protect = 0; - let res = - unsafe { VirtualProtect(start.to_mut_ptr(), size, PAGE_NOACCESS, &mut old_protect) }; - if res == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } - - fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - use windows_sys::Win32::System::Memory::*; - let prot = prot.get_native_flags(); - let mut old_protect = 0; - let res = unsafe { VirtualProtect(start.to_mut_ptr(), size, prot, &mut old_protect) }; - if res == 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } - } - - fn panic_if_unmapped(start: Address, size: usize) { - warn!( - "Check if {} of size {} is mapped is ignored on Windows", - start, size - ); - } - - fn is_mmap_oom(os_errno: i32) -> bool { - os_errno == windows_sys::Win32::Foundation::ERROR_NOT_ENOUGH_MEMORY as i32 - || os_errno == windows_sys::Win32::Foundation::ERROR_INVALID_ADDRESS as i32 - } -} - -impl MmapProtection { - fn get_native_flags(&self) -> u32 { - use windows_sys::Win32::System::Memory::*; - match self { - Self::ReadWrite => PAGE_READWRITE, - Self::ReadWriteExec => PAGE_EXECUTE_READWRITE, - Self::NoAccess => PAGE_NOACCESS, - } - } -} - -use crate::util::os::process::*; - -impl OSProcess for Windows { - fn get_process_memory_maps() -> Result { - // Windows-specific implementation to get process memory maps - Ok("get_process_memory_maps not implemented for Windows".to_string()) - } - - fn get_process_id() -> Result { - use windows_sys::Win32::System::Threading::GetCurrentProcessId; - Ok(format!("{}", unsafe { GetCurrentProcessId() })) - } - - fn get_thread_id() -> Result { - use windows_sys::Win32::System::Threading::GetCurrentThreadId; - Ok(format!("{}", unsafe { GetCurrentThreadId() })) - } - - fn get_total_num_cpus() -> CoreNum { - unsafe { - windows_sys::Win32::System::Threading::GetActiveProcessorCount( - windows_sys::Win32::System::Threading::ALL_PROCESSOR_GROUPS, - ) as CoreNum - } - } - - fn bind_current_thread_to_core(core_id: CoreId) { - unsafe { - windows_sys::Win32::System::Threading::SetThreadAffinityMask( - windows_sys::Win32::System::Threading::GetCurrentThread(), - 1 << core_id, - ); - } - } - - fn bind_current_thread_to_cpuset(core_ids: &[CoreId]) { - let mut mask = 0; - for cpu in core_ids { - mask |= 1 << cpu; - } - unsafe { - windows_sys::Win32::System::Threading::SetThreadAffinityMask( - windows_sys::Win32::System::Threading::GetCurrentThread(), - mask, - ); - } - } -} diff --git a/src/util/test_util/mod.rs b/src/util/test_util/mod.rs index be71905b97..4540cee3b8 100644 --- a/src/util/test_util/mod.rs +++ b/src/util/test_util/mod.rs @@ -53,9 +53,6 @@ const TEST_ADDRESS: Address = #[cfg(target_os = "macos")] const TEST_ADDRESS: Address = crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x2_0000_0000) }); -#[cfg(target_os = "windows")] -const TEST_ADDRESS: Address = - crate::util::conversions::chunk_align_down(unsafe { Address::from_usize(0x5_0000_0000) }); // util::heap::layout::mmapper::csm pub(crate) const CHUNK_STATE_MMAPPER_TEST_REGION: MmapTestRegion = diff --git a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs index e9c9d28b1e..003575dc6e 100644 --- a/src/vm/tests/mock_tests/mock_test_malloc_ms.rs +++ b/src/vm/tests/mock_tests/mock_test_malloc_ms.rs @@ -18,20 +18,7 @@ fn test_malloc() { assert!((address4 + 4_isize).is_aligned_to(64)); assert!(!bool1); - - // Since Windows HeapAlloc only guarantees 16-byte alignment, the allocation with 32-byte alignment - // without offset will be treated as an offset allocation. - if cfg!(all( - not(target_os = "windows"), - not(any( - feature = "malloc_jemalloc", - feature = "malloc_mimalloc" - )) - )) { - assert!(!bool2); - } else { - assert!(bool2); - } + assert!(!bool2); assert!(bool3); assert!(bool4); @@ -44,13 +31,8 @@ fn test_malloc() { unsafe { malloc_ms_util::free(address1.to_mut_ptr()); } - - if !bool2 { - unsafe { - malloc_ms_util::free(address2.to_mut_ptr()); - } - } else { - malloc_ms_util::offset_free(address2); + unsafe { + malloc_ms_util::free(address2.to_mut_ptr()); } malloc_ms_util::offset_free(address3); malloc_ms_util::offset_free(address4); From b638bad5a3e19b7362a25e6cf4128dc2dbfcf0f2 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 10 Feb 2026 04:15:19 +0000 Subject: [PATCH 18/27] Fix macos build --- src/util/os/imp/unix_like/macos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs index d90c7e2d9f..cb371acc95 100644 --- a/src/util/os/imp/unix_like/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -20,7 +20,7 @@ impl OSMemory for MacOS { // Zero memory if we actually reserve the memory if strategy.reserve { - Self::zero(start, size); + Self::memzero(start, size); } Ok(addr) } From 67c5789b2b142eb01034a5467b893c9c958246b9 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 11 Feb 2026 03:34:27 +0000 Subject: [PATCH 19/27] Put set/zero back to crate::util::memory. Other minor changes to address reviews. --- src/policy/copyspace.rs | 8 ++------ src/policy/immix/block.rs | 5 +---- src/policy/lockfreeimmortalspace.rs | 2 +- src/policy/markcompactspace.rs | 3 +-- src/policy/space.rs | 2 +- src/util/alloc/free_list_allocator.rs | 3 +-- src/util/alloc/immix_allocator.rs | 3 +-- src/util/malloc/malloc_ms_util.rs | 3 +-- src/util/memory.rs | 13 +++++++++++++ src/util/metadata/side_metadata/global.rs | 4 ++-- src/util/mod.rs | 2 ++ src/util/os/imp/unix_like/macos.rs | 2 +- src/util/os/memory.rs | 12 ------------ 13 files changed, 27 insertions(+), 35 deletions(-) create mode 100644 src/util/memory.rs diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 685231c0d5..46e02a606e 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -293,7 +293,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::os::OS::mprotect(start, extent) { + if let Err(e) = OS::mprotect(start, extent) { panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); @@ -308,11 +308,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = crate::util::os::OS::munprotect( - start, - extent, - crate::util::os::MmapProtection::ReadWriteExec, - ) { + if let Err(e) = OS::munprotect(start, extent, MmapProtection::ReadWriteExec) { panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index 7d7b2fcea8..ad6bc12de4 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -253,10 +253,7 @@ impl Block { line.mark(0); } #[cfg(feature = "immix_zero_on_release")] - { - use crate::util::os::*; - OS::memzero(line.start(), Line::BYTES); - } + crate::util::memory::zero(line.start(), Line::BYTES); // We need to clear the pin bit if it is on the side, as this line can be reused #[cfg(feature = "object_pinning")] diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index 5e04e5c6eb..6f6636dfc2 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -157,7 +157,7 @@ impl Space for LockFreeImmortalSpace { } } if self.slow_path_zeroing { - crate::util::os::OS::memzero(start, bytes); + crate::util::memory::zero(start, bytes); } start } diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index e6dd01134b..626422b6a4 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -12,7 +12,6 @@ use crate::util::copy::CopySemantics; use crate::util::heap::{MonotonePageResource, PageResource}; use crate::util::metadata::{extract_side_metadata, vo_bit}; use crate::util::object_enum::{self, ObjectEnumerator}; -use crate::util::os::*; use crate::util::{Address, ObjectReference}; use crate::{vm::*, ObjectQueue}; use atomic::Ordering; @@ -226,7 +225,7 @@ impl MarkCompactSpace { // Clear header forwarding pointer for an object fn clear_header_forwarding_pointer(object: ObjectReference) { - OS::memzero( + crate::util::memory::zero( Self::header_forwarding_pointer_address(object), GC_EXTRA_HEADER_BYTES, ); diff --git a/src/policy/space.rs b/src/policy/space.rs index 36aa50ca61..80d39d5589 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -232,7 +232,7 @@ pub trait Space: 'static + SFT + Sync + Downcast { // TODO: Concurrent zeroing if self.common().zeroed { - OS::memzero(res.start, bytes); + crate::util::memory::zero(res.start, bytes); } // Some assertions diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index c506902a10..9608973902 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -7,7 +7,6 @@ use crate::policy::marksweepspace::native_ms::*; use crate::util::alloc::allocator; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; -use crate::util::os::*; use crate::util::Address; use crate::util::VMThread; use crate::vm::VMBinding; @@ -166,7 +165,7 @@ impl FreeListAllocator { // Zeroing memory right before we return it. // If we move the zeroing to somewhere else, we need to clear the list link here: cell.store::
(Address::ZERO) let cell_size = block.load_block_cell_size(); - crate::util::os::OS::memzero(cell, cell_size); + crate::util::memory::zero(cell, cell_size); // Make sure the memory is zeroed. This looks silly as we zero the cell right before this check. // But we would need to move the zeroing to somewhere so we can do zeroing at a coarser grainularity. diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index 7dd9d5b201..eb2e5235fa 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -10,7 +10,6 @@ use crate::util::alloc::allocator::get_maximum_aligned_size; use crate::util::alloc::Allocator; use crate::util::linear_scan::Region; use crate::util::opaque_pointer::VMThread; -use crate::util::os::*; use crate::util::rust_util::unlikely; use crate::util::Address; use crate::vm::*; @@ -251,7 +250,7 @@ impl ImmixAllocator { end_line, self.tls ); - OS::memzero( + crate::util::memory::zero( self.bump_pointer.cursor, self.bump_pointer.limit - self.bump_pointer.cursor, ); diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 45076cf07d..6ebe939d4d 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -4,7 +4,6 @@ use crate::util::Address; use crate::vm::VMBinding; pub fn align_alloc(size: usize, align: usize) -> Address { - use crate::util::os::*; let mut ptr = std::ptr::null_mut::(); let ptr_ptr = std::ptr::addr_of_mut!(ptr); let result = unsafe { posix_memalign(ptr_ptr, align, size) }; @@ -12,7 +11,7 @@ pub fn align_alloc(size: usize, align: usize) -> Address { return Address::ZERO; } let address = Address::from_mut_ptr(ptr); - OS::memzero(address, size); + crate::util::memory::zero(address, size); address } diff --git a/src/util/memory.rs b/src/util/memory.rs new file mode 100644 index 0000000000..e3e04efc53 --- /dev/null +++ b/src/util/memory.rs @@ -0,0 +1,13 @@ +use crate::util::Address; + +/// Set a range of memory to 0. +pub fn zero(start: Address, len: usize) { + set(start, 0, len); +} + +/// Set a range of memory to the given value. Similar to memset. +pub fn set(start: Address, val: u8, len: usize) { + unsafe { + std::ptr::write_bytes(start.to_mut_ptr::(), val, len); + } +} diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index a60211bfbb..956ced34c5 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -174,7 +174,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - OS::memzero(start, end - start); + crate::util::memory::zero(start, end - start); false } BitByteRange::BitsInByte { @@ -211,7 +211,7 @@ impl SideMetadataSpec { let mut visitor = |range| { match range { BitByteRange::Bytes { start, end } => { - OS::memset(start, 0xff, end - start); + crate::util::memory::set(start, 0xff, end - start); false } BitByteRange::BitsInByte { diff --git a/src/util/mod.rs b/src/util/mod.rs index 30f6269fc0..59dcd5cce5 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -28,6 +28,8 @@ pub mod is_mmtk_object; pub mod linear_scan; /// Various malloc implementations (conditionally compiled by features) pub mod malloc; +/// Memory utilities (non-OS dependent). OS dependent memory utilities can be found in [`crate::util::os::memory`]. +pub mod memory; /// Metadata (OnSide or InHeader) implementation. pub mod metadata; /// Opaque pointers used in MMTk, e.g. VMThread. diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs index cb371acc95..13e2cd7846 100644 --- a/src/util/os/imp/unix_like/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -20,7 +20,7 @@ impl OSMemory for MacOS { // Zero memory if we actually reserve the memory if strategy.reserve { - Self::memzero(start, size); + crate::util::memory::zero(start, size); } Ok(addr) } diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index ec149a70d8..9e25930954 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -10,18 +10,6 @@ use crate::{ /// Abstraction for OS memory operations. pub trait OSMemory { - /// Set a memory region to zero. - fn memzero(start: Address, len: usize) { - Self::memset(start, 0, len); - } - - /// Set a memory region to a specific value. - fn memset(start: Address, val: u8, len: usize) { - unsafe { - std::ptr::write_bytes::(start.to_mut_ptr(), val, len); - } - } - /// Perform a demand-zero mmap. /// /// Falback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. From dd24170ab27470aa1e91600917841a9cf0b39749 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 11 Feb 2026 03:49:51 +0000 Subject: [PATCH 20/27] Remove MmapStrategy::SIDE_METADATA (unused). Minor fix to unprotect --- src/policy/copyspace.rs | 2 +- src/util/os/memory.rs | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 46e02a606e..aa22857a2f 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -308,7 +308,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = OS::munprotect(start, extent, MmapProtection::ReadWriteExec) { + if let Err(e) = OS::munprotect(start, extent, self.common().mmap_protection()) { panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 9e25930954..5fdd727d8b 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -199,13 +199,6 @@ impl MmapStrategy { reserve: true, }; - #[cfg(test)] - /// The strategy for MMTk side metadata (test) - pub const SIDE_METADATA: Self = Self::TEST; - #[cfg(not(test))] - /// The strategy for MMTk side metadata - pub const SIDE_METADATA: Self = Self::INTERNAL_MEMORY; - /// The strategy for MMTk's test memory #[cfg(test)] pub const TEST: Self = Self { From 474d60a0ba3e002e8c8a47ee4de8b32128e2dcb5 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 11 Feb 2026 03:53:23 +0000 Subject: [PATCH 21/27] Further remove some changes due to windows support --- .github/scripts/ci-common.sh | 24 +++---------------- .../ci-setup-x86_64-pc-windows-msvc.sh | 0 .github/workflows/minimal-tests-core.yml | 8 ------- 3 files changed, 3 insertions(+), 29 deletions(-) delete mode 100644 .github/scripts/ci-setup-x86_64-pc-windows-msvc.sh diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index 45a7c33b09..dd4e2d1370 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -12,19 +12,6 @@ dummyvm_toml=$project_root/docs/dummyvm/Cargo.toml # Pin certain deps for our MSRV cargo update -p home@0.5.12 --precise 0.5.5 # This requires Rust edition 2024 -# Read a line and strip trailing CR (works for CRLF and LF files) -strip_cr() { - local s=$1 - printf '%s' "${s%$'\r'}" -} - -# Trim all whitespace (for feature keys) -trim_ws() { - local s=$1 - # remove all whitespace characters - printf '%s' "${s//[[:space:]]/}" -} - # Repeat a command for all the features. Requires the command as one argument (with double quotes) for_all_features() { # without mutually exclusive features @@ -59,8 +46,6 @@ init_non_exclusive_features() { i=0 while IFS= read -r line; do - line=$(strip_cr "$line") - # Only parse non mutally exclusive features if [[ $line == *"-- Non mutually exclusive features --"* ]]; then parse_features=true @@ -81,8 +66,8 @@ init_non_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - feature_name=$(trim_ws "$feature") - features[i]=$feature_name + feature=$(echo $feature) + features[i]=$feature let "i++" fi fi @@ -104,8 +89,6 @@ init_exclusive_features() { declare -a features=() while IFS= read -r line; do - line=$(strip_cr "$line") - # Only parse mutally exclusive features if [[ $line == *"-- Mutally exclusive features --"* ]]; then parse_features=true @@ -134,8 +117,7 @@ init_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - feature_name=$(trim_ws "$feature") - features[i]=$feature_name + features[i]=$(echo $feature) let "i++" fi fi diff --git a/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh b/.github/scripts/ci-setup-x86_64-pc-windows-msvc.sh deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/.github/workflows/minimal-tests-core.yml b/.github/workflows/minimal-tests-core.yml index 88e9ae4b60..2a113ca850 100644 --- a/.github/workflows/minimal-tests-core.yml +++ b/.github/workflows/minimal-tests-core.yml @@ -42,10 +42,6 @@ jobs: name: minimal-tests-core/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} - defaults: - run: - shell: bash - env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" @@ -97,10 +93,6 @@ jobs: name: style-check/${{ matrix.target.triple }}/${{ matrix.rust }} runs-on: ${{ matrix.target.os }} - defaults: - run: - shell: bash - env: # This determines the default target which cargo-build, cargo-test, etc. use. CARGO_BUILD_TARGET: "${{ matrix.target.triple }}" From 98533468f0fb4e3afd7a9a9368897dba49886f6b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 11 Feb 2026 04:15:11 +0000 Subject: [PATCH 22/27] Fix style check --- src/util/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/mod.rs b/src/util/mod.rs index 59dcd5cce5..6a542648f8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -28,7 +28,7 @@ pub mod is_mmtk_object; pub mod linear_scan; /// Various malloc implementations (conditionally compiled by features) pub mod malloc; -/// Memory utilities (non-OS dependent). OS dependent memory utilities can be found in [`crate::util::os::memory`]. +/// Memory utilities (non-OS dependent). OS dependent memory utilities can be found in [`crate::util::os::OSMemory`]. pub mod memory; /// Metadata (OnSide or InHeader) implementation. pub mod metadata; From 43906003893e77b139288b7848f0e3edad5ed291 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 12 Feb 2026 03:32:29 +0000 Subject: [PATCH 23/27] Minor fix. Change mprotect/munprotect to set_memory_access. --- .github/scripts/ci-common.sh | 3 +-- src/policy/copyspace.rs | 4 ++-- src/util/heap/freelistpageresource.rs | 8 ++++++-- src/util/os/imp/unix_like/linux_like/android.rs | 8 ++------ src/util/os/imp/unix_like/linux_like/linux.rs | 8 ++------ src/util/os/imp/unix_like/macos.rs | 8 ++------ src/util/os/imp/unix_like/unix_common.rs | 10 +--------- src/util/os/memory.rs | 8 +++----- 8 files changed, 19 insertions(+), 38 deletions(-) diff --git a/.github/scripts/ci-common.sh b/.github/scripts/ci-common.sh index dd4e2d1370..0db925b015 100644 --- a/.github/scripts/ci-common.sh +++ b/.github/scripts/ci-common.sh @@ -66,8 +66,7 @@ init_non_exclusive_features() { IFS='='; feature=($line); unset IFS; if [[ ! -z "$feature" ]]; then # Trim whitespaces - feature=$(echo $feature) - features[i]=$feature + features[i]=$(echo $feature) let "i++" fi fi diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index aa22857a2f..c07f96839f 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -293,7 +293,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = OS::mprotect(start, extent) { + if let Err(e) = OS::set_memory_access(start, extent, MmapProtection::NoAccess) { panic!("Failed to protect memory: {:?}", e); } trace!("Protect {:x} {:x}", start, start + extent); @@ -308,7 +308,7 @@ impl CopySpace { } let start = self.common().start; let extent = self.common().extent; - if let Err(e) = OS::munprotect(start, extent, self.common().mmap_protection()) { + if let Err(e) = OS::set_memory_access(start, extent, self.common().mmap_protection()) { panic!("Failed to unprotect memory: {:?}", e); } trace!("Unprotect {:x} {:x}", start, start + extent); diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index c4a248fbe7..b80b522dda 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -219,7 +219,11 @@ impl FreeListPageResource { // > (e.g., read versus read/write protection) exceeding the // > allowed maximum. assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = OS::mprotect(start, conversions::pages_to_bytes(pages)) { + if let Err(e) = OS::set_memory_access( + start, + conversions::pages_to_bytes(pages), + MmapProtection::NoAccess, + ) { panic!( "Failed at protecting memory (starting at {}): {:?}", start, e @@ -230,7 +234,7 @@ impl FreeListPageResource { /// Unprotect the memory fn munprotect(&self, start: Address, pages: usize) { assert!(self.protect_memory_on_release.is_some()); - if let Err(e) = OS::munprotect( + if let Err(e) = OS::set_memory_access( start, conversions::pages_to_bytes(pages), self.protect_memory_on_release.unwrap(), diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs index 9a39b56813..573c085992 100644 --- a/src/util/os/imp/unix_like/linux_like/android.rs +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -32,12 +32,8 @@ impl OSMemory for Android { unix_common::munmap(start, size) } - fn mprotect(start: Address, size: usize) -> Result<()> { - unix_common::mprotect(start, size) - } - - fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - unix_common::munprotect(start, size, prot) + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) } fn is_mmap_oom(os_errno: i32) -> bool { diff --git a/src/util/os/imp/unix_like/linux_like/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs index 9e1f9fc1b4..a3b4d17ef6 100644 --- a/src/util/os/imp/unix_like/linux_like/linux.rs +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -32,12 +32,8 @@ impl OSMemory for Linux { unix_common::munmap(start, size) } - fn mprotect(start: Address, size: usize) -> Result<()> { - unix_common::mprotect(start, size) - } - - fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - unix_common::munprotect(start, size, prot) + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) } fn is_mmap_oom(os_errno: i32) -> bool { diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs index 13e2cd7846..7fa653fe85 100644 --- a/src/util/os/imp/unix_like/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -29,12 +29,8 @@ impl OSMemory for MacOS { unix_common::munmap(start, size) } - fn mprotect(start: Address, size: usize) -> Result<()> { - unix_common::mprotect(start, size) - } - - fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { - unix_common::munprotect(start, size, prot) + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()> { + unix_common::mprotect(start, size, prot) } fn is_mmap_oom(os_errno: i32) -> bool { diff --git a/src/util/os/imp/unix_like/unix_common.rs b/src/util/os/imp/unix_like/unix_common.rs index 8a32e215c6..ef589ae065 100644 --- a/src/util/os/imp/unix_like/unix_common.rs +++ b/src/util/os/imp/unix_like/unix_common.rs @@ -32,15 +32,7 @@ pub fn munmap(start: Address, size: usize) -> Result<()> { wrap_libc_call(&|| unsafe { libc::munmap(start.to_mut_ptr(), size) }, 0) } -pub fn mprotect(start: Address, size: usize) -> Result<()> { - let prot = libc::PROT_NONE; - wrap_libc_call( - &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot) }, - 0, - ) -} - -pub fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { +pub fn mprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> { wrap_libc_call( &|| unsafe { libc::mprotect(start.to_mut_ptr(), size, prot.get_native_flags()) }, 0, diff --git a/src/util/os/memory.rs b/src/util/os/memory.rs index 5fdd727d8b..006d4059e0 100644 --- a/src/util/os/memory.rs +++ b/src/util/os/memory.rs @@ -12,7 +12,8 @@ use crate::{ pub trait OSMemory { /// Perform a demand-zero mmap. /// - /// Falback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. + /// Fallback: `annotation` is only used for debugging. For platforms that do not support mmap annotations, this parameter can be ignored. + /// Fallback: see [`crate::util::os::MmapStrategy`] for fallbacks for `strategy`. fn dzmmap( start: Address, size: usize, @@ -75,11 +76,8 @@ pub trait OSMemory { /// Unmap a memory region. fn munmap(start: Address, size: usize) -> Result<()>; - /// Change the protection of a memory region to no access to forbit any access to the memory region. - fn mprotect(start: Address, size: usize) -> Result<()>; - /// Change the protection of a memory region to the specified protection. - fn munprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()>; + fn set_memory_access(start: Address, size: usize, prot: MmapProtection) -> Result<()>; /// Checks if the memory has already been mapped. If not, we panic. /// From 8e44495e1097d4244ea6bc1493c0b8b0c0d6f4e0 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 12 Feb 2026 03:59:02 +0000 Subject: [PATCH 24/27] Add Process/ThreadIDType for OSProcess --- src/util/os/imp/unix_like/linux_like/android.rs | 7 +++++-- src/util/os/imp/unix_like/linux_like/linux.rs | 7 +++++-- src/util/os/imp/unix_like/macos.rs | 7 +++++-- src/util/os/imp/unix_like/unix_common.rs | 13 +++++++------ src/util/os/process.rs | 8 ++++++-- src/util/rust_util/mod.rs | 4 ++-- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs index 573c085992..31343bee59 100644 --- a/src/util/os/imp/unix_like/linux_like/android.rs +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -61,15 +61,18 @@ impl OSMemory for Android { } impl OSProcess for Android { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + fn get_process_memory_maps() -> Result { linux_common::get_process_memory_maps() } - fn get_process_id() -> Result { + fn get_process_id() -> Result { unix_common::get_process_id() } - fn get_thread_id() -> Result { + fn get_thread_id() -> Result { unix_common::get_thread_id() } diff --git a/src/util/os/imp/unix_like/linux_like/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs index a3b4d17ef6..e18e8fead9 100644 --- a/src/util/os/imp/unix_like/linux_like/linux.rs +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -61,15 +61,18 @@ impl OSMemory for Linux { } impl OSProcess for Linux { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + fn get_process_memory_maps() -> Result { linux_common::get_process_memory_maps() } - fn get_process_id() -> Result { + fn get_process_id() -> Result { unix_common::get_process_id() } - fn get_thread_id() -> Result { + fn get_thread_id() -> Result { unix_common::get_thread_id() } diff --git a/src/util/os/imp/unix_like/macos.rs b/src/util/os/imp/unix_like/macos.rs index 7fa653fe85..5d73ddd01b 100644 --- a/src/util/os/imp/unix_like/macos.rs +++ b/src/util/os/imp/unix_like/macos.rs @@ -55,6 +55,9 @@ impl MmapStrategy { } impl OSProcess for MacOS { + type ProcessIDType = unix_common::ProcessIDType; + type ThreadIDType = unix_common::ThreadIDType; + fn get_process_memory_maps() -> Result { // Get the current process ID (replace this with a specific PID if needed) let pid = std::process::id(); @@ -82,11 +85,11 @@ impl OSProcess for MacOS { } } - fn get_process_id() -> Result { + fn get_process_id() -> Result { unix_common::get_process_id() } - fn get_thread_id() -> Result { + fn get_thread_id() -> Result { unix_common::get_thread_id() } diff --git a/src/util/os/imp/unix_like/unix_common.rs b/src/util/os/imp/unix_like/unix_common.rs index ef589ae065..6fa643dc48 100644 --- a/src/util/os/imp/unix_like/unix_common.rs +++ b/src/util/os/imp/unix_like/unix_common.rs @@ -39,14 +39,15 @@ pub fn mprotect(start: Address, size: usize, prot: MmapProtection) -> Result<()> ) } -pub fn get_process_id() -> Result { - let pid = unsafe { libc::getpid() }; - Ok(format!("{}", pid)) +pub type ProcessIDType = libc::pid_t; +pub type ThreadIDType = libc::pthread_t; + +pub fn get_process_id() -> Result { + Ok(unsafe { libc::getpid() }) } -pub fn get_thread_id() -> Result { - let tid = unsafe { libc::pthread_self() }; - Ok(format!("{}", tid)) +pub fn get_thread_id() -> Result { + Ok(unsafe { libc::pthread_self() }) } pub fn wrap_libc_call(f: &dyn Fn() -> T, expect: T) -> Result<()> { diff --git a/src/util/os/process.rs b/src/util/os/process.rs index 7eff4f3c03..6ade258b48 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -1,3 +1,4 @@ +use std::fmt::Display; use std::io::Result; /// Representation of a CPU core identifier. @@ -7,17 +8,20 @@ pub type CoreNum = u16; /// Abstraction for OS process operations. pub trait OSProcess { + type ProcessIDType: Display + Eq + Copy; + type ThreadIDType: Display + Eq + Copy; + /// Get the memory maps for the process. The returned string is a multi-line string. /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. fn get_process_memory_maps() -> Result; /// Get the process ID as a string. /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. - fn get_process_id() -> Result; + fn get_process_id() -> Result; //// Get the thread ID as a string. /// Fallback: This is only used for debugging. For unimplemented cases, this function can return a placeholder Ok value. - fn get_thread_id() -> Result; + fn get_thread_id() -> Result; /// Return the total number of cores allocated to the program. fn get_total_num_cpus() -> CoreNum; diff --git a/src/util/rust_util/mod.rs b/src/util/rust_util/mod.rs index 91aa50e610..00a972ac2e 100644 --- a/src/util/rust_util/mod.rs +++ b/src/util/rust_util/mod.rs @@ -111,8 +111,8 @@ pub fn debug_process_thread_id() -> String { use crate::util::os::*; format!( "PID: {}, TID: {}", - OS::get_process_id().unwrap_or_default(), - OS::get_thread_id().unwrap_or_default() + OS::get_process_id().map_or("(Failed to get PID)".to_string(), |pid| format!("{}", pid)), + OS::get_thread_id().map_or("(Failed to get TID)".to_string(), |tid| format!("{}", tid)), ) } From 46b0818180b75d9674b03b14eb0e19976a82b742 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 12 Feb 2026 04:29:16 +0000 Subject: [PATCH 25/27] Add doc for process/thread id type --- src/util/os/process.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/os/process.rs b/src/util/os/process.rs index 6ade258b48..da2f78fc93 100644 --- a/src/util/os/process.rs +++ b/src/util/os/process.rs @@ -8,7 +8,9 @@ pub type CoreNum = u16; /// Abstraction for OS process operations. pub trait OSProcess { + /// The process ID type for the OS. type ProcessIDType: Display + Eq + Copy; + /// The thread ID type for the OS. type ThreadIDType: Display + Eq + Copy; /// Get the memory maps for the process. The returned string is a multi-line string. From 70cba513c1b2df036373999c0bb77080b229a104 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 13 Feb 2026 02:33:23 +0000 Subject: [PATCH 26/27] Extract some methods from linux/android to linux_common --- src/util/metadata/side_metadata/helpers.rs | 1 - .../os/imp/unix_like/linux_like/android.rs | 29 +------------- src/util/os/imp/unix_like/linux_like/linux.rs | 29 +------------- .../imp/unix_like/linux_like/linux_common.rs | 38 +++++++++++++++++++ 4 files changed, 42 insertions(+), 55 deletions(-) diff --git a/src/util/metadata/side_metadata/helpers.rs b/src/util/metadata/side_metadata/helpers.rs index 859e37ddcc..bc1403003f 100644 --- a/src/util/metadata/side_metadata/helpers.rs +++ b/src/util/metadata/side_metadata/helpers.rs @@ -93,7 +93,6 @@ pub(super) fn align_metadata_address( /// Unmaps the specified metadata range, or panics. #[cfg(test)] pub(crate) fn ensure_munmap_metadata(start: Address, size: usize) { - use crate::util::os::*; trace!("ensure_munmap_metadata({}, 0x{:x})", start, size); assert!(OS::munmap(start, size).is_ok()) diff --git a/src/util/os/imp/unix_like/linux_like/android.rs b/src/util/os/imp/unix_like/linux_like/android.rs index 31343bee59..4fca815705 100644 --- a/src/util/os/imp/unix_like/linux_like/android.rs +++ b/src/util/os/imp/unix_like/linux_like/android.rs @@ -15,17 +15,7 @@ impl OSMemory for Android { strategy: MmapStrategy, annotation: &MmapAnnotation<'_>, ) -> Result
{ - let addr = unix_common::mmap(start, size, strategy)?; - - if !cfg!(feature = "no_mmap_annotation") { - linux_common::set_vma_name(addr, size, annotation); - } - - linux_common::set_hugepage(addr, size, strategy.huge_page)?; - - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - - Ok(addr) + linux_common::dzmmap(start, size, strategy, annotation) } fn munmap(start: Address, size: usize) -> Result<()> { @@ -41,22 +31,7 @@ impl OSMemory for Android { } fn panic_if_unmapped(start: Address, size: usize) { - let strategy = MmapStrategy { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - replace: false, - reserve: true, - }; - match unix_common::mmap(start, size, strategy) { - Ok(_) => panic!("{} of size {} is not mapped", start, size), - Err(e) => { - assert!( - e.kind() == std::io::ErrorKind::AlreadyExists, - "Failed to check mapped: {:?}", - e - ); - } - } + linux_common::panic_if_unmapped(start, size) } } diff --git a/src/util/os/imp/unix_like/linux_like/linux.rs b/src/util/os/imp/unix_like/linux_like/linux.rs index e18e8fead9..a75a3d3ee4 100644 --- a/src/util/os/imp/unix_like/linux_like/linux.rs +++ b/src/util/os/imp/unix_like/linux_like/linux.rs @@ -15,17 +15,7 @@ impl OSMemory for Linux { strategy: MmapStrategy, annotation: &MmapAnnotation<'_>, ) -> Result
{ - let addr = unix_common::mmap(start, size, strategy)?; - - if !cfg!(feature = "no_mmap_annotation") { - linux_common::set_vma_name(addr, size, annotation); - } - - linux_common::set_hugepage(addr, size, strategy.huge_page)?; - - // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) - - Ok(addr) + linux_common::dzmmap(start, size, strategy, annotation) } fn munmap(start: Address, size: usize) -> Result<()> { @@ -41,22 +31,7 @@ impl OSMemory for Linux { } fn panic_if_unmapped(start: Address, size: usize) { - let strategy = MmapStrategy { - huge_page: HugePageSupport::No, - prot: MmapProtection::ReadWrite, - replace: false, - reserve: true, - }; - match unix_common::mmap(start, size, strategy) { - Ok(_) => panic!("{} of size {} is not mapped", start, size), - Err(e) => { - assert!( - e.kind() == std::io::ErrorKind::AlreadyExists, - "Failed to check mapped: {:?}", - e - ); - } - } + linux_common::panic_if_unmapped(start, size) } } diff --git a/src/util/os/imp/unix_like/linux_like/linux_common.rs b/src/util/os/imp/unix_like/linux_like/linux_common.rs index 2ca3044095..d9c013bb65 100644 --- a/src/util/os/imp/unix_like/linux_like/linux_common.rs +++ b/src/util/os/imp/unix_like/linux_like/linux_common.rs @@ -100,3 +100,41 @@ pub fn bind_current_thread_to_cpuset(cpuset: &[CoreId]) { sched_setaffinity(0, std::mem::size_of::(), &cs); } } + +pub fn dzmmap( + start: Address, + size: usize, + strategy: MmapStrategy, + annotation: &MmapAnnotation<'_>, +) -> Result
{ + let addr = unix_common::mmap(start, size, strategy)?; + + if !cfg!(feature = "no_mmap_annotation") { + set_vma_name(addr, size, annotation); + } + + set_hugepage(addr, size, strategy.huge_page)?; + + // We do not need to explicitly zero for Linux (memory is guaranteed to be zeroed) + + Ok(addr) +} + +pub fn panic_if_unmapped(start: Address, size: usize) { + let strategy = MmapStrategy { + huge_page: HugePageSupport::No, + prot: MmapProtection::ReadWrite, + replace: false, + reserve: true, + }; + match unix_common::mmap(start, size, strategy) { + Ok(_) => panic!("{} of size {} is not mapped", start, size), + Err(e) => { + assert!( + e.kind() == std::io::ErrorKind::AlreadyExists, + "Failed to check mapped: {:?}", + e + ); + } + } +} From 8b1bf8aa60567ca829037f4368df39fcaf834829 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 13 Feb 2026 16:08:09 +1300 Subject: [PATCH 27/27] Add back comments for align_alloc --- src/util/malloc/malloc_ms_util.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/malloc/malloc_ms_util.rs b/src/util/malloc/malloc_ms_util.rs index 6ebe939d4d..5d27cf79cb 100644 --- a/src/util/malloc/malloc_ms_util.rs +++ b/src/util/malloc/malloc_ms_util.rs @@ -3,6 +3,7 @@ use crate::util::malloc::library::*; use crate::util::Address; use crate::vm::VMBinding; +/// Allocate with alignment. This also guarantees the memory is zero initialized. pub fn align_alloc(size: usize, align: usize) -> Address { let mut ptr = std::ptr::null_mut::(); let ptr_ptr = std::ptr::addr_of_mut!(ptr);