diff --git a/litebox/src/litebox.rs b/litebox/src/litebox.rs index 6007e891af..9157db8e03 100644 --- a/litebox/src/litebox.rs +++ b/litebox/src/litebox.rs @@ -6,6 +6,7 @@ use alloc::sync::Arc; use litebox_broker_local::BrokerLocal; +use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_transport::channel::LocalCallChannel; use litebox_platform::time::TimeProvider; @@ -14,6 +15,7 @@ use crate::{ broker, fd::Descriptors, sync::{RawSyncPrimitivesProvider, RwLock}, + thread::Thread, }; /// A full LiteBox system. @@ -48,16 +50,49 @@ impl LiteBox { platform: &'static Platform, broker_local: BrokerLocal, ) -> Self + where + Platform: TimeProvider, + Channel: LocalCallChannel + Send + Sync + 'static, + { + Self::new_with_broker_local_inner(platform, broker_local).0 + } + + fn new_with_broker_local_inner( + platform: &'static Platform, + broker_local: BrokerLocal, + ) -> (Self, Arc) where Platform: TimeProvider, Channel: LocalCallChannel + Send + Sync + 'static, { let broker_pollables = Arc::new(broker::BrokerPollableRegistry::new()); - let broker_control = Arc::new(broker::BrokerLocalControl::::new( - broker_local, - Arc::clone(&broker_pollables), - )); - Self::new_inner(platform, Some(broker_control), broker_pollables) + let broker_control: Arc = + Arc::new(broker::BrokerLocalControl::::new( + broker_local, + Arc::clone(&broker_pollables), + )); + let litebox = Self::new_inner( + platform, + Some(Arc::clone(&broker_control)), + broker_pollables, + ); + (litebox, broker_control) + } + + /// Creates a broker-backed process and its negotiated initial thread. + pub fn new_process_with_broker_local( + platform: &'static Platform, + broker_local: BrokerLocal, + ) -> (Self, ProcessId, Thread) + where + Platform: TimeProvider, + Channel: LocalCallChannel + Send + Sync + 'static, + { + let process_id = broker_local.process_id(); + let initial_thread_id = broker_local.initial_thread_id(); + let (litebox, broker) = Self::new_with_broker_local_inner(platform, broker_local); + let initial_thread = Thread::from_broker(initial_thread_id, broker); + (litebox, process_id, initial_thread) } fn new_inner( diff --git a/litebox/src/thread.rs b/litebox/src/thread.rs index 4610415599..bcd35d1790 100644 --- a/litebox/src/thread.rs +++ b/litebox/src/thread.rs @@ -50,6 +50,10 @@ pub struct Thread { } impl Thread { + pub(crate) fn from_broker(id: ThreadId, broker: Arc) -> Self { + Self { id, broker } + } + /// Returns the assigned thread ID. #[must_use] pub const fn id(&self) -> u32 { diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 99389bdde4..8d6b7342d3 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -52,6 +52,7 @@ pub use policy::{ use process::ObjectReference; pub use process::{ AssociationCancellation, BrokerProcess, BrokerThread, CallerCredential, ObjectRights, + ProcessLifecycleSink, ProcessShutdown, }; use random::RandomProvider; use socket::{BrokerSocketPorts, SocketProvider}; @@ -213,10 +214,17 @@ pub struct BrokerCore { pub(crate) socket_provider: Arc, pub(crate) fs: Arc, pub(crate) socket_ports: BrokerSocketPorts, + pub(crate) process_lifecycle_sink: Arc, } static BROKER_CORE_CREATED: AtomicBool = AtomicBool::new(false); +struct NoopProcessLifecycleSink; + +impl ProcessLifecycleSink for NoopProcessLifecycleSink { + fn changed(&self) {} +} + impl BrokerCore { /// Creates the broker core with broker-wide service providers. pub fn new( @@ -266,9 +274,25 @@ impl BrokerCore { socket_provider, fs, socket_ports: BrokerSocketPorts::default(), + process_lifecycle_sink: Arc::new(NoopProcessLifecycleSink), }) } + /// Returns a broker handle that publishes process lifecycle changes to `sink`. + #[must_use] + pub fn with_process_lifecycle_sink(&self, sink: Arc) -> Self { + Self { + process_lifecycle_sink: sink, + ..self.clone() + } + } + + /// Returns whether any broker process remains registered. + #[must_use] + pub fn has_processes(&self) -> bool { + !self.processes.read().is_empty() + } + /// Returns the configured authority-state limits. #[must_use] pub const fn limits(&self) -> BrokerCoreLimits { @@ -301,7 +325,7 @@ impl BrokerCore { Ok((first, second)) } - /// Allocates and registers one authenticated broker process. + /// Allocates one authenticated process awaiting association activation. /// /// # Panics /// @@ -310,6 +334,7 @@ impl BrokerCore { pub fn create_process( &self, caller_credential: CallerCredential, + parent_id: Option, ) -> Result> { let mut processes = self.processes.write(); if processes.len() >= self.limits.max_processes { @@ -323,7 +348,7 @@ impl BrokerCore { let process = Arc::new(BrokerProcess::new( self.clone(), id, - None, + parent_id, caller_credential, )); assert!( diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index bd9c3d3be4..ad7169bdcc 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -use alloc::{sync::Arc, vec::Vec}; +use alloc::{ + sync::{Arc, Weak}, + vec::Vec, +}; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use crate::event::EventObject; @@ -14,6 +17,15 @@ use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::{ObjectHandle, ProcessId, ThreadId}; use spin::{Mutex, rwlock::RwLock}; +/// Platform-provided notification destination for broker-process lifecycle changes. +pub trait ProcessLifecycleSink: Send + Sync { + /// Wakes waiters after authoritative process state changes. + fn changed(&self); +} + +/// Host runner shutdown action installed into a broker process. +pub type ProcessShutdown = Arc; + /// Caller identity information supplied by the broker entry layer. /// /// The first userland proof of concept does not authenticate Unix-socket peers, @@ -107,9 +119,8 @@ pub struct BrokerProcess { pub(crate) core: BrokerCore, /// Assigned process ID and internal authority. pub(crate) id: ProcessId, - cleaned_up: bool, - /// Authoritative parent process ID, absent for a root process. parent_id: Option, + state: Mutex, /// Broker-entry-authenticated caller credential for this process. pub(crate) caller_credential: CallerCredential, /// Handles of the live object references owned by this process. @@ -124,6 +135,55 @@ pub struct BrokerProcess { pub(crate) cancellation: AssociationCancellation, } +struct BrokerProcessState { + startup: ProcessStartupState, + retirement: ProcessRetirement, + shutdown_request: ProcessShutdownRequest, + shutdown: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProcessStartupState { + Starting, + Running, + Failed(BrokerError), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProcessRetirement { + Active { abnormal: bool }, + Retired { release_ids: bool }, + Cleaned, +} + +impl ProcessRetirement { + fn mark_abnormal(&mut self) { + match self { + Self::Active { abnormal } => *abnormal = true, + Self::Retired { release_ids } => *release_ids = false, + Self::Cleaned => {} + } + } + + fn retire(&mut self, release_ids: bool) { + let release_ids = match *self { + Self::Active { abnormal } => release_ids && !abnormal, + Self::Retired { + release_ids: current, + } => current && release_ids, + Self::Cleaned => return, + }; + *self = Self::Retired { release_ids }; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProcessShutdownRequest { + None, + Expected, + Unexpected, +} + impl BrokerProcess { /// Creates authenticated broker process state. pub(crate) fn new( @@ -135,8 +195,13 @@ impl BrokerProcess { Self { core, id, - cleaned_up: false, parent_id, + state: Mutex::new(BrokerProcessState { + startup: ProcessStartupState::Starting, + retirement: ProcessRetirement::Active { abnormal: false }, + shutdown_request: ProcessShutdownRequest::None, + shutdown: None, + }), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -155,10 +220,161 @@ impl BrokerProcess { self.id } - /// Returns the current parent process ID, if any. + /// Returns the credential authenticated for this process association. + #[must_use] + pub const fn caller_credential(&self) -> CallerCredential { + self.caller_credential + } + + /// Returns whether this process completed broker startup. + #[must_use] + pub fn is_running(&self) -> bool { + let state = self.state.lock(); + matches!(state.startup, ProcessStartupState::Running) + && matches!(state.retirement, ProcessRetirement::Active { .. }) + } + + /// Returns the completed startup outcome, or `None` while startup is pending. + pub fn startup_result(&self) -> Option> { + match self.state.lock().startup { + ProcessStartupState::Starting => None, + ProcessStartupState::Running => Some(Ok(())), + ProcessStartupState::Failed(error) => Some(Err(error)), + } + } + + /// Completes startup after the process association becomes active. + pub fn complete_start(&self) -> Result<()> { + { + let mut state = self.state.lock(); + if !matches!(state.retirement, ProcessRetirement::Active { .. }) { + return Err(BrokerError::PeerClosed); + } + match state.startup { + ProcessStartupState::Starting => {} + ProcessStartupState::Running => return Err(BrokerError::Internal), + ProcessStartupState::Failed(error) => return Err(error), + } + state.startup = ProcessStartupState::Running; + } + self.core.process_lifecycle_sink.changed(); + Ok(()) + } + + /// Installs the host runner termination action. + pub fn install_shutdown(&self, shutdown: ProcessShutdown) { + let shutdown = { + let mut state = self.state.lock(); + state.shutdown = Some(Arc::clone(&shutdown)); + (state.shutdown_request != ProcessShutdownRequest::None).then_some(shutdown) + }; + if let Some(shutdown) = shutdown { + shutdown(); + } + } + + /// Fails pending startup and returns the authoritative startup outcome. + pub fn fail_start( + &self, + error: BrokerError, + abnormal: bool, + expected_shutdown: bool, + ) -> Result<()> { + let shutdown = { + let mut state = self.state.lock(); + match state.startup { + ProcessStartupState::Starting => {} + ProcessStartupState::Running => return Ok(()), + ProcessStartupState::Failed(error) => return Err(error), + } + if abnormal { + state.retirement.mark_abnormal(); + } + state.startup = ProcessStartupState::Failed(error); + if state.shutdown_request == ProcessShutdownRequest::None { + state.shutdown_request = if expected_shutdown { + ProcessShutdownRequest::Expected + } else { + ProcessShutdownRequest::Unexpected + }; + } + state.shutdown.clone() + }; + self.core.process_lifecycle_sink.changed(); + if let Some(shutdown) = shutdown { + shutdown(); + } + Err(error) + } + + /// Returns whether the first runner shutdown request was expected. #[must_use] - pub const fn parent_id(&self) -> Option { - self.parent_id + pub fn shutdown_was_expected(&self) -> bool { + self.state.lock().shutdown_request == ProcessShutdownRequest::Expected + } + + /// Marks process retirement as abnormal. + pub fn mark_abnormal(&self) { + self.state.lock().retirement.mark_abnormal(); + } + + /// Records final retirement disposition without releasing resources early. + pub fn retire(&self, release_ids: bool) { + { + let mut state = self.state.lock(); + state.retirement.retire(release_ids); + state.shutdown = None; + } + } + + /// Fails every child process that is still awaiting association activation. + pub fn fail_starting_children(&self) { + let processes = { + let processes = self.core.processes.read(); + processes + .values() + .filter_map(Weak::upgrade) + .collect::>() + }; + for child in processes { + if child.parent_id == Some(self.id) { + let _ = child.fail_start(BrokerError::PeerClosed, false, true); + } + } + } + + /// Duplicates object references into another process. + /// + /// Each duplicate retains its source reference's rights. Returned handles + /// follow the requested source order. If duplication fails, references + /// already created by this call are removed from the target before the + /// error is returned. + pub fn duplicate_object_references_to( + &self, + handles: &[ObjectHandle], + target: &BrokerProcess, + ) -> Result> { + let mut duplicates = Vec::new(); + duplicates + .try_reserve_exact(handles.len()) + .map_err(|_| BrokerError::OutOfMemory)?; + for handle in handles { + let duplicate = self + .object_reference_rights(*handle) + .and_then(|rights| self.duplicate_object_reference_to(*handle, target, rights)); + match duplicate { + Ok(duplicate) => duplicates.push(duplicate), + Err(error) => { + for duplicate in duplicates.drain(..).rev() { + if target.close_object_reference(duplicate).is_err() { + return Err(BrokerError::Internal); + } + } + return Err(error); + } + } + } + Ok(duplicates) } /// Creates a broker thread belonging to this process. @@ -201,11 +417,11 @@ impl BrokerProcess { /// Records broker thread exit after its local task teardown completes. pub fn exit_thread(&self, thread_id: ThreadId) -> Result<()> { - let thread = self - .threads - .lock() + let mut threads = self.threads.lock(); + let thread = threads .remove(&thread_id) .ok_or(BrokerError::UnknownObject)?; + drop(threads); self.core .active_thread_count .fetch_sub(1, Ordering::Relaxed); @@ -219,19 +435,10 @@ impl BrokerProcess { self.cancellation.cancel(); } - /// Completes non-unwinding process teardown and releases its IDs. - /// - /// Dropping a process without calling this method performs authority - /// cleanup but leaves its numeric IDs occupied so they cannot be reused - /// after an unwind. - /// # Panics - /// - /// Panics if another owner still holds this process. - pub fn finish(self: Arc) { - let Ok(mut process) = Arc::try_unwrap(self) else { - panic!("all broker process owners must be released before teardown"); - }; - process.cleanup(true); + /// Returns whether association teardown requested cancellation. + #[must_use] + pub fn is_cancellation_requested(&self) -> bool { + self.cancellation.is_cancelled() } pub(crate) fn create_object_reference(&self, object: ObjectEntry) -> Result { @@ -345,6 +552,15 @@ impl BrokerProcess { target.create_object_reference_with_rights(object, rights) } + fn object_reference_rights(&self, handle: ObjectHandle) -> Result { + let references = self.core.references.read(); + let reference = references.get(&handle).ok_or(BrokerError::UnknownObject)?; + if reference.owner != self.id { + return Err(BrokerError::UnknownObject); + } + Ok(reference.rights) + } + pub(crate) fn create_object_reference_pair( &self, first: ObjectEntry, @@ -611,11 +827,24 @@ impl BrokerProcess { Ok(reference) } - fn cleanup(&mut self, release_ids: bool) -> bool { - if self.cleaned_up { - return false; - } - self.cleaned_up = true; + /// Cleans up this process, optionally releasing its numeric IDs for reuse. + /// + /// Set `release_ids` only after fully accounted teardown. Final process + /// drop otherwise retains IDs after an unwind or uncertain retirement. + /// Calling this method more than once is harmless. + pub fn cleanup(&self, release_ids: bool) { + let release_ids = { + let mut state = self.state.lock(); + let release_ids = match state.retirement { + ProcessRetirement::Active { abnormal } => release_ids && !abnormal, + ProcessRetirement::Retired { + release_ids: decided, + } => release_ids && decided, + ProcessRetirement::Cleaned => return, + }; + state.retirement = ProcessRetirement::Cleaned; + release_ids + }; let mut invariant_fault = self.references.lock().pending_handles != 0; loop { @@ -685,7 +914,7 @@ impl BrokerProcess { } ids.release(self.id.0); } - invariant_fault + self.core.process_lifecycle_sink.changed(); } } @@ -770,7 +999,14 @@ fn release_pending_reference( impl Drop for BrokerProcess { fn drop(&mut self) { - let _ = self.cleanup(false); + let release_ids = { + let state = self.state.lock(); + matches!( + state.retirement, + ProcessRetirement::Retired { release_ids: true } + ) + }; + self.cleanup(release_ids); } } @@ -778,7 +1014,7 @@ impl Drop for BrokerProcess { mod tests { use core::sync::atomic::{AtomicUsize, Ordering}; - use super::{ProcessReferences, release_pending_reference}; + use super::{ProcessLifecycleSink, ProcessReferences, release_pending_reference}; use crate::test_platform::TestPlatform; use crate::test_support::{TestBrokerCoreBuilder, TestStdioProvider}; use crate::{ @@ -800,6 +1036,17 @@ mod tests { const TEST_MAX_PIPE_CAPACITY_PER_PROCESS: usize = 4; const ROOT: FileUser = FileUser { user: 0, group: 0 }; + #[derive(Default)] + struct TestProcessLifecycleSink { + changes: AtomicUsize, + } + + impl ProcessLifecycleSink for TestProcessLifecycleSink { + fn changed(&self) { + self.changes.fetch_add(1, Ordering::Relaxed); + } + } + #[test] fn process_and_thread_ids_share_one_numeric_namespace() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( @@ -808,18 +1055,111 @@ mod tests { .build() .unwrap(); let first = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let thread = first.create_thread().unwrap(); let second = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_eq!(first.id().0, 1); - assert_eq!(first.parent_id(), None); assert_eq!(thread.0, 2); assert_eq!(second.id().0, 3); - assert_eq!(second.parent_id(), None); + } + + #[test] + fn parent_teardown_fails_a_starting_child() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated, None) + .unwrap(); + parent.complete_start().unwrap(); + let child = broker + .create_process(parent.caller_credential(), Some(parent.id())) + .unwrap(); + let shutdowns = Arc::new(AtomicUsize::new(0)); + let shutdown_count = Arc::clone(&shutdowns); + child.install_shutdown(Arc::new(move || { + shutdown_count.fetch_add(1, Ordering::Relaxed); + })); + + parent.fail_starting_children(); + + assert_eq!(child.parent_id, Some(parent.id())); + assert_eq!(child.startup_result(), Some(Err(BrokerError::PeerClosed))); + assert_eq!(child.complete_start(), Err(BrokerError::PeerClosed)); + assert_eq!(shutdowns.load(Ordering::Relaxed), 1); + } + + #[test] + fn retirement_waits_for_the_final_process_owner() { + let sink = Arc::new(TestProcessLifecycleSink::default()); + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .with_limits(BrokerCoreLimits::DEFAULT.with_process_limit(1)) + .build() + .unwrap() + .with_process_lifecycle_sink(sink.clone()); + let process = broker + .create_process(CallerCredential::Unauthenticated, None) + .unwrap(); + let retained = Arc::clone(&process); + process.complete_start().unwrap(); + assert_eq!( + process.fail_start(BrokerError::PeerClosed, true, true), + Ok(()) + ); + assert!(!process.shutdown_was_expected()); + process.retire(true); + assert_eq!(process.complete_start(), Err(BrokerError::PeerClosed)); + drop(process); + + assert!(matches!( + broker.create_process(CallerCredential::Unauthenticated, None), + Err(BrokerError::ResourceExhausted) + )); + assert_eq!(sink.changes.load(Ordering::Relaxed), 1); + + drop(retained); + + assert_eq!(sink.changes.load(Ordering::Relaxed), 2); + assert!( + broker + .create_process(CallerCredential::Unauthenticated, None) + .is_ok() + ); + } + + #[test] + fn failed_reference_inheritance_rolls_back_target_references() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let source = broker + .create_process(CallerCredential::Unauthenticated, None) + .unwrap(); + let target = broker + .create_process(source.caller_credential(), None) + .unwrap(); + let source_handle = crate::event::create(&source, 1).unwrap(); + + assert_eq!( + source + .duplicate_object_references_to(&[source_handle, ObjectHandle(u64::MAX)], &target,), + Err(BrokerError::UnknownObject) + ); + assert!(target.references.lock().handles.is_empty()); + assert_eq!( + source.check_readiness(source_handle).unwrap(), + ReadinessFlags::READ | ReadinessFlags::WRITE + ); } #[test] @@ -832,7 +1172,7 @@ mod tests { broker.ids = alloc::sync::Arc::new(spin::Mutex::new(crate::id::IdAllocator::new(2).unwrap())); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first = process.create_thread().unwrap(); @@ -849,10 +1189,10 @@ mod tests { .build() .unwrap(); let first = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let thread = first.create_thread().unwrap(); @@ -869,13 +1209,13 @@ mod tests { .build() .unwrap(); let first = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let third = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first_thread = first.create_thread().unwrap(); let second_thread = second.create_thread().unwrap(); @@ -897,18 +1237,18 @@ mod tests { .build() .unwrap(); let first = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert!(matches!( - broker.create_process(CallerCredential::Unauthenticated), + broker.create_process(CallerCredential::Unauthenticated, None), Err(BrokerError::ResourceExhausted) )); - first.finish(); + first.cleanup(true); assert!( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .is_ok() ); } @@ -923,7 +1263,7 @@ mod tests { broker.ids = alloc::sync::Arc::new(spin::Mutex::new(crate::id::IdAllocator::new(2).unwrap())); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let process_id = process.id(); let thread_id = process.create_thread().unwrap(); @@ -937,12 +1277,12 @@ mod tests { assert!(Arc::ptr_eq(&process, ®istered)); drop(registered); - process.finish(); + process.cleanup(true); assert!(!broker.processes.read().contains_key(&process_id)); assert_eq!(broker.active_thread_count.load(Ordering::Relaxed), 0); let replacement = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_eq!(replacement.id(), process_id); assert_eq!(replacement.create_thread().unwrap(), thread_id); @@ -959,7 +1299,7 @@ mod tests { broker.ids = alloc::sync::Arc::new(spin::Mutex::new(crate::id::IdAllocator::new(4).unwrap())); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let process_id = process.id(); let thread_id = process.create_thread().unwrap(); @@ -968,10 +1308,10 @@ mod tests { assert_eq!(broker.active_thread_count.load(Ordering::Relaxed), 1); let first_replacement = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second_replacement = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_ne!(first_replacement.id(), process_id); assert_ne!(first_replacement.id().0, thread_id.0); @@ -982,7 +1322,7 @@ mod tests { Err(BrokerError::ResourceExhausted) ); assert!(matches!( - broker.create_process(CallerCredential::Unauthenticated), + broker.create_process(CallerCredential::Unauthenticated, None), Err(BrokerError::ResourceExhausted) )); } @@ -995,7 +1335,7 @@ mod tests { .build() .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let process_id = process.id(); crate::event::create(&process, 1).unwrap(); @@ -1005,7 +1345,7 @@ mod tests { assert!(broker.references.read().is_empty()); let replacement = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_ne!(replacement.id(), process_id); } @@ -1034,13 +1374,13 @@ mod tests { fn check_supported_references_duplicate_between_processes(broker: &BrokerCore) { let source = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let target = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let denied_target = broker - .create_process(CallerCredential::HostGuaranteed) + .create_process(CallerCredential::HostGuaranteed, None) .unwrap(); let event = crate::event::create(&source, 1).unwrap(); @@ -1097,10 +1437,10 @@ mod tests { fn check_file_reference_lifecycle(broker: &BrokerCore, stdio_provider: &TestStdioProvider) { let source = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let target = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let mode = FileMode::from_bits(0o600).unwrap(); let file = crate::fs::open( @@ -1375,10 +1715,10 @@ mod tests { fn check_event_reference_lifecycle(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let other = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let handle = crate::event::create(&process, 0).unwrap(); let unknown_handle = ObjectHandle(handle.0.checked_add(1).unwrap()); @@ -1429,7 +1769,7 @@ mod tests { fn check_process_drop_releases_references(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first = crate::event::create(&process, 0).unwrap(); let second = crate::event::create(&process, 0).unwrap(); @@ -1449,7 +1789,7 @@ mod tests { fn check_pipe_lifecycle(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_eq!( crate::pipe::create(&process, 5, 2), @@ -1497,7 +1837,7 @@ mod tests { fn check_pipe_reader_closure(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (reader, writer) = crate::pipe::create(&process, 4, 2).unwrap(); assert_eq!(broker.reserved_pipe_capacity.load(Ordering::Relaxed), 4); @@ -1517,7 +1857,7 @@ mod tests { fn check_corrupt_index_fails_without_mutation(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let older = crate::event::create(&process, 0).unwrap(); let newer = crate::event::create(&process, 0).unwrap(); @@ -1540,7 +1880,7 @@ mod tests { fn check_corrupt_index_does_not_break_teardown(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let _older = crate::event::create(&process, 0).unwrap(); let newer = crate::event::create(&process, 0).unwrap(); @@ -1558,10 +1898,10 @@ mod tests { fn check_reference_quota_is_per_process(broker: &BrokerCore) { let greedy = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let neighbor = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let greedy_first = crate::event::create(&greedy, 0).unwrap(); @@ -1576,7 +1916,7 @@ mod tests { assert_eq!(broker.references.read().len(), TEST_MAX_REFERENCES); let latecomer = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_eq!( crate::event::create(&latecomer, 0), @@ -1595,10 +1935,10 @@ mod tests { fn check_pending_references_count_toward_process_quota(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let neighbor = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first = process @@ -1622,10 +1962,10 @@ mod tests { fn check_pipe_capacity_quota_is_per_process(broker: &BrokerCore) { let greedy = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let neighbor = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (greedy_reader, greedy_writer) = @@ -1647,7 +1987,7 @@ mod tests { ); let latecomer = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); assert_eq!( crate::pipe::create(&latecomer, 1, 1), @@ -1667,7 +2007,7 @@ mod tests { fn check_pipe_capacity_outlives_process_for_in_flight_object(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (reader, _writer) = crate::pipe::create(&process, TEST_MAX_PIPE_CAPACITY_PER_PROCESS as u64, 2).unwrap(); @@ -1696,7 +2036,7 @@ mod tests { fn check_pair_handle_exhaustion(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); { let mut next_reference_handle = broker.next_reference_handle.write(); diff --git a/litebox_broker_core/src/socket/tests.rs b/litebox_broker_core/src/socket/tests.rs index 42aa850176..b45de414de 100644 --- a/litebox_broker_core/src/socket/tests.rs +++ b/litebox_broker_core/src/socket/tests.rs @@ -72,7 +72,7 @@ fn gateway_destinations_require_external_policy() { let provider = Arc::new(TestSocketProvider::default()); let broker = test_broker(Arc::clone(&provider) as Arc); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let tcp = create( &process, @@ -113,7 +113,7 @@ fn gateway_destinations_reach_the_platform_untranslated() { .unwrap(); let broker = test_broker_with_policy(Arc::clone(&provider) as Arc, &policy); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let tcp = create( &process, @@ -303,7 +303,7 @@ fn rejected_external_route_preserves_an_exact_loopback_socket() { .unwrap(); let broker = test_broker_with_policy(Arc::clone(&provider) as Arc, &policy); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let socket = create( &process, @@ -351,7 +351,7 @@ fn rejected_udp_external_routes_preserve_an_exact_loopback_socket() { .unwrap(); let broker = test_broker_with_policy(Arc::clone(&provider) as Arc, &policy); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let socket = create( &process, @@ -1086,7 +1086,7 @@ fn zero_port_connect_fails_before_platform_dispatch() { let provider = Arc::new(TestSocketProvider::default()); let broker = test_broker(Arc::clone(&provider) as Arc); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let socket = create( &process, @@ -1116,10 +1116,10 @@ fn accepted_guest_source_lease_is_retained() { let provider = Arc::new(TestSocketProvider::default()); let broker = test_broker(Arc::clone(&provider) as Arc); let listener_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let listener = create( &listener_session, @@ -1212,10 +1212,10 @@ fn check_queued_guest_source_lease_release(stop_listener: bool) { let provider = Arc::new(TestSocketProvider::default()); let broker = test_broker(Arc::clone(&provider) as Arc); let listener_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let listener_address = SocketAddrV4::new(Ipv4Addr::LOCALHOST, 44010); let listener = create( @@ -1340,6 +1340,7 @@ fn test_broker_with_policy( socket_provider, fs: Arc::new(crate::fs::UnsupportedFileService), socket_ports: BrokerSocketPorts::default(), + process_lifecycle_sink: Arc::new(crate::NoopProcessLifecycleSink), } } @@ -1372,7 +1373,7 @@ fn check_platform_socket_retires_before_last_arc_drop( provider: &TestSocketProvider, ) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let retired_before = provider.state.retired_sockets.load(Ordering::Relaxed); let dropped_before = provider.state.dropped_sockets.load(Ordering::Relaxed); @@ -1413,7 +1414,7 @@ fn check_platform_socket_retires_before_last_arc_drop( fn check_failed_create_rolls_back(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); provider.fail_next_create(); let readiness = Arc::new(TestReadinessSink::default()); @@ -1442,7 +1443,7 @@ fn failed_accept_rolls_back_readiness_and_quota() { let provider = Arc::new(TestSocketProvider::default()); let broker = test_broker(Arc::clone(&provider) as Arc); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let listener = create( &process, @@ -1518,10 +1519,10 @@ fn invalid_accepted_metadata_retires_socket_readiness_and_quota() { ), ] { let listener_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let listener = create( &listener_session, @@ -1600,7 +1601,7 @@ fn invalid_accepted_metadata_retires_socket_readiness_and_quota() { fn check_in_flight_connect_preserves_local_address(broker: &BrokerCore) { let process = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let (started_tx, started_rx) = mpsc::channel(); @@ -1639,10 +1640,10 @@ fn check_in_flight_connect_preserves_local_address(broker: &BrokerCore) { fn check_invalid_bind_response_retires_socket(broker: &BrokerCore, provider: &TestSocketProvider) { let first_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let readiness = Arc::new(TestReadinessSink::default()); let retired_before = provider.state.retired_sockets.load(Ordering::Relaxed); @@ -1711,7 +1712,7 @@ fn check_invalid_bind_response_retires_socket(broker: &BrokerCore, provider: &Te let blocking_session = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let invalid = create( @@ -1791,7 +1792,7 @@ fn check_automatic_bind_retains_reservation_during_retirement( }; let process = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let handle = create(&process, request, Arc::new(TestReadinessSink::default())).unwrap(); @@ -1840,7 +1841,7 @@ fn check_automatic_bind_retains_reservation_during_retirement( fn check_duplicate_port_binding_retires_socket(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let handle = create( &process, @@ -1902,10 +1903,10 @@ fn check_duplicate_port_binding_retires_socket(broker: &BrokerCore, provider: &T fn check_socket_operations_and_policy(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let other = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let readiness = Arc::new(TestReadinessSink::default()); let handle = create(&process, create_request(), readiness.clone()).unwrap(); @@ -2112,7 +2113,7 @@ fn check_socket_operations_and_policy(broker: &BrokerCore, provider: &TestSocket fn check_tcp_option_state_is_per_socket(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first = create( &process, @@ -2154,7 +2155,7 @@ fn check_private_tcp_connect_uses_private_source_for_wildcard_binding( provider: &TestSocketProvider, ) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let handle = create( &process, @@ -2201,7 +2202,7 @@ fn check_udp_socket_operations(broker: &BrokerCore, provider: &TestSocketProvide .unwrap() .len(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let readiness = Arc::new(TestReadinessSink::default()); let request = CreateSocketRequest { @@ -2479,7 +2480,7 @@ fn check_udp_socket_operations(broker: &BrokerCore, provider: &TestSocketProvide fn check_udp_status_validates_local_address(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let unbound = create( &process, @@ -2738,7 +2739,7 @@ fn check_udp_status_validates_local_address(broker: &BrokerCore, provider: &Test fn check_server_socket_operations(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let readiness = Arc::new(TestReadinessSink::default()); let listener = create(&process, create_request(), readiness.clone()).unwrap(); @@ -2812,7 +2813,7 @@ fn check_server_socket_operations(broker: &BrokerCore, provider: &TestSocketProv ); let competing_session = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let competitor = create( &competing_session, @@ -2854,7 +2855,7 @@ fn check_concurrent_udp_status_does_not_regress_connection( ) { let process = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let handle = create( @@ -2974,7 +2975,7 @@ fn check_failed_listener_shutdown_preserves_state( provider: &TestSocketProvider, ) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let handle = create( &process, @@ -3004,7 +3005,7 @@ fn check_listener_shutdown_does_not_race_listen( ) { let process = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let handle = create( @@ -3043,13 +3044,13 @@ fn check_listener_shutdown_does_not_race_listen( fn check_socket_quotas(broker: &BrokerCore) { let first = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let third = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first_handle = create( &first, @@ -3118,7 +3119,7 @@ impl ReadinessSink for BlockingReadinessSink { fn check_quota_waits_for_deferred_retirement(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (started_tx, started_rx) = mpsc::channel(); let (release_tx, release_rx) = mpsc::channel(); @@ -3154,7 +3155,7 @@ fn check_quota_waits_for_deferred_retirement(broker: &BrokerCore, provider: &Tes fn check_connect_errors_classify_peer_state(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let retryable = create( &process, @@ -3324,7 +3325,7 @@ fn check_concurrent_status_preserves_terminal_state( ) { let process = Arc::new( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), ); let handle = create( @@ -3503,7 +3504,7 @@ fn check_concurrent_status_preserves_terminal_state( fn check_stream_status_validates_local_address(broker: &BrokerCore, provider: &TestSocketProvider) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let valid = create( &process, @@ -3760,7 +3761,7 @@ fn check_terminal_stream_status_preserves_refined_address( provider: &TestSocketProvider, ) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let handle = create( &process, diff --git a/litebox_broker_host/src/error.rs b/litebox_broker_host/src/error.rs index c52813d084..af88432e6f 100644 --- a/litebox_broker_host/src/error.rs +++ b/litebox_broker_host/src/error.rs @@ -13,6 +13,8 @@ pub enum BrokerHostError { Channel(#[source] E), #[error("broker setup failed: {0}")] Broker(#[source] ErrorCode), + #[error("broker association had already failed")] + AssociationFailed, #[error("broker association shared-buffer layout does not match the protocol layout")] SharedBufferLayoutMismatch, } diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index b57ad0415c..b0e36bc3cb 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -24,7 +24,7 @@ extern crate std; use alloc::{sync::Arc, vec::Vec}; use litebox_broker_core::readiness::ReadinessSink; -use litebox_broker_core::{BrokerCore, BrokerProcess, CallerCredential}; +use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess, CallerCredential}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::event::{AddEventResponse, CreateEventResponse}; use litebox_broker_protocol::fs::{ @@ -43,9 +43,14 @@ use litebox_broker_protocol::message::{ use litebox_broker_protocol::pipe::{ CreatePipeResponse, MAX_PIPE_TRANSFER_SIZE, ReadPipeResponse, WritePipeResponse, }; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupData, ProcessStartupDescriptor, +}; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ SHARED_BUFFER_LAYOUT, SHARED_BUFFER_SLOT_COUNT, SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence, + SharedBufferSlotIndex, }; use litebox_broker_protocol::socket::{ AcceptSocketResponse, BindSocketResponse, ConnectSocketResponse, CreateSocketResponse, @@ -57,9 +62,9 @@ use litebox_broker_protocol::stdio::{ IsTerminalStdioRequest, IsTerminalStdioResponse, MAX_STDIO_TRANSFER_SIZE, ReadStdioRequest, ReadStdioResponse, WriteStdioRequest, WriteStdioResponse, }; -use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId}; +use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId, ThreadId}; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; -use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; +use litebox_broker_transport::shared_memory::{SharedBufferError, SharedBufferPool, SharedMemory}; use spin::mutex::SpinMutex; mod error; @@ -70,17 +75,17 @@ pub mod test_support; pub use error::{BrokerHostError, Result}; /// Negotiated active association, or a terminal outcome reached during setup. -pub type ConnectionSetup<'a, Memory> = - core::result::Result, ConnectionTermination>; +pub type ConnectionSetup = + core::result::Result, ConnectionTermination>; /// Active portable broker association. /// /// Deployments may share this value across bounded workers. Each request is /// executed independently, while shared-buffer usage is synchronized and /// released immediately before publishing the response. -pub struct BrokerHostAssociation<'a, Memory: SharedMemory> { +pub struct BrokerHostAssociation { process: Arc, - shared_buffers: &'a SharedBufferPool, + shared_buffers: Arc>, readiness_sink: Arc, state: SpinMutex, } @@ -90,7 +95,33 @@ struct AssociationState { shared_buffer_usage: SharedBufferUsage, } -impl BrokerHostAssociation<'_, Memory> { +impl BrokerHostAssociation { + fn new( + process: Arc, + shared_buffers: Arc>, + readiness_sink: Arc, + ) -> Self { + Self { + process, + shared_buffers, + readiness_sink, + state: SpinMutex::new(AssociationState { + failed: false, + shared_buffer_usage: SharedBufferUsage::new(), + }), + } + } + + /// Marks the process running after deployment-specific association activation. + pub fn activate_process(&self) -> litebox_broker_core::Result<()> { + self.process.complete_start() + } + + /// Records association teardown and fails children still awaiting activation. + pub fn association_ending(&self) { + self.process.fail_starting_children(); + } + /// Requests cancellation of provider operations after the peer disconnects. pub fn request_cancellation(&self) { self.process.request_cancellation(); @@ -98,7 +129,7 @@ impl BrokerHostAssociation<'_, Memory> { /// Completes non-unwinding association teardown and releases its process ID. pub fn finish(self) { - BrokerProcess::finish(self.process); + self.process.retire(true); } /// Executes one active request and emits its response. @@ -110,6 +141,27 @@ impl BrokerHostAssociation<'_, Memory> { &self, request: BrokerRequest, send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, + ) -> Result<(), ChannelError> { + self.execute_request_with( + request, + |_process, _operation, _shared_buffers| None, + send_response, + ) + } + + /// Executes one active request with an optional deployment-specific operation handler. + /// + /// The extension runs after shared-buffer ownership is validated and while + /// any operation-scoped sequence remains reserved for this request. + pub fn execute_request_with( + &self, + request: BrokerRequest, + extension: impl FnOnce( + &BrokerProcess, + &BrokerOperation, + &SharedBufferPool, + ) -> Option>, + send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, ) -> Result<(), ChannelError> { let BrokerRequest { request_id, @@ -120,7 +172,7 @@ impl BrokerHostAssociation<'_, Memory> { { let mut state = self.state.lock(); if state.failed { - return Err(BrokerHostError::Broker(ErrorCode::Internal)); + return Err(BrokerHostError::AssociationFailed); } if let Some(sequence) = buffer_sequence && let Err(error) = state.shared_buffer_usage.begin( @@ -134,12 +186,16 @@ impl BrokerHostAssociation<'_, Memory> { } } - let result = match complete_request(handle_request( - &self.process, - operation, - self.shared_buffers, - &self.readiness_sink, - )) { + let request_result = match extension(&self.process, &operation, &self.shared_buffers) { + Some(result) => result, + None => handle_request( + &self.process, + operation, + &self.shared_buffers, + &self.readiness_sink, + ), + }; + let result = match complete_request(request_result) { Ok(result) => result, Err(error) => { self.state.lock().failed = true; @@ -152,7 +208,8 @@ impl BrokerHostAssociation<'_, Memory> { .shared_buffer_usage .end(request_id, sequence); } - if let Err(error) = send_response(&BrokerResponse { request_id, result }) { + let response = BrokerResponse { request_id, result }; + if let Err(error) = send_response(&response) { self.state.lock().failed = true; return Err(BrokerHostError::Channel(error)); } @@ -160,17 +217,19 @@ impl BrokerHostAssociation<'_, Memory> { } } -/// Authenticates and negotiates one broker control connection. -/// -/// `send_shared_memory` runs after version negotiation and before the active -/// association is returned. -pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( +/// Authenticates and negotiates one broker control connection while allowing +/// a deployment owner to retain the negotiated process before setup completes. +#[allow(clippy::too_many_arguments)] +pub fn setup_connection( core: &BrokerCore, + process: Option<(Arc, ThreadId)>, + startup: Option, setup_channel: &mut SetupChannel, - shared_buffers: &'a SharedBufferPool, + shared_buffers: Arc>, readiness_sink: Arc, + retain_process: impl FnOnce(&Arc) -> bool, send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, -) -> Result, ChannelError> +) -> Result, ChannelError> where SetupChannel: HostSetupChannel, Memory: SharedMemory, @@ -178,6 +237,39 @@ where if shared_buffers.layout() != SHARED_BUFFER_LAYOUT { return Err(BrokerHostError::SharedBufferLayoutMismatch); } + if process.is_none() && startup.is_some() { + return Err(BrokerHostError::Broker(ErrorCode::Internal)); + } + let startup = match startup { + Some(ProcessStartupData { + format, + version, + payload, + inherited_objects, + }) => { + if payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { + return Err(BrokerHostError::Broker(ErrorCode::ResourceExhausted)); + } + let bootstrap_length = u32::try_from(payload.len()) + .map_err(|_| BrokerHostError::Broker(ErrorCode::ResourceExhausted))?; + let buffer = SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], bootstrap_length) + .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; + write_shared_buffer( + &shared_buffers, + buffer, + &payload, + MAX_PROCESS_BOOTSTRAP_SIZE, + ) + .map_err(|error| BrokerHostError::Broker(error.into()))?; + Some(ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }) + } + None => None, + }; let limits = core.limits(); // Sockets are currently the only externally backed objects. Add future // resource limits here so every live registration fits in the @@ -195,6 +287,7 @@ where PeerCredential::Unauthenticated => CallerCredential::Unauthenticated, _ => return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)), }; + let mut process = process; loop { let request = match setup_channel .recv_handshake_request() @@ -224,48 +317,84 @@ where continue; } - let process = match core.create_process(caller_credential) { - Ok(process) => process, - Err(litebox_broker_core::BrokerError::ResourceExhausted) => { - let error = ErrorCode::ResourceExhausted; + let finish_on_setup_error = process.is_none(); + let (process, initial_thread_id) = match process.take() { + Some((process, initial_thread_id)) + if process.caller_credential() == caller_credential => + { + (process, initial_thread_id) + } + Some(_) => { + let error = ErrorCode::PolicyDenied; setup_channel .send_handshake_response(&BrokerHandshakeResponse::Error(error)) .map_err(BrokerHostError::Channel)?; return Ok(Err(ConnectionTermination::Rejected(error))); } - Err(error) => return Err(BrokerHostError::from(error)), + None => match core.create_process(caller_credential, None) { + Ok(process) => match process.create_thread() { + Ok(initial_thread_id) => (process, initial_thread_id), + Err( + error @ (litebox_broker_core::BrokerError::ResourceExhausted + | litebox_broker_core::BrokerError::OutOfMemory), + ) => { + process.retire(true); + let error = ErrorCode::from(error); + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::Error(error)) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::Rejected(error))); + } + Err(error) => { + process.retire(true); + return Err(BrokerHostError::from(error)); + } + }, + Err(litebox_broker_core::BrokerError::ResourceExhausted) => { + let error = ErrorCode::ResourceExhausted; + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::Error(error)) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::Rejected(error))); + } + Err(error) => return Err(BrokerHostError::from(error)), + }, }; let response = BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: process.id(), + initial_thread_id, + startup, }; + let process_retained = retain_process(&process); if let Err(error) = setup_channel.send_handshake_response(&response) { - BrokerProcess::finish(process); + if finish_on_setup_error && !process_retained { + process.retire(true); + } return Err(BrokerHostError::Channel(error)); } if let Err(error) = send_shared_memory(setup_channel) { - BrokerProcess::finish(process); + if finish_on_setup_error && !process_retained { + process.retire(true); + } return Err(BrokerHostError::Channel(error)); } - return Ok(Ok(BrokerHostAssociation { + return Ok(Ok(BrokerHostAssociation::new( process, shared_buffers, readiness_sink, - state: SpinMutex::new(AssociationState { - failed: false, - shared_buffer_usage: SharedBufferUsage::new(), - }), - })); + ))); } } type RequestResult = core::result::Result; +/// Failure classification for broker request handling. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RequestFailure { - /// Send an error response and continue serving the association. +pub enum RequestFailure { + /// Return an error result and keep serving the association. Respond(ErrorCode), - /// Terminate the association without sending a response. + /// Fail the association without publishing a response. Abort(ErrorCode), } @@ -281,6 +410,14 @@ impl From for RequestFailure { } } +impl From for ErrorCode { + fn from(error: RequestFailure) -> Self { + match error { + RequestFailure::Respond(error) | RequestFailure::Abort(error) => error, + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum SharedBufferSlotState { Unused, @@ -404,6 +541,9 @@ fn handle_request( BrokerOperation::File(request) => { handle_file_request(process, request, shared_buffers).map(BrokerResult::File) } + BrokerOperation::StartChildProcess(_) => { + Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) + } } } @@ -630,27 +770,122 @@ fn allocate_zeroed(length: u32) -> RequestResult> { Ok(data) } -fn read_shared_buffer( +/// Reads a validated operation-scoped shared-buffer sequence. +pub fn read_shared_buffer( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, max_length: u32, ) -> RequestResult> { validate_shared_buffer(buffer, max_length)?; let mut data = allocate_zeroed(buffer.length())?; - let mut offset = 0; - for descriptor in buffer - .descriptors(shared_buffers.layout()) - .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))? - { - let end = offset + descriptor.length as usize; - shared_buffers - .read(descriptor.slot_index, &mut data[offset..end]) - .map_err(|_| RequestFailure::Abort(ErrorCode::Internal))?; - offset = end; - } + shared_buffers + .read_sequence(buffer, &mut data) + .map_err(shared_buffer_access_failure)?; Ok(data) } +/// Platform implementation that starts execution for a broker-created process. +/// +/// Once called, the launcher owns final process retirement even when launch +/// fails. Success means process startup reached `Running`. +pub trait ProcessLauncher: Send + Sync { + /// Starts one process and waits for startup to commit or fail. + fn launch( + self: Arc, + process: Arc, + initial_thread_id: ThreadId, + startup: ProcessStartupData, + ) -> core::result::Result<(), BrokerError>; +} + +/// Handles a process operation using the configured platform launcher. +pub fn handle_process_operation( + broker: &BrokerCore, + launcher: &Arc, + parent: &BrokerProcess, + operation: &BrokerOperation, + shared_buffers: &SharedBufferPool, +) -> Option> +where + Memory: SharedMemory, + Launcher: ProcessLauncher + ?Sized, +{ + match operation { + BrokerOperation::StartChildProcess(request) => Some( + read_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) + .and_then(|payload| { + start_child_process( + broker, + Arc::clone(launcher), + parent, + request.format, + request.version, + payload, + request.inherited_objects, + ) + }) + .map(BrokerResult::ProcessStarted), + ), + _ => None, + } +} + +fn start_child_process( + broker: &BrokerCore, + launcher: Arc, + parent: &BrokerProcess, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + payload: Vec, + requested_inherited_objects: InheritedProcessObjects, +) -> RequestResult { + if !parent.is_running() { + return Err(RequestFailure::Abort(ErrorCode::ProtocolState)); + } + let process = broker + .create_process(parent.caller_credential(), Some(parent.id())) + .map_err(RequestFailure::from)?; + let inherited_objects = match parent + .duplicate_object_references_to(requested_inherited_objects.as_slice(), &process) + { + Ok(inherited_objects) => inherited_objects, + Err(error) => { + process.retire(true); + return Err(RequestFailure::from(error)); + } + }; + let initial_thread_id = match process.create_thread() { + Ok(initial_thread_id) => initial_thread_id, + Err(error) => { + process.retire(true); + return Err(RequestFailure::from(error)); + } + }; + let inherited_objects = InheritedProcessObjects::new(&inherited_objects) + .expect("child handle count must match the bounded inheritance request"); + let process_id = process.id(); + if parent.is_cancellation_requested() { + process.retire(true); + return Err(RequestFailure::Respond(ErrorCode::PeerClosed)); + } + launcher + .launch( + process, + initial_thread_id, + ProcessStartupData { + format, + version, + payload, + inherited_objects, + }, + ) + .map_err(RequestFailure::from)?; + Ok(ProcessIdentity { + process_id, + initial_thread_id, + }) +} + fn write_shared_buffer( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, @@ -658,28 +893,16 @@ fn write_shared_buffer( max_length: u32, ) -> RequestResult<()> { validate_shared_buffer(buffer, max_length)?; - if data.len() > buffer.length() as usize { - return Err(RequestFailure::Abort(ErrorCode::Internal)); - } - let mut offset = 0; - for descriptor in buffer - .descriptors(shared_buffers.layout()) - .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))? - { - if offset == data.len() { - break; - } - let length = (data.len() - offset).min(descriptor.length as usize); - let end = offset + length; - shared_buffers - .write(descriptor.slot_index, &data[offset..end]) - .map_err(|_| RequestFailure::Abort(ErrorCode::Internal))?; - offset = end; - } - if offset != data.len() { - return Err(RequestFailure::Abort(ErrorCode::Internal)); + shared_buffers + .write_sequence(buffer, data) + .map_err(shared_buffer_access_failure) +} + +fn shared_buffer_access_failure(error: SharedBufferError) -> RequestFailure { + match error { + SharedBufferError::Layout(_) => RequestFailure::Abort(ErrorCode::MalformedRequest), + _ => RequestFailure::Abort(ErrorCode::Internal), } - Ok(()) } fn read_file_path( @@ -1411,6 +1634,8 @@ mod tests { test_channel_continues_after_recoverable_request_failure(&broker); test_channel_aborts_on_stale_shared_buffer_request(&broker); test_channel_aborts_without_response_on_shared_memory_failure(&broker); + setup_failure_transfers_process_to_the_deployment_owner(&broker); + precreated_root_negotiates_without_startup_data(&broker); test_channel_rejects_incompatible_shared_buffer_layout(&broker); active_request_allocates_and_releases_thread_id(&broker); active_request_closes_object_reference(&broker); @@ -1423,11 +1648,77 @@ mod tests { association_executes_distinct_slots_concurrently(&broker); association_allows_slot_reuse_during_response_emission(&broker); association_allows_out_of_order_responses(&broker); + association_preserves_the_initiating_failure(&broker); + } + + fn setup_failure_transfers_process_to_the_deployment_owner(broker: &BrokerCore) { + let mut channel = FakeHostControlChannel::new( + std::vec::Vec::from([Ok(HostReceive::Message(BrokerHandshakeRequest { + protocol_version: BROKER_PROTOCOL_VERSION, + }))]), + std::vec::Vec::new(), + ); + let shared_buffers = Arc::new(test_shared_buffers()); + let retained = Mutex::new(None); + + assert!(matches!( + setup_connection( + broker, + None, + None, + &mut channel, + shared_buffers, + test_readiness_sink(), + |process| { + *retained.lock().unwrap() = Some(Arc::clone(process)); + true + }, + |_| Err(()), + ), + Err(BrokerHostError::Channel(())) + )); + + let process = retained + .into_inner() + .unwrap() + .expect("deployment owner must retain the negotiated process"); + assert!(!process.is_running()); + process.retire(true); + } + + fn precreated_root_negotiates_without_startup_data(broker: &BrokerCore) { + let process = broker + .create_process(CallerCredential::Unauthenticated, None) + .unwrap(); + let initial_thread_id = process.create_thread().unwrap(); + let mut channel = FakeHostControlChannel::new( + std::vec::Vec::from([Ok(HostReceive::Message(BrokerHandshakeRequest { + protocol_version: BROKER_PROTOCOL_VERSION, + }))]), + std::vec::Vec::new(), + ); + + let association = setup_connection( + broker, + Some((Arc::clone(&process), initial_thread_id)), + None, + &mut channel, + Arc::new(test_shared_buffers()), + test_readiness_sink(), + |_| false, + |_| Ok(()), + ) + .unwrap() + .unwrap(); + + assert_eq!(association.process.id(), process.id()); + association.activate_process().unwrap(); + association.finish(); } fn association_shared_buffer_sequences_stage_file_data(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let shared_buffers = test_shared_buffers(); shared_buffers @@ -1539,7 +1830,7 @@ mod tests { fn association_shared_buffer_sequence_stages_random_data(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let shared_buffers = test_shared_buffers(); shared_buffers @@ -1594,7 +1885,7 @@ mod tests { provider: &TestStdioProvider, ) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let shared_buffers = test_shared_buffers(); shared_buffers @@ -1726,7 +2017,7 @@ mod tests { ); channel.next_request_id = 41; assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::PeerClosed ); assert_eq!( @@ -1734,6 +2025,8 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: root_process_id(1), + initial_thread_id: ThreadId(2), + startup: None, } ); let handle = match &channel.results[0] { @@ -1757,7 +2050,7 @@ mod tests { std::vec::Vec::from([Ok(HostReceive::PeerClosed)]), ); assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::PeerClosed ); assert_eq!( @@ -1768,7 +2061,9 @@ mod tests { }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: root_process_id(2), + process_id: root_process_id(3), + initial_thread_id: ThreadId(4), + startup: None, } ] ); @@ -1787,7 +2082,7 @@ mod tests { let setup_called = Cell::new(false); assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| { + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| { setup_called.set(true); Ok(()) }) @@ -1803,10 +2098,10 @@ mod tests { assert!(!setup_called.get()); assert_eq!( broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap() .id(), - root_process_id(3) + root_process_id(5) ); } @@ -1816,7 +2111,7 @@ mod tests { std::vec::Vec::new(), ); assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::ProtocolViolation ); assert_eq!( @@ -1834,14 +2129,16 @@ mod tests { std::vec::Vec::from([Ok(HostReceive::ProtocolViolation)]), ); assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::ProtocolViolation ); assert_eq!( channel.handshake_responses, [BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: root_process_id(4), + process_id: root_process_id(6), + initial_thread_id: ThreadId(7), + startup: None, }] ); assert!(channel.results.is_empty()); @@ -1857,7 +2154,7 @@ mod tests { )))]), ); channel.response_send_error = true; - match serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())) { + match serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())) { Err(BrokerHostError::Channel(())) => {} result => panic!("unexpected serve result: {result:?}"), } @@ -1876,7 +2173,7 @@ mod tests { ); channel.enqueue_readiness_requests_after_create = true; assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::PeerClosed ); assert_eq!( @@ -1915,7 +2212,7 @@ mod tests { ]), ); assert_eq!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())).unwrap(), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())).unwrap(), ConnectionTermination::PeerClosed ); assert_eq!( @@ -1945,7 +2242,7 @@ mod tests { ); channel.request_id_step = 0; assert!(matches!( - serve_test_channel(broker, &mut channel, &test_shared_buffers(), |_| Ok(())), + serve_test_channel(broker, &mut channel, test_shared_buffers(), |_| Ok(())), Err(BrokerHostError::Broker(ErrorCode::MalformedRequest)) )); assert_eq!( @@ -1972,7 +2269,7 @@ mod tests { serve_test_channel( broker, &mut channel, - &SharedBufferPool::new(FailingSharedMemory, SHARED_BUFFER_LAYOUT).unwrap(), + SharedBufferPool::new(FailingSharedMemory, SHARED_BUFFER_LAYOUT).unwrap(), |_| Ok(()), ), Err(BrokerHostError::Broker(ErrorCode::Internal)) @@ -2004,7 +2301,7 @@ mod tests { let setup_called = Cell::new(false); assert!(matches!( - serve_test_channel(broker, &mut channel, &shared_buffers, |_| { + serve_test_channel(broker, &mut channel, shared_buffers, |_| { setup_called.set(true); Ok(()) }), @@ -2016,7 +2313,7 @@ mod tests { fn active_request_closes_object_reference(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let response = handle_test_request( &process, @@ -2048,7 +2345,7 @@ mod tests { fn active_request_allocates_and_releases_thread_id(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let response = handle_test_request(&process, BrokerOperation::CreateThread); let BrokerResult::ThreadCreated(thread_id) = response else { @@ -2067,7 +2364,7 @@ mod tests { fn association_shared_buffer_sequences_stage_pipe_data(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let memory = TestSharedMemory::new(SHARED_BUFFER_POOL_SIZE); let shared_buffers = SharedBufferPool::new(memory.clone(), SHARED_BUFFER_LAYOUT).unwrap(); @@ -2128,7 +2425,7 @@ mod tests { fn association_shared_buffer_sequences_stage_socket_data(broker: &BrokerCore) { let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let shared_buffers = test_shared_buffers(); let created = handle_test_request_with_buffers( @@ -2416,8 +2713,8 @@ mod tests { entered_sender, release: Arc::clone(&release), }; - let shared_buffers = SharedBufferPool::new(memory, SHARED_BUFFER_LAYOUT).unwrap(); - let association = test_association(broker, &shared_buffers); + let shared_buffers = Arc::new(SharedBufferPool::new(memory, SHARED_BUFFER_LAYOUT).unwrap()); + let association = test_association(broker, Arc::clone(&shared_buffers)); let (_, first_write_handle) = litebox_broker_core::pipe::create(&association.process, 64, 16).unwrap(); let (_, second_write_handle) = @@ -2451,8 +2748,8 @@ mod tests { } fn association_allows_slot_reuse_during_response_emission(broker: &BrokerCore) { - let shared_buffers = test_shared_buffers(); - let association = test_association(broker, &shared_buffers); + let shared_buffers = Arc::new(test_shared_buffers()); + let association = test_association(broker, Arc::clone(&shared_buffers)); let release = Arc::new((Mutex::new(false), Condvar::new())); let (started_sender, started_receiver) = mpsc::sync_channel(1); @@ -2481,8 +2778,8 @@ mod tests { } fn association_allows_out_of_order_responses(broker: &BrokerCore) { - let shared_buffers = test_shared_buffers(); - let association = test_association(broker, &shared_buffers); + let shared_buffers = Arc::new(test_shared_buffers()); + let association = test_association(broker, Arc::clone(&shared_buffers)); let release = Arc::new((Mutex::new(false), Condvar::new())); let (first_started_sender, first_started_receiver) = mpsc::sync_channel(1); let (response_sender, response_receiver) = mpsc::sync_channel(2); @@ -2530,13 +2827,27 @@ mod tests { }); } - fn test_association<'a, Memory: SharedMemory>( + fn association_preserves_the_initiating_failure(broker: &BrokerCore) { + let shared_buffers = Arc::new(test_shared_buffers()); + let association = test_association(broker, Arc::clone(&shared_buffers)); + + assert!(matches!( + association.execute_request(event_create_request(1), |_| Err::<(), _>(())), + Err(BrokerHostError::Channel(())) + )); + assert!(matches!( + association.execute_request(event_create_request(2), |_| Ok::<_, ()>(())), + Err(BrokerHostError::AssociationFailed) + )); + } + + fn test_association( broker: &BrokerCore, - shared_buffers: &'a SharedBufferPool, - ) -> BrokerHostAssociation<'a, Memory> { + shared_buffers: Arc>, + ) -> BrokerHostAssociation { BrokerHostAssociation { process: broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(), shared_buffers, readiness_sink: test_readiness_sink(), @@ -2622,19 +2933,25 @@ mod tests { fn serve_test_channel( broker: &BrokerCore, control_channel: &mut FakeHostControlChannel, - shared_buffers: &SharedBufferPool, + shared_buffers: SharedBufferPool, send_shared_memory: impl FnOnce(&mut FakeHostControlChannel) -> core::result::Result<(), ()>, ) -> Result { let association = match setup_connection( broker, + None, + None, control_channel, - shared_buffers, + Arc::new(shared_buffers), test_readiness_sink(), + |_| false, send_shared_memory, )? { Ok(association) => association, Err(termination) => return Ok(termination), }; + association + .activate_process() + .expect("test broker process must activate once"); let result = (|| { loop { let request = match control_channel diff --git a/litebox_broker_host/src/test_support.rs b/litebox_broker_host/src/test_support.rs index 5a9911b4da..e42f3ae757 100644 --- a/litebox_broker_host/src/test_support.rs +++ b/litebox_broker_host/src/test_support.rs @@ -30,7 +30,7 @@ pub struct InProcessBrokerSetup { broker: BrokerCore, memory: Arc, readiness: Arc, - association: Option>>, + association: Option>>, } impl InProcessBrokerSetup { @@ -65,6 +65,9 @@ impl InProcessBrokerSetup { .association .take() .expect("the in-process local endpoint must negotiate before activation"); + association + .activate_process() + .expect("the in-process broker process must activate once"); InProcessBrokerChannel { association: Some(association), panicked: AtomicBool::new(false), @@ -90,17 +93,20 @@ impl LocalSetupChannel for InProcessBrokerSetup { self.association.is_none(), "the in-process broker association must be negotiated only once" ); - let shared_buffers = Box::leak(Box::new( + let shared_buffers = Arc::new( SharedBufferPool::new(Arc::clone(&self.memory), SHARED_BUFFER_LAYOUT) .expect("the in-process shared-buffer layout must be valid"), - )); + ); let mut host_setup = InProcessHostSetup { response: None }; let readiness: Arc = self.readiness.clone(); let association = crate::setup_connection( &self.broker, + None, + None, &mut host_setup, shared_buffers, readiness, + |_| false, |_| Ok(()), ) .expect("the in-process broker setup must succeed") @@ -114,7 +120,7 @@ impl LocalSetupChannel for InProcessBrokerSetup { /// Active request channel for an in-process broker association. pub struct InProcessBrokerChannel { - association: Option>>, + association: Option>>, panicked: AtomicBool, } @@ -323,7 +329,7 @@ mod tests { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) - .with_limits(BrokerCoreLimits::DEFAULT.with_thread_quotas(1, 1)) + .with_limits(BrokerCoreLimits::DEFAULT.with_thread_quotas(2, 2)) .build() .unwrap(); diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 523ff3f590..f8b05af910 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -31,7 +31,7 @@ mod stdio; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::sync::atomic::{AtomicU64, Ordering}; use litebox_broker_protocol::error::ErrorCode; @@ -39,6 +39,10 @@ use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, }; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupData, ProcessStartupDescriptor, +}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; use litebox_broker_protocol::{ @@ -58,6 +62,7 @@ pub use error::{BrokerLocalError, Result}; pub struct BrokerLocal { channel: Channel, process_id: ProcessId, + initial_thread_id: ThreadId, shared_buffers: SharedBufferPool>, next_request_id: AtomicU64, } @@ -68,12 +73,18 @@ pub struct BrokerNotifications { } impl BrokerLocal { - fn new(channel: Channel, process_id: ProcessId, shared_memory: Arc) -> Self { + fn new( + channel: Channel, + process_id: ProcessId, + initial_thread_id: ThreadId, + shared_memory: Arc, + ) -> Self { let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) .expect("broker association shared memory has an invalid size"); Self { channel, process_id, + initial_thread_id, shared_buffers, next_request_id: AtomicU64::new(0), } @@ -87,6 +98,8 @@ impl BrokerLocal { /// after negotiation, such as receiving shared memory and starting the /// active transport. Any additional endpoints activation produces, such as /// a notification receiver, are returned to the caller as `Activated`. + /// Child startup data is copied from the negotiated shared-buffer pool and + /// returned when present. /// /// # Panics /// @@ -101,7 +114,7 @@ impl BrokerLocal { (Channel, Arc, Activated), Channel::Error, >, - ) -> Result<(Self, Activated), Channel::Error> { + ) -> Result<(Self, Option, Activated), Channel::Error> { let requested = BROKER_PROTOCOL_VERSION; let request = BrokerHandshakeRequest { protocol_version: requested, @@ -117,6 +130,8 @@ impl BrokerLocal { response @ BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + initial_thread_id, + startup, } => { assert_eq!( requested, broker_protocol_version, @@ -124,7 +139,25 @@ impl BrokerLocal { ); let (channel, shared_memory, activated) = activate(setup).map_err(BrokerLocalError::Channel)?; - Ok((Self::new(channel, process_id, shared_memory), activated)) + let local = Self::new(channel, process_id, initial_thread_id, shared_memory); + let startup = match startup { + Some(startup) => { + let mut payload = Vec::new(); + payload + .try_reserve_exact(startup.buffer.length() as usize) + .map_err(|_| BrokerLocalError::Broker(ErrorCode::OutOfMemory))?; + payload.resize(startup.buffer.length() as usize, 0); + local.read_shared_buffer(startup.buffer, &mut payload); + Some(ProcessStartupData { + format: startup.format, + version: startup.version, + payload, + inherited_objects: startup.inherited_objects, + }) + } + None => None, + }; + Ok((local, startup, activated)) } BrokerHandshakeResponse::VersionMismatch { .. } => { Err(BrokerLocalError::Broker(ErrorCode::UnsupportedVersion)) @@ -132,7 +165,8 @@ impl BrokerLocal { BrokerHandshakeResponse::Error(error) => match error { ErrorCode::UnsupportedVersion | ErrorCode::PolicyDenied - | ErrorCode::ResourceExhausted => Err(BrokerLocalError::Broker(error)), + | ErrorCode::ResourceExhausted + | ErrorCode::OutOfMemory => Err(BrokerLocalError::Broker(error)), ErrorCode::MalformedRequest | ErrorCode::ProtocolState | ErrorCode::UnsupportedOperation @@ -148,6 +182,48 @@ impl BrokerLocal { self.process_id } + /// Returns the broker-assigned initial thread ID. + #[must_use] + pub const fn initial_thread_id(&self) -> ThreadId { + self.initial_thread_id + } + + /// Starts one child process. + /// + /// This call blocks until the child's broker association is active or + /// launch fails. The caller must retain exclusive ownership of the + /// bootstrap sequence until this method returns. + /// + /// # Panics + /// + /// Panics if the bootstrap length differs from the shared-buffer sequence + /// or the broker returns a response for another operation. + pub fn start_child_process( + &self, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + buffer: SharedBufferSequence, + bootstrap: &[u8], + inherited_objects: InheritedProcessObjects, + ) -> Result { + if buffer.length() > MAX_PROCESS_BOOTSTRAP_SIZE { + return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); + } + self.write_shared_buffer(buffer, bootstrap); + match self.request(BrokerOperation::StartChildProcess( + ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }, + ))? { + BrokerResult::ProcessStarted(started) => Ok(started), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => panic!("broker returned unexpected process-start response: {response:?}"), + } + } + /// Creates a broker thread belonging to this process. /// /// # Panics @@ -235,17 +311,9 @@ impl BrokerLocal { buffer.length() as usize, "shared data must match its buffer sequence" ); - let mut offset = 0; - for descriptor in buffer - .descriptors(self.shared_buffers.layout()) - .expect("shared buffer sequence must identify valid slot ranges") - { - let end = offset + descriptor.length as usize; - self.shared_buffers - .write(descriptor.slot_index, &data[offset..end]) - .expect("validated shared buffer sequence must be accessible"); - offset = end; - } + self.shared_buffers + .write_sequence(buffer, data) + .expect("validated shared buffer sequence must be accessible"); } fn read_shared_buffer(&self, buffer: SharedBufferSequence, destination: &mut [u8]) { @@ -253,26 +321,9 @@ impl BrokerLocal { destination.len() <= buffer.length() as usize, "shared buffer sequence must cover the destination" ); - let mut offset = 0; - for descriptor in buffer - .descriptors(self.shared_buffers.layout()) - .expect("shared buffer sequence must identify valid slot ranges") - { - if offset == destination.len() { - break; - } - let length = (destination.len() - offset).min(descriptor.length as usize); - let end = offset + length; - self.shared_buffers - .read(descriptor.slot_index, &mut destination[offset..end]) - .expect("validated shared buffer sequence must be accessible"); - offset = end; - } - assert_eq!( - offset, - destination.len(), - "shared buffer sequence must cover the destination" - ); + self.shared_buffers + .read_sequence(buffer, destination) + .expect("validated shared buffer sequence must be accessible"); } /// Checks the current readiness of a broker-owned object. @@ -350,11 +401,13 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), + startup: None, }), None, ); let setup_calls = Cell::new(0); - let (local, ()) = BrokerLocal::negotiate(channel, |channel| { + let (local, startup, ()) = BrokerLocal::negotiate(channel, |channel| { assert!(channel.sent_handshake_request.is_some()); assert!(channel.handshake_response.is_none()); assert!(channel.sent_request.borrow().is_none()); @@ -370,7 +423,9 @@ mod tests { }) ); assert_eq!(setup_calls.get(), 1); + assert!(startup.is_none()); assert_eq!(local.process_id(), test_process_id()); + assert_eq!(local.initial_thread_id(), ThreadId(2)); } #[test] @@ -612,6 +667,8 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id: test_process_id(), + initial_thread_id: ThreadId(2), + startup: None, }), None, ); @@ -663,21 +720,21 @@ mod tests { } #[test] - fn negotiate_returns_process_id_exhaustion() { - let channel = FakeControlChannel::new( - Some(BrokerHandshakeResponse::Error(ErrorCode::ResourceExhausted)), - None, - ); - let setup_called = Cell::new(false); + fn negotiate_returns_resource_allocation_errors() { + for error in [ErrorCode::ResourceExhausted, ErrorCode::OutOfMemory] { + let channel = + FakeControlChannel::new(Some(BrokerHandshakeResponse::Error(error)), None); + let setup_called = Cell::new(false); - assert!(matches!( - BrokerLocal::negotiate(channel, |channel| { - setup_called.set(true); - Ok((channel, noop_shared_memory(), ())) - }), - Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)) - )); - assert!(!setup_called.get()); + assert!(matches!( + BrokerLocal::negotiate(channel, |channel| { + setup_called.set(true); + Ok((channel, noop_shared_memory(), ())) + }), + Err(BrokerLocalError::Broker(reported)) if reported == error + )); + assert!(!setup_called.get()); + } } #[test] @@ -704,6 +761,8 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), + startup: None, }), None, ); @@ -727,6 +786,8 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), + startup: None, }), None, ); diff --git a/litebox_broker_local/src/test_support.rs b/litebox_broker_local/src/test_support.rs index 6712bf5dbf..3d1b6891a0 100644 --- a/litebox_broker_local/src/test_support.rs +++ b/litebox_broker_local/src/test_support.rs @@ -5,13 +5,14 @@ use alloc::sync::Arc; -use litebox_broker_protocol::ProcessId; +use litebox_broker_protocol::{ProcessId, ThreadId}; use litebox_broker_transport::channel::LocalCallChannel; use litebox_broker_transport::shared_memory::SharedMemory; use crate::BrokerLocal; const TEST_PROCESS_ID: ProcessId = ProcessId(1); +const TEST_THREAD_ID: ThreadId = ThreadId(2); /// Constructs an active broker-local association without exercising protocol negotiation. /// @@ -46,5 +47,5 @@ pub fn test_broker_local_with_process_id( where Channel: LocalCallChannel, { - BrokerLocal::new(channel, process_id, shared_memory) + BrokerLocal::new(channel, process_id, TEST_THREAD_ID, shared_memory) } diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index 01b4b54ad3..b29ebc87c9 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -14,6 +14,7 @@ use std::{ use anyhow::{Context as _, Result}; use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport_linux_userland::unix_socket::{ @@ -39,7 +40,9 @@ pub struct BrokerConnection { } /// Connects to and negotiates an association with a Linux-userland broker. -pub fn connect(control_socket_path: &Path) -> Result { +pub fn connect( + control_socket_path: &Path, +) -> Result<(BrokerConnection, Option)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let setup_channel = connect_with_retry( control_socket_path, @@ -54,7 +57,7 @@ pub fn connect(control_socket_path: &Path) -> Result { ) })?; let association_coordinator = Arc::new(BrokerAssociationFailureCoordinator::new()); - let (local, (notification_channel, positional_io_fds, shutdown_fd)) = + let (local, startup, (notification_channel, positional_io_fds, shutdown_fd)) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; @@ -85,13 +88,16 @@ pub fn connect(control_socket_path: &Path) -> Result { )) }) .context("broker negotiation failed")?; - Ok(BrokerConnection { - local, - notifications: BrokerNotifications::new(notification_channel), - coordinator: association_coordinator, - positional_io_fds, - shutdown_fd, - }) + Ok(( + BrokerConnection { + local, + notifications: BrokerNotifications::new(notification_channel), + coordinator: association_coordinator, + positional_io_fds, + shutdown_fd, + }, + startup, + )) } /// Starts the broker notification receiver for an active association. @@ -262,6 +268,8 @@ mod tests { &litebox_broker_protocol::message::BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }, ) .unwrap(); diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index 8ef679fd65..bd7906b8ce 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context as _, Result}; use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport_windows_userland::control_ring::{ @@ -26,7 +27,7 @@ pub struct BrokerConnection { } /// Connects to and negotiates an association with a Windows-userland broker. -pub fn connect(control_pipe: &OsStr) -> Result { +pub fn connect(control_pipe: &OsStr) -> Result<(BrokerConnection, Option)> { let deadline = Instant::now() + SETUP_TIMEOUT; let setup = WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(control_pipe, deadline) @@ -36,7 +37,7 @@ pub fn connect(control_pipe: &OsStr) -> Result { std::path::Path::new(control_pipe).display() ) })?; - let (local, notifications) = BrokerLocal::negotiate(setup, |mut setup| { + let (local, startup, notifications) = BrokerLocal::negotiate(setup, |mut setup| { let shared_memory = Arc::new(setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?); let control_memory = setup.receive_control_ring()?; let control_ring = ControlRing::new(control_memory).map_err(|error| { @@ -49,10 +50,13 @@ pub fn connect(control_pipe: &OsStr) -> Result { Ok((calls, shared_memory, notifications)) }) .context("broker negotiation failed")?; - Ok(BrokerConnection { - local, - notifications: BrokerNotifications::new(notifications), - }) + Ok(( + BrokerConnection { + local, + notifications: BrokerNotifications::new(notifications), + }, + startup, + )) } /// Starts the broker notification receiver for an active association. diff --git a/litebox_broker_platform_linux_userland/src/socket/tests/mod.rs b/litebox_broker_platform_linux_userland/src/socket/tests/mod.rs index c415b41ade..4ce729a0c4 100644 --- a/litebox_broker_platform_linux_userland/src/socket/tests/mod.rs +++ b/litebox_broker_platform_linux_userland/src/socket/tests/mod.rs @@ -378,7 +378,7 @@ fn directional_shutdown_survives_readiness_publication_failure() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); diff --git a/litebox_broker_platform_linux_userland/src/socket/tests/tcp.rs b/litebox_broker_platform_linux_userland/src/socket/tests/tcp.rs index 1835363f51..03c901ae60 100644 --- a/litebox_broker_platform_linux_userland/src/socket/tests/tcp.rs +++ b/litebox_broker_platform_linux_userland/src/socket/tests/tcp.rs @@ -24,10 +24,10 @@ fn connected_guest_tcp_pair(port: u16) -> GuestTcpPair { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -154,7 +154,7 @@ fn reactor_drives_a_loopback_tcp_socket() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -483,7 +483,7 @@ fn external_tcp_deferred_abortive_close_resets_peer() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -528,7 +528,7 @@ fn external_tcp_gateway_uses_host_loopback_and_keeps_guest_identity() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -604,7 +604,7 @@ fn external_tcp_route_keeps_guest_private_identity() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -645,7 +645,7 @@ fn tcp_connect_to_zero_port_returns_an_ordinary_socket_outcome() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -683,7 +683,7 @@ fn tcp_receive_survives_readiness_publication_failure() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -745,7 +745,7 @@ fn tcp_status_publication_failure_preserves_consumed_error() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -819,7 +819,7 @@ fn external_tcp_readiness_failure_does_not_fail_shared_reactor() { .unwrap(); let process_a = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published_a, publications_a) = channel(); let (retired_a, _retirements_a) = channel(); @@ -846,7 +846,7 @@ fn external_tcp_readiness_failure_does_not_fail_shared_reactor() { // Process B gets its own readiness sink, mirroring production's // per-association sinks. let process_b = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published_b, publications_b) = channel(); let (retired_b, _retirements_b) = channel(); @@ -942,7 +942,7 @@ fn external_tcp_connect_completion_readiness_failure_does_not_fail_shared_reacto ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -1029,7 +1029,7 @@ fn exhausted_tcp_peek_cache_refreshes_before_terminal_eof() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -1112,10 +1112,10 @@ fn accepted_guest_tcp_close_with_unread_data_preserves_reset() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1470,10 +1470,10 @@ fn guest_tcp_namespace_routes_across_processs_and_hides_private_backend() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let client_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1612,10 +1612,10 @@ fn tcp_exact_bindings_coexist_and_wildcard_accepts_concrete_destinations() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -1764,10 +1764,10 @@ fn connector_and_process_teardown_clean_bounded_pending_state() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1826,7 +1826,7 @@ fn connector_and_process_teardown_clean_bounded_pending_state() { assert_ne!(retirements.recv_timeout(TEST_TIMEOUT).unwrap(), listener); let final_connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let final_connector = create_socket(&final_connector_process, readiness); assert!(matches!( @@ -1870,10 +1870,10 @@ fn graceful_connector_close_preserves_late_accept_and_eof() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1964,10 +1964,10 @@ fn guest_tcp_zero_backlog_accepts_one_unspecified_destination() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, retirements) = channel(); @@ -2052,10 +2052,10 @@ fn guest_tcp_backlog_relisten_and_fifo_are_bounded() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -2171,10 +2171,10 @@ fn guest_tcp_stream_preserves_options_peek_waitall_and_half_close() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -2631,10 +2631,10 @@ fn guest_tcp_connect_publication_failure_purges_committed_queue() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, retirements) = channel(); @@ -2686,10 +2686,10 @@ fn guest_tcp_accept_publication_failure_purges_registered_endpoint() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -2751,10 +2751,10 @@ fn queued_guest_accept_transfers_capacity_without_global_growth() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, retirements) = channel(); @@ -2777,7 +2777,7 @@ fn queued_guest_accept_transfers_capacity_without_global_growth() { ); assert_eq!(provider.reactor.queued_guest_connection_count(), 1); let capacity_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let _global_capacity = create_socket(&capacity_process, readiness.clone()); assert_eq!( @@ -2811,10 +2811,10 @@ fn queued_guest_accept_rejects_exhausted_listener_process_capacity() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, retirements) = channel(); @@ -2868,10 +2868,10 @@ fn abortive_connector_close_releases_descriptor_capacity() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -2949,10 +2949,10 @@ fn stop_listening_cleanup_survives_readiness_failure() { ) .unwrap(); let listener_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let connector_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -3033,7 +3033,7 @@ fn reactor_drives_a_loopback_tcp_listener() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); diff --git a/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs b/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs index e4eec9c3c3..3421e967a1 100644 --- a/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs +++ b/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs @@ -48,7 +48,7 @@ fn udp_gateway_translates_sources_filters_spoofing_and_reuses_endpoint() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -133,7 +133,7 @@ fn connected_udp_gateway_preserves_guest_visible_mapping() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -188,7 +188,7 @@ fn unmatched_guest_udp_destinations_fail_closed() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -232,7 +232,7 @@ fn failed_initial_udp_readiness_does_not_retain_process_state() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -268,10 +268,10 @@ fn guest_udp_readiness_failure_rolls_back_enqueue() { ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let sender_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -365,7 +365,7 @@ fn external_udp_readiness_failure_does_not_fail_shared_reactor() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -477,7 +477,7 @@ fn udp_status_publication_failure_still_rearms_native_endpoint() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -550,7 +550,7 @@ fn udp_status_republishes_when_another_error_remains_pending() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -602,10 +602,10 @@ fn guest_udp_queue_pressure_drops_new_datagrams_successfully() { ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let sender_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -693,7 +693,7 @@ fn udp_external_peer_authorization_is_bounded_without_eviction() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -785,7 +785,7 @@ fn reactor_preserves_udp_datagram_semantics() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1139,10 +1139,10 @@ fn guest_udp_namespace_routes_across_processes_and_filters_private_endpoints() { ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let sender_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1403,10 +1403,10 @@ fn udp_exact_bindings_coexist_and_wildcard_covers_guest_addresses() { ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let sender_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -1591,7 +1591,7 @@ fn udp_native_endpoint_is_reused_and_retired() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, retirements) = channel(); @@ -1656,7 +1656,7 @@ fn udp_endpoint_staging_error_rolls_back_external_peer_reservation() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -1694,10 +1694,10 @@ fn stale_udp_datagrams_are_not_relabelled_after_guest_port_reuse() { ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let source_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1801,16 +1801,16 @@ fn udp_queued_datagrams_survive_source_process_teardown() { ) .unwrap(); let source_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let first_receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second_receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let replacement_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, retirements) = channel(); @@ -1903,7 +1903,7 @@ fn udp_queued_datagrams_survive_source_process_teardown() { second_receiver, ReadinessFlags::READ, ); - source_process.finish(); + source_process.cleanup(true); assert_eq!(retirements.recv_timeout(TEST_TIMEOUT).unwrap(), source); assert_eq!(provider.reactor.udp_queued_datagram_count(), 1); assert!( @@ -1946,10 +1946,10 @@ fn connected_guest_udp_enforces_barriers_peek_and_peer_generations() { ) .unwrap(); let first_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let second_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -2078,7 +2078,7 @@ fn connected_guest_udp_filters_other_wildcard_peer_aliases() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -2157,7 +2157,7 @@ fn wildcard_udp_reconnect_updates_guest_source_identity() { ) .unwrap(); let process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, _publications) = channel(); let (retired, _retirements) = channel(); @@ -2244,10 +2244,10 @@ fn externally_connected_udp_preserves_guest_routing_identity() { ) .unwrap(); let source_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); @@ -2354,10 +2354,10 @@ fn internally_connected_udp_drains_external_datagrams_without_delivering_them() ) .unwrap(); let receiver_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let sender_process = broker - .create_process(CallerCredential::Unauthenticated) + .create_process(CallerCredential::Unauthenticated, None) .unwrap(); let (published, publications) = channel(); let (retired, _retirements) = channel(); diff --git a/litebox_broker_protocol/src/lib.rs b/litebox_broker_protocol/src/lib.rs index 681b4f71eb..8806d1c297 100644 --- a/litebox_broker_protocol/src/lib.rs +++ b/litebox_broker_protocol/src/lib.rs @@ -21,6 +21,7 @@ pub mod event; pub mod fs; pub mod message; pub mod pipe; +pub mod process; pub mod random; pub mod readiness; pub mod shared_buffer; diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index 56b778f114..4e8a232622 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -17,6 +17,7 @@ use crate::pipe::{ CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; +use crate::process::{ProcessIdentity, ProcessStartupDescriptor}; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; use crate::socket::{ @@ -64,6 +65,8 @@ pub enum BrokerOperation { Stdio(StdioRequest), /// File request family. File(FileRequest), + /// Start one child process from an opaque platform bootstrap. + StartChildProcess(ProcessStartupDescriptor), } impl BrokerOperation { @@ -97,7 +100,8 @@ impl BrokerOperation { | FileRequest::Unlink(UnlinkFileRequest { path: buffer, .. }) | FileRequest::Mkdir(MkdirFileRequest { path: buffer, .. }) | FileRequest::Rmdir(RmdirFileRequest { path: buffer, .. }), - ) => Some(*buffer), + ) + | Self::StartChildProcess(ProcessStartupDescriptor { buffer, .. }) => Some(*buffer), Self::CreateThread | Self::ExitThread(_) | Self::CloseObject(_) @@ -144,6 +148,10 @@ pub enum BrokerHandshakeResponse { broker_protocol_version: ProtocolVersion, /// Assigned process ID. process_id: ProcessId, + /// Assigned initial thread ID. + initial_thread_id: ThreadId, + /// Child startup data, absent for the initial process. + startup: Option, }, /// Negotiation failed because the requested version is unsupported. /// @@ -234,6 +242,8 @@ pub enum BrokerResult { Stdio(StdioResponse), /// File response family. File(FileResponse), + /// A child established its broker association. + ProcessStarted(ProcessIdentity), /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), } diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs new file mode 100644 index 0000000000..87e42e6b86 --- /dev/null +++ b/litebox_broker_protocol/src/process.rs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +use alloc::vec::Vec; + +use crate::shared_buffer::SharedBufferSequence; +use crate::{ObjectHandle, ProcessId, ThreadId}; + +/// Maximum size of one versioned process bootstrap carried through the broker. +pub const MAX_PROCESS_BOOTSTRAP_SIZE: u32 = 64 * 1024; + +/// Maximum number of broker objects inherited by the bounded initial contract. +pub const MAX_INHERITED_PROCESS_OBJECTS: usize = 4; + +/// Platform-defined process-bootstrap format. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProcessBootstrapFormat(pub u32); + +/// Version of a platform-defined process-bootstrap format. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProcessBootstrapVersion(pub u16); + +/// Ordered broker-object handles inherited by a child. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct InheritedProcessObjects { + handles: [ObjectHandle; MAX_INHERITED_PROCESS_OBJECTS], + count: u8, +} + +impl InheritedProcessObjects { + /// Empty inherited-object list. + pub const EMPTY: Self = Self { + handles: [ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS], + count: 0, + }; + + /// Creates a bounded ordered inherited-object list. + pub fn new(handles: &[ObjectHandle]) -> Option { + if handles.len() > MAX_INHERITED_PROCESS_OBJECTS { + return None; + } + let Ok(count) = u8::try_from(handles.len()) else { + return None; + }; + let mut stored = [ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS]; + stored[..handles.len()].copy_from_slice(handles); + Some(Self { + handles: stored, + count, + }) + } + + /// Returns inherited handles in manifest order. + #[must_use] + pub fn as_slice(&self) -> &[ObjectHandle] { + &self.handles[..usize::from(self.count)] + } +} + +/// Child startup descriptor transported through the broker protocol. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessStartupDescriptor { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Operation-scoped shared-buffer sequence containing the bootstrap bytes. + pub buffer: SharedBufferSequence, + /// Broker handles inherited in manifest order. + pub inherited_objects: InheritedProcessObjects, +} + +/// Owned child startup data delivered during broker negotiation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProcessStartupData { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Opaque platform bytes. + pub payload: Vec, + /// Child-owned broker handles in the parent's inheritance-manifest order. + pub inherited_objects: InheritedProcessObjects, +} + +/// Broker-assigned process and initial-thread identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessIdentity { + /// Broker-assigned process ID. + pub process_id: ProcessId, + /// Broker-assigned initial thread ID. + pub initial_thread_id: ThreadId, +} diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 6e2087c683..6db82545b3 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -12,8 +12,7 @@ //! //! New object families should add a top-level broker message tag and a private //! family codec module instead of adding flat helpers here. Existing payloads -//! are positional; changing fields is an ABI change, so prefer a new operation -//! tag or explicit negotiated-version gate for payload evolution. +//! are positional, so update both endpoints and their codec tests together. use alloc::vec::Vec; use thiserror::Error; @@ -23,6 +22,10 @@ use crate::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, ReadinessNotification, }; +use crate::process::{ + InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupDescriptor, +}; use crate::readiness::ReadinessFlags; use primitive::{Decoder, Encoder}; @@ -45,6 +48,7 @@ const REQUEST_TAG_STDIO: u8 = 7; const REQUEST_TAG_FILE: u8 = 8; const REQUEST_TAG_CREATE_THREAD: u8 = 9; const REQUEST_TAG_EXIT_THREAD: u8 = 10; +const REQUEST_TAG_START_CHILD_PROCESS: u8 = 11; // Paired request and successful-response tags intentionally share values. const RESPONSE_TAG_NEGOTIATED: u8 = 0; @@ -58,6 +62,7 @@ const RESPONSE_TAG_STDIO: u8 = 7; const RESPONSE_TAG_FILE: u8 = 8; const RESPONSE_TAG_THREAD_CREATED: u8 = 9; const RESPONSE_TAG_THREAD_EXITED: u8 = 10; +const RESPONSE_TAG_PROCESS_STARTED: u8 = 11; // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; @@ -67,7 +72,7 @@ const RESPONSE_TAG_VERSION_MISMATCH: u8 = 255; const NOTIFICATION_TAG_READINESS: u8 = 0; /// Maximum byte length of any encoded active request or response. -pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 67; +pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 85; /// Maximum byte length of any encoded broker notification. pub const MAX_ENCODED_NOTIFICATION_SIZE: usize = 13; @@ -116,7 +121,8 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_EXIT_THREAD + | REQUEST_TAG_START_CHILD_PROCESS => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -185,6 +191,19 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); fs::encode_fs_request(&mut encoder, request); } + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }) => { + encoder.u8(REQUEST_TAG_START_CHILD_PROCESS); + encoder.request_id(request_id); + encoder.u32(format.0); + encoder.u16(version.0); + encoder.shared_buffer_sequence(buffer); + encode_inherited_objects(&mut encoder, inherited_objects); + } } encoder.finish() } @@ -204,7 +223,8 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_STDIO | REQUEST_TAG_FILE | REQUEST_TAG_CREATE_THREAD - | REQUEST_TAG_EXIT_THREAD => {} + | REQUEST_TAG_EXIT_THREAD + | REQUEST_TAG_START_CHILD_PROCESS => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -219,6 +239,14 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_FILL_RANDOM => BrokerOperation::FillRandom(decoder.shared_buffer_sequence()?), REQUEST_TAG_STDIO => BrokerOperation::Stdio(stdio::decode_stdio_request(&mut decoder)?), REQUEST_TAG_FILE => BrokerOperation::File(fs::decode_fs_request(&mut decoder)?), + REQUEST_TAG_START_CHILD_PROCESS => { + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }) + } _ => unreachable!("active request tag was validated"), }; decoder.finish()?; @@ -238,10 +266,28 @@ pub fn encode_handshake_response(response: BrokerHandshakeResponse) -> Vec { BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + initial_thread_id, + startup, } => { encoder.u8(RESPONSE_TAG_NEGOTIATED); encoder.protocol_version(broker_protocol_version); encoder.process_id(process_id); + encoder.thread_id(initial_thread_id); + match startup { + Some(ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }) => { + encoder.u8(1); + encoder.u32(format.0); + encoder.u16(version.0); + encoder.shared_buffer_sequence(buffer); + encode_inherited_objects(&mut encoder, inherited_objects); + } + None => encoder.u8(0), + } } BrokerHandshakeResponse::VersionMismatch { broker_protocol_version, @@ -265,6 +311,17 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result BrokerHandshakeResponse::Negotiated { broker_protocol_version: decoder.protocol_version()?, process_id: decoder.process_id()?, + initial_thread_id: decoder.thread_id()?, + startup: match decoder.u8()? { + 0 => None, + 1 => Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }), + _ => return Err(WireError::InvalidTag), + }, }, RESPONSE_TAG_EVENT | RESPONSE_TAG_OBJECT_CLOSED @@ -276,7 +333,8 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { + | RESPONSE_TAG_THREAD_EXITED + | RESPONSE_TAG_PROCESS_STARTED => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_VERSION_MISMATCH => BrokerHandshakeResponse::VersionMismatch { @@ -346,6 +404,15 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.request_id(request_id); fs::encode_fs_response(&mut encoder, response); } + BrokerResult::ProcessStarted(ProcessIdentity { + process_id, + initial_thread_id, + }) => { + encoder.u8(RESPONSE_TAG_PROCESS_STARTED); + encoder.request_id(request_id); + encoder.process_id(process_id); + encoder.thread_id(initial_thread_id); + } BrokerResult::Error(error) => { encoder.u8(RESPONSE_TAG_ERROR); encoder.request_id(request_id); @@ -373,7 +440,8 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_STDIO | RESPONSE_TAG_FILE | RESPONSE_TAG_THREAD_CREATED - | RESPONSE_TAG_THREAD_EXITED => {} + | RESPONSE_TAG_THREAD_EXITED + | RESPONSE_TAG_PROCESS_STARTED => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -389,12 +457,38 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_RANDOM_FILLED => BrokerResult::RandomFilled, RESPONSE_TAG_STDIO => BrokerResult::Stdio(stdio::decode_stdio_response(&mut decoder)?), RESPONSE_TAG_FILE => BrokerResult::File(fs::decode_fs_response(&mut decoder)?), + RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(ProcessIdentity { + process_id: decoder.process_id()?, + initial_thread_id: decoder.thread_id()?, + }), _ => unreachable!("active response tag was validated"), }; decoder.finish()?; Ok(BrokerResponse { request_id, result }) } +fn encode_inherited_objects(encoder: &mut Encoder, objects: InheritedProcessObjects) { + encoder.u8(u8::try_from(objects.as_slice().len()) + .expect("bounded inherited-object count must fit in u8")); + for handle in objects.as_slice() { + encoder.handle(*handle); + } +} + +fn decode_inherited_objects( + decoder: &mut Decoder<'_>, +) -> Result { + let count = usize::from(decoder.u8()?); + if count > MAX_INHERITED_PROCESS_OBJECTS { + return Err(WireError::InvalidTag); + } + let mut handles = [crate::ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS]; + for handle in &mut handles[..count] { + *handle = decoder.handle()?; + } + InheritedProcessObjects::new(&handles[..count]).ok_or(WireError::InvalidTag) +} + fn encode_error_code(encoder: &mut Encoder, error: ErrorCode) { encoder.u16(match error { ErrorCode::UnsupportedVersion => 1, @@ -484,6 +578,10 @@ mod tests { CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; + use crate::process::{ + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessIdentity, + ProcessStartupDescriptor, + }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ AcceptSocketRequest, AcceptSocketResponse, AddressFamily, BindSocketRequest, @@ -539,6 +637,9 @@ mod tests { RESPONSE_TAG_RANDOM_FILLED, RESPONSE_TAG_STDIO, RESPONSE_TAG_FILE, + RESPONSE_TAG_THREAD_CREATED, + RESPONSE_TAG_THREAD_EXITED, + RESPONSE_TAG_PROCESS_STARTED, ], [ REQUEST_TAG_NEGOTIATE, @@ -550,6 +651,9 @@ mod tests { REQUEST_TAG_FILL_RANDOM, REQUEST_TAG_STDIO, REQUEST_TAG_FILE, + REQUEST_TAG_CREATE_THREAD, + REQUEST_TAG_EXIT_THREAD, + REQUEST_TAG_START_CHILD_PROCESS, ] ); assert_eq!( @@ -818,6 +922,18 @@ mod tests { name: TcpOptionName::KeepAlive, })), BrokerOperation::Socket(SocketRequest::Status(SocketStatusRequest { handle })), + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(u32::MAX), + version: ProcessBootstrapVersion(u16::MAX), + buffer: largest_sequence, + inherited_objects: InheritedProcessObjects::new(&[ + ObjectHandle(1), + ObjectHandle(2), + ObjectHandle(3), + ObjectHandle(4), + ]) + .unwrap(), + }), ]; let mut maximum_encoded_size = 0; @@ -991,10 +1107,23 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + initial_thread_id: thread_id(2), + startup: None, }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(7), + initial_thread_id: thread_id(8), + startup: Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(0x7465_7374), + version: ProcessBootstrapVersion(1), + buffer: sequence(0, 37), + inherited_objects: InheritedProcessObjects::new(&[ + ObjectHandle(5), + ObjectHandle(6), + ]) + .unwrap(), + }), }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), @@ -1150,6 +1279,14 @@ mod tests { BrokerResult::File(FileResponse::Mkdir), BrokerResult::File(FileResponse::Rmdir), BrokerResult::File(FileResponse::Failed(FileError::Io)), + BrokerResult::ProcessStarted(ProcessIdentity { + process_id: process_id(u32::MAX), + initial_thread_id: thread_id(u32::MAX - 1), + }), + BrokerResult::ProcessStarted(ProcessIdentity { + process_id: process_id(9), + initial_thread_id: thread_id(11), + }), BrokerResult::Error(ErrorCode::PolicyDenied), BrokerResult::Error(ErrorCode::WouldBlock), BrokerResult::Error(ErrorCode::PeerClosed), @@ -1617,7 +1754,7 @@ mod tests { #[test] fn decode_rejects_malformed_handshake_response_frames() { assert_eq!( - decode_handshake_response(&[0xfc, 1, 2, 3]), + decode_handshake_response(&[0xfb, 1, 2, 3]), Err(WireError::InvalidTag) ); assert_eq!( @@ -1652,10 +1789,20 @@ mod tests { Err(WireError::WrongMessagePhase) ); - let mut frame = encode_handshake_response(BrokerHandshakeResponse::Negotiated { + let negotiated = BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), - }); + initial_thread_id: thread_id(2), + startup: None, + }; + let mut invalid_startup = encode_handshake_response(negotiated.clone()); + *invalid_startup.last_mut().unwrap() = 2; + assert_eq!( + decode_handshake_response(&invalid_startup), + Err(WireError::InvalidTag) + ); + + let mut frame = encode_handshake_response(negotiated); frame.push(0xff); assert_eq!( decode_handshake_response(&frame), @@ -1666,13 +1813,26 @@ mod tests { #[test] fn decode_rejects_malformed_response_frames() { assert_eq!( - decode_response(&[0xfc, 1, 2, 3]), + decode_response(&[0xfb, 1, 2, 3]), Err(WireError::InvalidTag) ); for response in [ BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + initial_thread_id: thread_id(2), + startup: None, + }, + BrokerHandshakeResponse::Negotiated { + broker_protocol_version: ProtocolVersion(1), + process_id: process_id(2), + initial_thread_id: thread_id(3), + startup: Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(3), + version: ProcessBootstrapVersion(4), + buffer: sequence(0, 5), + inherited_objects: InheritedProcessObjects::EMPTY, + }), }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs index fad5613cdb..c831404a9f 100644 --- a/litebox_broker_transport/src/pending_calls.rs +++ b/litebox_broker_transport/src/pending_calls.rs @@ -77,11 +77,11 @@ pub enum PendingCallsError { /// Concurrent registry of requests awaiting broker responses. pub struct PendingCalls { - state: Sync::Mutex>, - capacity_available: Sync::Condvar>, + state: Sync::Mutex>, + capacity_available: Sync::Condvar>, } -struct PendingCallsState { +struct PendingCallsInner { calls: BTreeMap>>, failure: Option>, } @@ -123,7 +123,7 @@ impl PendingCalls { /// Creates an empty live pending-call registry. pub fn new() -> Self { Self { - state: Sync::mutex(PendingCallsState { + state: Sync::mutex(PendingCallsInner { calls: BTreeMap::new(), failure: None, }), @@ -138,7 +138,7 @@ impl PendingCalls { ) -> Result>, PendingCallsError> { let pending_call = Arc::new(PendingCall::new()); let mut state = self.state.lock(); - while state.calls.len() == MAX_PENDING_CALLS && state.failure.is_none() { + while state.calls.len() >= MAX_PENDING_CALLS && state.failure.is_none() { state = self.capacity_available.wait(state); } if let Some(error) = state.failure.as_ref() { @@ -154,6 +154,11 @@ impl PendingCalls { } /// Completes the pending call identified by `response`. + /// + /// # Panics + /// + /// Panics if the internal ordinary-call count is inconsistent with the + /// registered requests. pub fn complete(&self, response: BrokerResponse) -> Result<(), PendingCallsError> { let pending_call = { let mut state = self.state.lock(); @@ -163,7 +168,7 @@ impl PendingCalls { let Some(pending_call) = state.calls.remove(&response.request_id) else { return Err(PendingCallsError::UnknownResponseId); }; - self.capacity_available.notify_one(); + self.capacity_available.notify_all(); pending_call }; pending_call.resolve(Ok(response)); diff --git a/litebox_broker_transport/src/shared_memory.rs b/litebox_broker_transport/src/shared_memory.rs index df15ba0dd4..8e08e7ba2e 100644 --- a/litebox_broker_transport/src/shared_memory.rs +++ b/litebox_broker_transport/src/shared_memory.rs @@ -13,7 +13,7 @@ use alloc::sync::Arc; use thiserror::Error; use litebox_broker_protocol::shared_buffer::{ - SharedBufferLayout, SharedBufferLayoutError, SharedBufferSlotIndex, + SharedBufferLayout, SharedBufferLayoutError, SharedBufferSequence, SharedBufferSlotIndex, }; /// Error accessing a shared-memory resource. @@ -138,6 +138,9 @@ pub enum SharedBufferError { /// The backing shared-memory length does not exactly match the layout. #[error("shared-memory length does not match the shared-buffer layout")] MemoryLengthMismatch, + /// The requested transfer does not fit in the shared-buffer sequence. + #[error("shared-buffer sequence does not cover the requested transfer")] + TransferExceedsSequence, /// The backing shared-memory access failed. #[error("shared-memory access failed: {0}")] SharedMemory(#[from] SharedMemoryError), @@ -196,6 +199,58 @@ impl SharedBufferPool { self.memory.write(range.start, source)?; Ok(()) } + + /// Copies a sequence prefix into `destination`. + pub fn read_sequence( + &self, + sequence: SharedBufferSequence, + destination: &mut [u8], + ) -> Result<(), SharedBufferError> { + let descriptors = sequence.descriptors(self.layout)?; + if destination.len() > sequence.length() as usize { + return Err(SharedBufferError::TransferExceedsSequence); + } + let mut offset = 0; + for descriptor in descriptors { + if offset == destination.len() { + break; + } + let length = (destination.len() - offset).min(descriptor.length as usize); + let end = offset + length; + self.read(descriptor.slot_index, &mut destination[offset..end])?; + offset = end; + } + if offset != destination.len() { + return Err(SharedBufferError::TransferExceedsSequence); + } + Ok(()) + } + + /// Copies `source` into a sequence prefix. + pub fn write_sequence( + &self, + sequence: SharedBufferSequence, + source: &[u8], + ) -> Result<(), SharedBufferError> { + let descriptors = sequence.descriptors(self.layout)?; + if source.len() > sequence.length() as usize { + return Err(SharedBufferError::TransferExceedsSequence); + } + let mut offset = 0; + for descriptor in descriptors { + if offset == source.len() { + break; + } + let length = (source.len() - offset).min(descriptor.length as usize); + let end = offset + length; + self.write(descriptor.slot_index, &source[offset..end])?; + offset = end; + } + if offset != source.len() { + return Err(SharedBufferError::TransferExceedsSequence); + } + Ok(()) + } } #[cfg(test)] @@ -229,6 +284,31 @@ mod tests { ); } + #[test] + fn pool_copies_sequence_prefixes_across_slots() { + let layout = SharedBufferLayout::new(4, 3).unwrap(); + let memory = Arc::new(TestSharedMemory::new(layout.total_len())); + let pool = SharedBufferPool::new(Arc::clone(&memory), layout).unwrap(); + let sequence = + SharedBufferSequence::new(&[SharedBufferSlotIndex(0), SharedBufferSlotIndex(2)], 6) + .unwrap(); + + pool.write_sequence(sequence, &[1, 2, 3, 4, 5, 6]).unwrap(); + assert_eq!(memory.bytes(), [1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 0, 0]); + + let mut destination = [0; 5]; + pool.read_sequence(sequence, &mut destination).unwrap(); + assert_eq!(destination, [1, 2, 3, 4, 5]); + assert_eq!( + pool.write_sequence(sequence, &[0; 7]), + Err(SharedBufferError::TransferExceedsSequence) + ); + assert_eq!( + pool.read_sequence(sequence, &mut [0; 7]), + Err(SharedBufferError::TransferExceedsSequence) + ); + } + struct TestSharedMemory(Mutex>); impl TestSharedMemory { diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 1ddfb23d17..671386f1f5 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -613,6 +613,8 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); assert!(matches!( diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 7ea2f9f291..d4b48f296a 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -524,7 +524,7 @@ mod control_ring_tests { use std::os::unix::net::UnixListener; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Barrier, mpsc}; + use std::sync::mpsc; type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; @@ -708,6 +708,8 @@ mod control_ring_tests { &encode_handshake_response(BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }), None, ) @@ -765,39 +767,6 @@ mod control_ring_tests { )); } - #[test] - fn pending_capacity_blocks_before_sixty_fifth_publication() { - let (channel, shutdown, mut responses, mut requests, _peer) = activate_local(|| {}); - let channel = Arc::new(channel); - let start = Arc::new(Barrier::new(MAX_PENDING_CALLS + 2)); - let callers = (0..=MAX_PENDING_CALLS) - .map(|id| { - let channel = Arc::clone(&channel); - let start = Arc::clone(&start); - thread::spawn(move || { - start.wait(); - channel.call(request(id as u64)) - }) - }) - .collect::>(); - start.wait(); - - let mut published = Vec::new(); - for _ in 0..MAX_PENDING_CALLS { - published.push(read_request(&mut requests).request_id); - } - write_payload(&mut responses, &encode_response(response(published[0]))); - let released = read_request(&mut requests).request_id; - assert!(!published.contains(&released)); - - shutdown.shutdown().unwrap(); - let completed = callers - .into_iter() - .map(|caller| usize::from(caller.join().unwrap().is_ok())) - .sum::(); - assert_eq!(completed, 1); - } - #[test] fn unknown_duplicate_and_malformed_responses_fail_closed() { for payload_kind in 0..3 { diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index a02f987ec2..d796e31b83 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -138,7 +138,7 @@ fn wide_string(value: &OsStr) -> IoResult> { #[cfg(test)] mod tests { use super::*; - use std::sync::{Arc, Barrier}; + use std::sync::Arc; use std::time::{Duration, Instant}; use crate::setup::{OwnedEvent, read_pipe_until_cancelled, write_frame}; @@ -295,6 +295,8 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); let memory = WindowsSharedMemory::create(4096).unwrap(); @@ -337,6 +339,8 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); let (mut requests, responses, mut notifications, _shutdown) = @@ -413,6 +417,8 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); let (mut requests, responses, _notifications, _shutdown) = @@ -470,84 +476,6 @@ mod tests { host.join().unwrap(); } - #[test] - fn pending_capacity_blocks_before_sixty_fifth_publication() { - let control_name = pipe_name("pending-capacity-control"); - let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); - let (local_ring, host_ring) = control_rings(); - let host = std::thread::spawn(move || { - let mut setup = accept_host_guaranteed(control_listener); - assert!(matches!( - setup.recv_handshake_request().unwrap(), - HostReceive::Message(BrokerHandshakeRequest { .. }) - )); - setup - .send_handshake_response(&BrokerHandshakeResponse::Negotiated { - broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: litebox_broker_protocol::ProcessId(1), - }) - .unwrap(); - let (mut requests, responses, _notifications, shutdown) = - setup.into_active(host_ring).unwrap(); - - let mut published = Vec::new(); - for _ in 0..litebox_broker_transport::pending_calls::MAX_PENDING_CALLS { - let HostReceive::Message(request) = requests.recv_request().unwrap() else { - panic!("expected pending request"); - }; - published.push(request.request_id); - } - responses - .send_response(&BrokerResponse { - request_id: published[0], - result: BrokerResult::ObjectClosed, - }) - .unwrap(); - let HostReceive::Message(released) = requests.recv_request().unwrap() else { - panic!("expected released request"); - }; - assert!(!published.contains(&released.request_id)); - shutdown.shutdown().unwrap(); - }); - - let deadline = Instant::now() + Duration::from_secs(5); - let mut setup = - WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(&control_name, deadline) - .unwrap(); - setup - .send_handshake_request(&BrokerHandshakeRequest { - protocol_version: BROKER_PROTOCOL_VERSION, - }) - .unwrap(); - setup.recv_handshake_response().unwrap().unwrap(); - let (calls, _notifications) = setup.into_active(local_ring).unwrap(); - let calls = Arc::new(calls); - let start = Arc::new(Barrier::new( - litebox_broker_transport::pending_calls::MAX_PENDING_CALLS + 2, - )); - let callers = (0..=litebox_broker_transport::pending_calls::MAX_PENDING_CALLS) - .map(|id| { - let calls = Arc::clone(&calls); - let start = Arc::clone(&start); - std::thread::spawn(move || { - start.wait(); - calls.call(BrokerRequest { - request_id: RequestId(id as u64), - operation: BrokerOperation::CloseObject(ObjectHandle(id as u64)), - }) - }) - }) - .collect::>(); - start.wait(); - - host.join().unwrap(); - let completed = callers - .into_iter() - .map(|caller| usize::from(caller.join().unwrap().is_ok())) - .sum::(); - assert_eq!(completed, 1); - } - #[test] fn host_shutdown_interrupts_blocked_request_receive() { let control_name = pipe_name("shutdown-control"); @@ -563,6 +491,8 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); let (mut requests, _responses, _notifications, shutdown) = diff --git a/litebox_broker_userland/src/lib.rs b/litebox_broker_userland/src/lib.rs index 6650955e58..f22d14f728 100644 --- a/litebox_broker_userland/src/lib.rs +++ b/litebox_broker_userland/src/lib.rs @@ -15,6 +15,8 @@ //! mode. pub mod builder; +#[cfg(any(target_os = "linux", all(windows, target_arch = "x86_64")))] +mod process_launcher; pub mod readiness; #[cfg(any(target_os = "linux", all(windows, target_arch = "x86_64")))] pub mod runner; diff --git a/litebox_broker_userland/src/linux.rs b/litebox_broker_userland/src/linux.rs index 3f6add0788..1a229d60d9 100644 --- a/litebox_broker_userland/src/linux.rs +++ b/litebox_broker_userland/src/linux.rs @@ -222,7 +222,7 @@ fn serve_control_stream( ) -> IoResult<()> { let control_channel = UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - litebox_broker_userland::runtime::serve_association( + litebox_broker_userland::runtime::serve_in_process_runner_association( broker, control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), diff --git a/litebox_broker_userland/src/main.rs b/litebox_broker_userland/src/main.rs index d05dc79428..74b5a9fdf0 100644 --- a/litebox_broker_userland/src/main.rs +++ b/litebox_broker_userland/src/main.rs @@ -202,8 +202,7 @@ fn run_runner_instance( if let Some(proxy_url) = proxy_url { config = config.with_proxy_url(proxy_url.to_owned()); } - let runner_status = litebox_broker_userland::runner::RunnerInstance::start(config)? - .run_to_completion(broker)?; + let runner_status = litebox_broker_userland::runner::run_to_completion(config, broker)?; if !runner_status.success() { return Err(IoError::other(format!("runner exited with {runner_status}")).into()); } diff --git a/litebox_broker_userland/src/process_launcher.rs b/litebox_broker_userland/src/process_launcher.rs new file mode 100644 index 0000000000..bab5c445c7 --- /dev/null +++ b/litebox_broker_userland/src/process_launcher.rs @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Runner launch support backed by broker-owned process lifecycle state. + +use std::io::{Error as IoError, Result as IoResult}; +use std::process::ExitStatus; +use std::sync::mpsc::{SyncSender, sync_channel}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Instant; + +use litebox_broker_core::{ + BrokerCore, BrokerError, BrokerProcess, CallerCredential, ProcessLifecycleSink, +}; +use litebox_broker_host::ProcessLauncher; +use litebox_broker_protocol::ThreadId; +use litebox_broker_protocol::process::ProcessStartupData; + +use crate::runner::{RunnerCompletion, RunnerConfig, RunnerInstance}; + +/// Userland implementation that starts one out-of-process runner per process. +pub(crate) struct UserlandProcessLauncher { + broker: BrokerCore, + started_runner_config: RunnerConfig, + lifecycle: Arc, +} + +/// Pending association for a broker-created runner process. +pub(crate) struct PendingRunnerAssociation { + pub(super) process: Arc, + initial_thread_id: ThreadId, + data: Option, +} + +impl PendingRunnerAssociation { + fn new( + process: Arc, + initial_thread_id: ThreadId, + data: Option, + ) -> Self { + Self { + process, + initial_thread_id, + data, + } + } + + pub(crate) fn into_process_and_startup( + self, + ) -> ((Arc, ThreadId), Option) { + ((self.process, self.initial_thread_id), self.data) + } +} + +#[derive(Default)] +struct ProcessLifecycleNotifier { + state: Mutex<()>, + changed: Condvar, +} + +impl ProcessLifecycleSink for ProcessLifecycleNotifier { + fn changed(&self) { + self.notify(); + } +} + +impl ProcessLifecycleNotifier { + fn notify(&self) { + let _state = self.state.lock().expect("process lifecycle mutex poisoned"); + self.changed.notify_all(); + } + + fn wait_for_start( + &self, + process: &BrokerProcess, + deadline: Instant, + ) -> Result<(), BrokerError> { + let mut state = self.state.lock().expect("process lifecycle mutex poisoned"); + loop { + if let Some(result) = process.startup_result() { + return result; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + drop(state); + return process.fail_start(BrokerError::PeerClosed, true, true); + } + let (next_state, wait_result) = self + .changed + .wait_timeout(state, remaining) + .expect("process lifecycle mutex poisoned"); + state = next_state; + if wait_result.timed_out() { + drop(state); + return process.fail_start(BrokerError::PeerClosed, true, true); + } + } + } + + fn wait_for_drain(&self, broker: &BrokerCore) { + let mut state = self.state.lock().expect("process lifecycle mutex poisoned"); + while broker.has_processes() { + state = self + .changed + .wait(state) + .expect("process lifecycle mutex poisoned"); + } + } +} + +impl UserlandProcessLauncher { + fn new(started_runner_config: RunnerConfig, broker: BrokerCore) -> Arc { + let lifecycle = Arc::new(ProcessLifecycleNotifier::default()); + let broker = broker.with_process_lifecycle_sink(lifecycle.clone()); + Arc::new(Self { + broker, + started_runner_config, + lifecycle, + }) + } + + fn wait_for_drain(&self) { + self.lifecycle.wait_for_drain(&self.broker); + } + + pub(crate) fn run_root(config: RunnerConfig, broker: &BrokerCore) -> IoResult { + let launcher = Self::new(config.without_initial_arguments(), broker.clone()); + let process = launcher + .broker + .create_process(CallerCredential::HostGuaranteed, None) + .map_err(broker_io_error)?; + let initial_thread_id = match process.create_thread() { + Ok(initial_thread_id) => initial_thread_id, + Err(error) => { + process.retire(true); + return Err(broker_io_error(error)); + } + }; + let association = + PendingRunnerAssociation::new(Arc::clone(&process), initial_thread_id, None); + let (completion_sender, completion_receiver) = sync_channel(1); + let startup = + Arc::clone(&launcher).launch_runner(association, config, Some(completion_sender)); + if let Err(error) = startup { + drop(process); + launcher.wait_for_drain(); + let fallback = broker_io_error(error); + return match completion_receiver.recv() { + Ok(Err(error)) => Err(error), + Ok(Ok(_)) | Err(_) => Err(fallback), + }; + } + let result = completion_receiver + .recv() + .unwrap_or_else(|_| Err(IoError::other("root runner completion channel closed"))); + drop(process); + launcher.wait_for_drain(); + result + } + + fn launch_runner( + self: Arc, + association: PendingRunnerAssociation, + config: RunnerConfig, + completion_sender: Option>>, + ) -> Result<(), BrokerError> { + let process = Arc::clone(&association.process); + let process_id = process.id(); + let Ok(instance) = RunnerInstance::start(config) else { + let _ = process.fail_start(BrokerError::PeerClosed, false, false); + process.retire(true); + return Err(BrokerError::PeerClosed); + }; + let setup_deadline = instance.setup_deadline(); + let broker = self.broker.clone(); + let launcher = Arc::clone(&self); + let completion_process = Arc::clone(&process); + let thread = std::thread::Builder::new() + .name(format!("litebox-runner-{}", process_id.0)) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + instance.run_process_to_completion(association, broker, launcher) + })); + let (completion, thread_panicked) = match outcome { + Ok(completion) => (completion, false), + Err(_) => (RunnerCompletion::panicked(), true), + }; + let abnormal = completion.is_abnormal(thread_panicked); + Self::runner_finished(&completion_process, abnormal); + if let Some(completion_sender) = completion_sender { + let _ = completion_sender.send(completion.into_result()); + } + }); + if thread.is_err() { + let _ = process.fail_start(BrokerError::OutOfMemory, false, true); + process.retire(true); + return Err(BrokerError::OutOfMemory); + } + drop(thread); + + self.lifecycle.wait_for_start(&process, setup_deadline) + } + + fn runner_finished(process: &BrokerProcess, abnormal: bool) { + let _ = process.fail_start(BrokerError::PeerClosed, abnormal, false); + process.retire(!abnormal); + } +} + +impl ProcessLauncher for UserlandProcessLauncher { + fn launch( + self: Arc, + process: Arc, + initial_thread_id: ThreadId, + data: ProcessStartupData, + ) -> Result<(), BrokerError> { + let config = self.started_runner_config.clone(); + self.launch_runner( + PendingRunnerAssociation::new(process, initial_thread_id, Some(data)), + config, + None, + ) + } +} + +fn broker_io_error(error: BrokerError) -> IoError { + IoError::other(error) +} diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 61a184d561..12163385cb 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -7,6 +7,10 @@ use std::ffi::{OsStr, OsString}; use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::path::PathBuf; use std::process::{Child, Command, ExitStatus}; +use std::sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, Ordering}, +}; use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; @@ -16,15 +20,21 @@ mod linux; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; +use crate::process_launcher::{PendingRunnerAssociation, UserlandProcessLauncher}; #[cfg(target_os = "linux")] use linux::PlatformRunnerEndpoint; #[cfg(all(windows, target_arch = "x86_64"))] use windows::PlatformRunnerEndpoint; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_EXIT_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); /// Configuration for starting one out-of-process runner. +/// +/// Dynamically started descendants use the same executable without the root +/// arguments and derive their role from broker startup data. +#[derive(Clone)] pub struct RunnerConfig { executable: PathBuf, arguments: Vec, @@ -63,54 +73,306 @@ impl RunnerConfig { arguments.extend(self.arguments.iter().cloned()); arguments } + + pub(crate) fn without_initial_arguments(&self) -> Self { + Self { + executable: self.executable.clone(), + arguments: Vec::new(), + proxy_url: self.proxy_url.clone(), + } + } } /// One out-of-process runner and its dedicated broker control endpoint. /// -/// Dropping an instance before [`Self::run_to_completion`] completes +/// Dropping an instance before [`Self::run_process_to_completion`] completes /// terminates and reaps the runner. -pub struct RunnerInstance { - runner: Child, +pub(crate) struct RunnerInstance { + runner: Arc>, + shutdown: Arc, endpoint: PlatformRunnerEndpoint, + setup_deadline: Instant, +} + +struct RunnerShutdown { + runner: Arc>, + state: Mutex, + changed: Condvar, + termination_dispatched: AtomicBool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RunnerShutdownState { + Active, + Terminating, + Retired, +} + +pub(crate) struct RunnerCompletion { + result: IoResult, + abnormal: bool, + runner_signal: Option, + runner_exit_code: Option, + broker_termination: bool, +} + +impl RunnerCompletion { + pub(crate) fn panicked() -> Self { + Self { + result: Err(IoError::other("runner supervision thread panicked")), + abnormal: true, + runner_signal: None, + runner_exit_code: None, + broker_termination: false, + } + } + + pub(crate) fn is_abnormal(&self, thread_panicked: bool) -> bool { + thread_panicked + || self.abnormal + || runner_signal_is_abnormal(self.runner_signal, self.broker_termination) + || runner_exit_code_is_crash(self.runner_exit_code) + } + + pub(crate) fn into_result(self) -> IoResult { + self.result + } +} + +impl RunnerShutdown { + fn shutdown(&self) { + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + loop { + match *state { + RunnerShutdownState::Active => { + *state = RunnerShutdownState::Terminating; + break; + } + RunnerShutdownState::Terminating => { + state = self + .changed + .wait(state) + .expect("runner shutdown mutex poisoned"); + } + RunnerShutdownState::Retired => return, + } + } + drop(state); + let termination_dispatched = { + // Serialize observation and termination so collecting an exit + // status can never expose a reusable PID between the check and + // the kill request. + let mut runner = self.runner.lock().expect("runner process mutex poisoned"); + match runner.try_wait() { + Ok(Some(_)) => false, + Ok(None) => runner.kill().is_ok(), + Err(_) => { + let _ = runner.kill(); + false + } + } + }; + if termination_dispatched { + self.termination_dispatched.store(true, Ordering::Release); + } + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + debug_assert_eq!(*state, RunnerShutdownState::Terminating); + *state = RunnerShutdownState::Retired; + self.changed.notify_all(); + } + + fn retire(&self) { + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + while *state == RunnerShutdownState::Terminating { + state = self + .changed + .wait(state) + .expect("runner shutdown mutex poisoned"); + } + *state = RunnerShutdownState::Retired; + self.changed.notify_all(); + } + + fn has_exited(&self) -> IoResult { + runner_has_exited(&self.runner) + } + + fn termination_was_dispatched(&self) -> bool { + self.termination_dispatched.load(Ordering::Acquire) + } + + fn wait_for_exit(&self, timeout: Duration) -> IoResult { + let deadline = Instant::now() + timeout; + loop { + if self.has_exited()? { + return Ok(true); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(false); + } + std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); + } + } } impl RunnerInstance { /// Creates the runner's dedicated control endpoint and starts the runner. - pub fn start(config: RunnerConfig) -> IoResult { + pub(crate) fn start(config: RunnerConfig) -> IoResult { + let setup_deadline = Instant::now() + SETUP_TIMEOUT; let endpoint = PlatformRunnerEndpoint::create()?; - let runner = Command::new(&config.executable) - .args(config.arguments(endpoint.control_channel())) - .spawn()?; - Ok(Self { runner, endpoint }) - } - - /// Serves the runner's broker association and waits for its host process. - /// - /// Association failure terminates the runner before it is reaped. A - /// non-successful runner exit is returned as ordinary instance data for the - /// caller to interpret. - pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { - let association_result = self.endpoint.serve(broker, &mut self.runner); + let runner = Arc::new(Mutex::new( + Command::new(&config.executable) + .args(config.arguments(endpoint.control_channel())) + .spawn()?, + )); + let shutdown = Arc::new(RunnerShutdown { + runner: Arc::clone(&runner), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }); + Ok(Self { + runner, + shutdown, + endpoint, + setup_deadline, + }) + } + + pub(crate) const fn setup_deadline(&self) -> Instant { + self.setup_deadline + } + + pub(crate) fn run_process_to_completion( + mut self, + association: PendingRunnerAssociation, + broker: BrokerCore, + launcher: Arc, + ) -> RunnerCompletion { + let process = Arc::clone(&association.process); + let shutdown = Arc::clone(&self.shutdown); + process.install_shutdown(Arc::new(move || shutdown.shutdown())); + let association_result = self.endpoint.serve( + &self.runner, + association, + self.setup_deadline, + broker, + launcher, + ); self.endpoint.close(); - if association_result.is_err() { - let _ = self.runner.kill(); + let shutdown_was_expected = process.shutdown_was_expected(); + let mut abnormal = association_result.abnormal; + let runner_exited = if !shutdown_was_expected && !association_result.abnormal { + self.shutdown + .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) + } else { + self.shutdown.has_exited() + }; + match runner_exited { + Ok(true) => {} + Ok(false) => { + if !shutdown_was_expected { + abnormal = true; + } + self.shutdown.shutdown(); + } + Err(_) => { + abnormal = true; + self.shutdown.shutdown(); + } + } + self.shutdown.retire(); + let runner_status = wait_for_runner_exit(&self.runner); + if runner_status.is_err() { + abnormal = true; + } + let runner_signal = runner_status + .as_ref() + .ok() + .copied() + .and_then(runner_exit_signal); + let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); + let result = runner_status.and_then(|runner_status| { + runner_exited.map(|_| ())?; + association_result.result?; + Ok(runner_status) + }); + RunnerCompletion { + result, + abnormal, + runner_signal, + runner_exit_code, + broker_termination: self.shutdown.termination_was_dispatched(), } - let runner_status = self.runner.wait()?; - association_result?; - Ok(runner_status) } } +/// Launches and supervises the initial runner process. +pub fn run_to_completion(config: RunnerConfig, broker: &BrokerCore) -> IoResult { + UserlandProcessLauncher::run_root(config, broker) +} + impl Drop for RunnerInstance { fn drop(&mut self) { self.endpoint.close(); - if !matches!(self.runner.try_wait(), Ok(Some(_status))) { - let _ = self.runner.kill(); - let _ = self.runner.wait(); - } + self.shutdown.shutdown(); + self.shutdown.retire(); + let _ = wait_for_runner_exit(&self.runner); } } +#[cfg(target_os = "linux")] +fn runner_exit_signal(status: ExitStatus) -> Option { + use std::os::unix::process::ExitStatusExt; + + status.signal() +} + +#[cfg(not(target_os = "linux"))] +fn runner_exit_signal(_status: ExitStatus) -> Option { + None +} + +#[cfg(target_os = "linux")] +const LINUX_SIGKILL: i32 = 9; + +#[cfg(target_os = "linux")] +const fn runner_signal_is_abnormal(signal: Option, broker_termination: bool) -> bool { + matches!(signal, Some(signal) if signal != LINUX_SIGKILL || !broker_termination) +} + +#[cfg(not(target_os = "linux"))] +const fn runner_signal_is_abnormal(_signal: Option, _broker_termination: bool) -> bool { + false +} + +#[cfg(target_os = "linux")] +const _: () = { + assert!(runner_signal_is_abnormal(Some(LINUX_SIGKILL + 1), true)); + assert!(runner_signal_is_abnormal(Some(LINUX_SIGKILL), false)); + assert!(!runner_signal_is_abnormal(Some(LINUX_SIGKILL), true)); + assert!(!runner_signal_is_abnormal(None, false)); +}; + +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_code_is_crash(exit_code: Option) -> bool { + matches!(exit_code, Some(code) if code.cast_unsigned() >= 0x8000_0000) +} + +#[cfg(not(all(windows, target_arch = "x86_64")))] +const fn runner_exit_code_is_crash(_exit_code: Option) -> bool { + false +} + +#[cfg(all(windows, target_arch = "x86_64"))] +const _: () = { + let access_violation = 0xc000_0005_u32.cast_signed(); + let breakpoint = 0x8000_0003_u32.cast_signed(); + assert!(runner_exit_code_is_crash(Some(access_violation))); + assert!(runner_exit_code_is_crash(Some(breakpoint))); +}; + fn accept_runner_channel( deadline: Instant, channel_name: &'static str, @@ -118,6 +380,12 @@ fn accept_runner_channel( mut try_accept: impl FnMut() -> IoResult, ) -> IoResult { loop { + if let Some(status) = runner_status()? { + return Err(IoError::new( + ErrorKind::BrokenPipe, + format!("runner {status} before connecting its {channel_name} channel"), + )); + } let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { return Err(IoError::new( @@ -125,12 +393,6 @@ fn accept_runner_channel( format!("timed out waiting for runner {channel_name} channel"), )); } - if let Some(status) = runner_status()? { - return Err(IoError::new( - ErrorKind::BrokenPipe, - format!("runner {status} before connecting its {channel_name} channel"), - )); - } match try_accept() { Ok(channel) => return Ok(channel), Err(error) if error.kind() == ErrorKind::WouldBlock => {} @@ -139,3 +401,118 @@ fn accept_runner_channel( std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); } } + +fn runner_has_exited(runner: &Arc>) -> IoResult { + // A pre-authentication caller stops accepting before acting on `true`; + // post-authentication callers no longer rely on PID-based authentication. + runner + .lock() + .expect("runner process mutex poisoned") + .try_wait() + .map(|status| status.is_some()) +} + +fn wait_for_runner_exit(runner: &Arc>) -> IoResult { + loop { + if let Some(status) = runner + .lock() + .expect("runner process mutex poisoned") + .try_wait()? + { + return Ok(status); + } + std::thread::sleep(ACCEPT_RETRY_DELAY); + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Condvar, Mutex, atomic::AtomicBool, mpsc}; + use std::time::Duration; + + #[cfg(target_os = "linux")] + #[test] + fn root_startup_preserves_runner_connection_error() { + use litebox_broker_core::test_support::TestBrokerCoreBuilder; + use litebox_broker_core::{ObjectRights, PolicyEngine}; + + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_host_guaranteed_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let error = super::run_to_completion( + super::RunnerConfig::new("/bin/false".into(), Vec::new()), + &broker, + ) + .unwrap_err(); + + assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe); + assert!( + error + .to_string() + .contains("runner exited before connecting its control channel"), + "unexpected root startup error: {error}" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn shutdown_can_terminate_while_runner_waits_for_exit() { + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; + use std::process::Command; + + let runner = Arc::new(Mutex::new( + Command::new("sh") + .args(["-c", "exec sleep 30"]) + .spawn() + .unwrap(), + )); + let shutdown = RunnerShutdown { + runner: Arc::clone(&runner), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }; + let waiting = Arc::clone(&runner); + let (finished, completion) = mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || { + let status = wait_for_runner_exit(&waiting).unwrap(); + finished.send(status).unwrap(); + }); + + shutdown.shutdown(); + assert!(shutdown.termination_was_dispatched()); + assert!( + !completion + .recv_timeout(Duration::from_secs(1)) + .unwrap() + .success() + ); + waiter.join().unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn shutdown_after_observed_exit_records_no_termination() { + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; + use std::process::Command; + + let runner = Arc::new(Mutex::new( + Command::new("sh").args(["-c", "exit 1"]).spawn().unwrap(), + )); + let shutdown = RunnerShutdown { + runner: Arc::clone(&runner), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }; + assert!(shutdown.wait_for_exit(Duration::from_secs(1)).unwrap()); + + shutdown.shutdown(); + shutdown.retire(); + + assert!(!shutdown.termination_was_dispatched()); + assert!(!wait_for_runner_exit(&runner).unwrap().success()); + } +} diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index c3f687fd8f..b61635b6b3 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -2,10 +2,11 @@ // Licensed under the MIT license. use std::ffi::OsStr; -use std::io::Result as IoResult; +use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::os::unix::net::UnixListener; use std::path::PathBuf; use std::process::Child; +use std::sync::{Arc, Mutex}; use std::time::Instant; use litebox_broker_core::BrokerCore; @@ -15,7 +16,10 @@ use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; -use super::{SETUP_TIMEOUT, accept_runner_channel}; +use super::{ + PendingRunnerAssociation, UserlandProcessLauncher, accept_runner_channel, runner_has_exited, +}; +use crate::runtime::{AssociationOutcome, is_peer_closed_error}; pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, @@ -42,13 +46,23 @@ impl PlatformRunnerEndpoint { self.socket_path.as_os_str() } - pub(super) fn serve(&mut self, broker: &BrokerCore, runner: &mut Child) -> IoResult<()> { - serve_runner_process( - broker, + pub(super) fn serve( + &mut self, + runner: &Arc>, + startup: PendingRunnerAssociation, + setup_deadline: Instant, + broker: BrokerCore, + launcher: Arc, + ) -> AssociationOutcome { + serve_association( self.listener .as_ref() .expect("a live runner instance must own its control listener"), runner, + startup, + setup_deadline, + broker, + launcher, ) } @@ -58,26 +72,28 @@ impl PlatformRunnerEndpoint { } } -fn serve_runner_process( - broker: &BrokerCore, +fn serve_association( control_listener: &UnixListener, - runner: &mut Child, -) -> IoResult<()> { - let setup_deadline = Instant::now() + SETUP_TIMEOUT; - let control_stream = accept_runner_channel( - setup_deadline, - "control", - || { - runner - .try_wait() - .map(|status| status.map(|status| format!("exited with {status}"))) - }, - || control_listener.accept().map(|(stream, _)| stream), - )?; - validate_peer_process(&control_stream, runner.id())?; - let control_channel = - UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - crate::runtime::serve_association( + runner: &Arc>, + startup: PendingRunnerAssociation, + setup_deadline: Instant, + broker: BrokerCore, + launcher: Arc, +) -> AssociationOutcome { + let shutdown_was_expected = startup.process.shutdown_was_expected(); + let control_channel = match accept_control_channel(control_listener, runner, setup_deadline) { + Ok(connection) => connection, + Err(error) => { + let abnormal = !(runner_has_exited(runner).unwrap_or(false) + || shutdown_was_expected && is_peer_closed_error(&error)); + return AssociationOutcome { + result: Err(error), + abnormal, + }; + } + }; + crate::runtime::serve_out_of_process_runner_association( + startup, broker, control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), @@ -88,5 +104,33 @@ fn serve_runner_process( Ok(()) }, UnixStreamHostSetupChannel::into_active, + launcher, ) } + +fn accept_control_channel( + control_listener: &UnixListener, + runner: &Arc>, + setup_deadline: Instant, +) -> IoResult { + let control_stream = accept_runner_channel( + setup_deadline, + "control", + || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), + || control_listener.accept().map(|(stream, _)| stream), + )?; + { + let mut runner = runner.lock().expect("runner process mutex poisoned"); + if runner.try_wait()?.is_some() { + return Err(IoError::new( + ErrorKind::BrokenPipe, + "runner exited before peer authentication", + )); + } + validate_peer_process(&control_stream, runner.id())?; + } + Ok(UnixStreamHostSetupChannel::from_host_guaranteed( + control_stream, + setup_deadline, + )) +} diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 62d8e47adc..360efbb203 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -5,6 +5,7 @@ use std::ffi::OsString; use std::io::Result as IoResult; use std::os::windows::io::AsRawHandle; use std::process::Child; +use std::sync::{Arc, Mutex}; use std::time::Instant; use litebox_broker_core::BrokerCore; @@ -14,7 +15,10 @@ use litebox_broker_transport_windows_userland::named_pipe::{ }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; -use super::{SETUP_TIMEOUT, accept_runner_channel}; +use super::{ + PendingRunnerAssociation, UserlandProcessLauncher, accept_runner_channel, runner_has_exited, +}; +use crate::runtime::{AssociationOutcome, is_peer_closed_error}; pub(super) struct PlatformRunnerEndpoint { pipe_name: OsString, @@ -35,13 +39,23 @@ impl PlatformRunnerEndpoint { &self.pipe_name } - pub(super) fn serve(&mut self, broker: &BrokerCore, runner: &mut Child) -> IoResult<()> { - serve_runner_process( - broker, + pub(super) fn serve( + &mut self, + runner: &Arc>, + startup: PendingRunnerAssociation, + setup_deadline: Instant, + broker: BrokerCore, + launcher: Arc, + ) -> AssociationOutcome { + serve_association( self.listener .as_mut() .expect("a live runner instance must own its control listener"), runner, + startup, + setup_deadline, + broker, + launcher, ) } @@ -50,27 +64,32 @@ impl PlatformRunnerEndpoint { } } -fn serve_runner_process( - broker: &BrokerCore, +fn serve_association( control_listener: &mut WindowsNamedPipeListener, - runner: &mut Child, -) -> IoResult<()> { - let setup_deadline = Instant::now() + SETUP_TIMEOUT; - let control_stream = accept_runner_channel( - setup_deadline, - "control", - || { - runner - .try_wait() - .map(|status| status.map(|status| format!("exited with {status}"))) - }, - || control_listener.try_accept(), - )?; - validate_client_process(&control_stream, runner.id())?; - let runner_process = runner.as_raw_handle(); - let control_channel = - WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - crate::runtime::serve_association( + runner: &Arc>, + startup: PendingRunnerAssociation, + setup_deadline: Instant, + broker: BrokerCore, + launcher: Arc, +) -> AssociationOutcome { + let shutdown_was_expected = startup.process.shutdown_was_expected(); + let control_channel = match accept_control_channel(control_listener, runner, setup_deadline) { + Ok(connection) => connection, + Err(error) => { + let abnormal = !(runner_has_exited(runner).unwrap_or(false) + || shutdown_was_expected && is_peer_closed_error(&error)); + return AssociationOutcome { + result: Err(error), + abnormal, + }; + } + }; + let runner_process = runner + .lock() + .expect("runner process mutex poisoned") + .as_raw_handle(); + crate::runtime::serve_out_of_process_runner_association( + startup, broker, control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), @@ -80,9 +99,29 @@ fn serve_runner_process( channel.send_shared_memory(control_memory, runner_process) }, WindowsNamedPipeHostSetupChannel::into_active, + launcher, ) } +fn accept_control_channel( + control_listener: &mut WindowsNamedPipeListener, + runner: &Arc>, + setup_deadline: Instant, +) -> IoResult { + let control_stream = accept_runner_channel( + setup_deadline, + "control", + || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), + || control_listener.try_accept(), + )?; + let runner_id = runner.lock().expect("runner process mutex poisoned").id(); + validate_client_process(&control_stream, runner_id)?; + Ok(WindowsNamedPipeHostSetupChannel::from_host_guaranteed( + control_stream, + setup_deadline, + )) +} + fn unique_control_pipe_name() -> OsString { let process_id = std::process::id(); let nonce = std::time::SystemTime::now() diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 554e968908..e6c91ff9ad 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -14,8 +14,8 @@ //! transport. //! //! Concurrency and worker sizing are deliberately not part of the public -//! surface: [`serve_association`] is the only entry point, and it owns how -//! many workers dispatch requests and how deeply the request queue may buffer. +//! surface. Both association entry points delegate to one internal runtime +//! that owns worker counts and request queue capacity. use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::sync::mpsc::{Receiver, SyncSender, TrySendError, sync_channel}; @@ -27,7 +27,8 @@ use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; use litebox_broker_host::{ - BrokerHostAssociation, BrokerHostError, ConnectionTermination, setup_connection, + BrokerHostAssociation, BrokerHostError, ConnectionTermination, handle_process_operation, + setup_connection, }; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::BrokerRequest; @@ -39,13 +40,29 @@ use litebox_broker_transport::channel::{ use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; +use crate::process_launcher::{PendingRunnerAssociation, UserlandProcessLauncher}; use crate::readiness::ReadinessPublisherRuntime; const REQUEST_QUEUE_CAPACITY: usize = 64; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); const REQUEST_QUEUE_STALL_TIMEOUT: Duration = Duration::from_secs(5); -/// Serves one broker association from setup through teardown. +pub(crate) struct AssociationOutcome { + pub(crate) result: IoResult<()>, + pub(crate) abnormal: bool, +} + +pub(crate) fn is_peer_closed_error(error: &IoError) -> bool { + matches!( + error.kind(), + ErrorKind::BrokenPipe + | ErrorKind::UnexpectedEof + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + ) +} + +/// Serves an association for the development-only in-process runner. /// /// `control_channel` must already be configured with whatever deadline and /// peer authentication its transport requires; this function only negotiates @@ -58,7 +75,7 @@ const REQUEST_QUEUE_STALL_TIMEOUT: Duration = Duration::from_secs(5); /// Returns once the association ends, whether by a clean peer close or a /// failure. Layout mismatches and broker setup failures are mapped to a /// precise [`std::io::Error`] rather than left as an opaque boxed error. -pub fn serve_association< +pub fn serve_in_process_runner_association< Memory, SetupChannel, RequestSource, @@ -67,7 +84,7 @@ pub fn serve_association< Shutdown, >( broker: &BrokerCore, - mut control_channel: SetupChannel, + control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, create_control_memory: impl FnOnce() -> IoResult, send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, @@ -82,20 +99,133 @@ where RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, +{ + serve_association_inner( + broker, + control_channel, + create_shared_memory, + create_control_memory, + send_shared_memory, + activate, + None, + None, + ) + .and_then(|outcome| outcome.result) +} + +/// Serves an initial or parent-started association for an out-of-process runner. +/// +/// Unlike the in-process path, this reports process ownership and failure +/// details to `RunnerInstance`, which owns runner termination and cleanup. +#[allow(clippy::too_many_arguments)] +pub(crate) fn serve_out_of_process_runner_association< + Memory, + SetupChannel, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, +>( + startup: PendingRunnerAssociation, + broker: BrokerCore, + control_channel: SetupChannel, + create_shared_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce() -> IoResult, + send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, + activate: impl FnOnce( + SetupChannel, + ControlRing, + ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + launcher: Arc, +) -> AssociationOutcome +where + Memory: ControlRingMemory, + SetupChannel: HostSetupChannel, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { + let outcome = serve_association_inner( + &broker, + control_channel, + create_shared_memory, + create_control_memory, + send_shared_memory, + activate, + Some(launcher), + Some(startup), + ); + let mut outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => AssociationOutcome { + result: Err(error), + abnormal: false, + }, + }; + let result_is_abnormal = outcome + .result + .as_ref() + .is_err_and(|error| !is_peer_closed_error(error)); + outcome.abnormal |= result_is_abnormal; + outcome +} + +#[allow(clippy::too_many_arguments)] +fn serve_association_inner< + Memory, + SetupChannel, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, +>( + broker: &BrokerCore, + mut control_channel: SetupChannel, + create_shared_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce() -> IoResult, + send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, + activate: impl FnOnce( + SetupChannel, + ControlRing, + ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + launcher: Option>, + startup: Option, +) -> IoResult +where + Memory: ControlRingMemory, + SetupChannel: HostSetupChannel, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, +{ + let (process, startup) = match startup { + Some(startup) => { + let (process, data) = startup.into_process_and_startup(); + (Some(process), data) + } + None => (None, None), + }; + let finish_process = process.is_none(); let shared_memory = create_shared_memory()?; - let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) - .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?; + let shared_buffers = Arc::new( + SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) + .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?, + ); let control_memory = create_control_memory()?; let control_ring = ControlRing::new(control_memory) .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; let readiness = Arc::new(ReadinessPublisherRuntime::new()); let association = match setup_connection( broker, + process, + startup, &mut control_channel, - &shared_buffers, + Arc::clone(&shared_buffers), readiness.clone(), + |_| false, |channel| send_shared_memory(channel, shared_buffers.memory(), control_ring.memory()), ) .map_err(map_host_error)? @@ -124,18 +254,23 @@ where match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { - association.finish(); + if finish_process { + association.finish(); + } return Err(error); } }; - dispatch_requests( + Ok(dispatch_requests( + broker.clone(), association, readiness, request_source, response_sink, notification_channel, shutdown, - ) + launcher, + finish_process, + )) } /// Maps a host setup or request failure to a precise [`std::io::Error`]. @@ -165,6 +300,9 @@ fn map_host_error(error: BrokerHostError) -> IoError { BrokerHostError::SharedBufferLayoutMismatch => { IoError::new(ErrorKind::InvalidData, description) } + BrokerHostError::AssociationFailed => { + IoError::new(ErrorKind::ConnectionAborted, description) + } _ => IoError::other(description), } } @@ -178,6 +316,7 @@ struct HostAssociationFailureCoordinator { failed: AtomicBool, /// Whether an association worker panicked, requiring teardown without ID reuse. panicked: AtomicBool, + abnormal: AtomicBool, error: Mutex>, shutdown: Shutdown, } @@ -189,6 +328,7 @@ impl> Self { failed: AtomicBool::new(false), panicked: AtomicBool::new(false), + abnormal: AtomicBool::new(false), error: Mutex::new(None), shutdown, } @@ -199,6 +339,9 @@ impl> } fn report(&self, error: IoError) { + if !is_peer_closed_error(&error) { + self.abnormal.store(true, Ordering::Release); + } if self.failed.swap(true, Ordering::AcqRel) { return; } @@ -218,6 +361,10 @@ impl> self.panicked.load(Ordering::Acquire) } + fn abnormal(&self) -> bool { + self.abnormal.load(Ordering::Acquire) + } + /// Ends the association transport without recording a failure. /// /// Teardown uses this to release blocked endpoints without turning a @@ -281,11 +428,11 @@ impl> Drop } /// Requests cancellation before an association scope joins its workers. -struct AssociationCancellationGuard<'association, 'memory, Memory: SharedMemory> { - association: &'association BrokerHostAssociation<'memory, Memory>, +struct AssociationCancellationGuard<'association, Memory: SharedMemory> { + association: &'association BrokerHostAssociation, } -impl Drop for AssociationCancellationGuard<'_, '_, Memory> { +impl Drop for AssociationCancellationGuard<'_, Memory> { fn drop(&mut self) { self.association.request_cancellation(); } @@ -296,23 +443,35 @@ impl Drop for AssociationCancellationGuard<'_, '_, Memory> /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network /// reactor is currently its production source. +#[allow(clippy::too_many_arguments)] fn dispatch_requests( - association: BrokerHostAssociation<'_, Memory>, + broker: BrokerCore, + association: BrokerHostAssociation, readiness: Arc, mut request_source: RequestSource, response_sink: ResponseSink, mut notification_channel: NotificationChannel, shutdown: Shutdown, -) -> IoResult<()> + launcher: Option>, + finish_process: bool, +) -> AssociationOutcome where Memory: SharedMemory, RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { let association = Arc::new(association); let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(shutdown)); + if let Err(error) = association.activate_process() { + return AssociationOutcome { + result: Err(IoError::other(format!( + "failed to activate broker process association: {error}" + ))), + abnormal: true, + }; + } let (request_sender, request_receiver) = sync_channel(REQUEST_QUEUE_CAPACITY); let request_receiver = Arc::new(Mutex::new(request_receiver)); @@ -359,14 +518,18 @@ where let request_receiver = Arc::clone(&request_receiver); let response_sink = response_sink.clone(); let worker_failure_coordinator = Arc::clone(&failure_coordinator); + let launcher_for_worker = launcher.clone(); + let broker_for_worker = broker.clone(); match std::thread::Builder::new() .name(format!("litebox-broker-worker-{worker_id}")) .spawn_scoped(scope, move || { run_worker( + &broker_for_worker, &association, &request_receiver, &response_sink, &worker_failure_coordinator, + launcher_for_worker.as_ref(), ); }) { Ok(worker) => workers.push(worker), @@ -379,6 +542,7 @@ where read_requests(&mut request_source, request_sender, &failure_coordinator); drop(cancellation); + association.association_ending(); for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -407,12 +571,14 @@ where let Ok(association) = Arc::try_unwrap(association) else { panic!("all broker association workers must be joined before teardown"); }; - if failure_coordinator.panicked() { + let panicked = failure_coordinator.panicked(); + let abnormal = failure_coordinator.abnormal(); + if panicked || !finish_process { drop(association); } else { association.finish(); } - result + AssociationOutcome { result, abnormal } } fn read_requests( @@ -493,10 +659,12 @@ where } fn run_worker( - association: &BrokerHostAssociation<'_, Memory>, + broker: &BrokerCore, + association: &BrokerHostAssociation, request_receiver: &Mutex>, response_sink: &ResponseSink, failure_coordinator: &HostAssociationFailureCoordinator, + launcher: Option<&Arc>, ) where Memory: SharedMemory, ResponseSink: HostResponseSink, @@ -514,9 +682,23 @@ fn run_worker( continue; } match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - association.execute_request(request, |response| response_sink.send_response(response)) + association.execute_request_with( + request, + |process, operation, shared_buffers| { + launcher.and_then(|launcher| { + handle_process_operation( + broker, + launcher, + process, + operation, + shared_buffers, + ) + }) + }, + |response| response_sink.send_response(response), + ) })) { - Ok(Ok(())) => {} + Ok(Ok(()) | Err(BrokerHostError::AssociationFailed)) => {} Ok(Err(error)) => failure_coordinator.report(map_host_error(error)), Err(_) => { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -624,6 +806,8 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), + startup: None, }) .unwrap(); local_setup.recv_handshake_response().unwrap().unwrap(); @@ -666,27 +850,28 @@ mod tests { UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, ) { - let (local, (notifications, shutdown)) = litebox_broker_local::BrokerLocal::negotiate( - UnixStreamLocalSetupChannel::from_connected(stream), - |mut setup| { - let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - IoError::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; - let (call_channel, notifications, shutdown) = - setup.into_active(control_ring, || {})?; - Ok(( - call_channel, - Arc::new(shared_memory), - (notifications, shutdown), - )) - }, - ) - .unwrap(); + let (local, _startup, (notifications, shutdown)) = + litebox_broker_local::BrokerLocal::negotiate( + UnixStreamLocalSetupChannel::from_connected(stream), + |mut setup| { + let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; + let control_memory = setup.receive_control_ring(None)?; + let control_ring = ControlRing::new(control_memory).map_err(|error| { + IoError::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; + let (call_channel, notifications, shutdown) = + setup.into_active(control_ring, || {})?; + Ok(( + call_channel, + Arc::new(shared_memory), + (notifications, shutdown), + )) + }, + ) + .unwrap(); (local, notifications, shutdown) } @@ -719,7 +904,7 @@ mod tests { .unwrap(); let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let shared_buffers = - SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap(); + Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); let control_ring = ControlRing::new(control_memory).unwrap(); let mut control = UnixStreamHostSetupChannel::from_host_guaranteed( @@ -728,9 +913,12 @@ mod tests { ); let association = setup_connection( &broker, + None, + None, &mut control, - &shared_buffers, + Arc::clone(&shared_buffers), readiness.clone(), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), None)?; channel.send_memfd(control_ring.memory(), None) @@ -742,14 +930,20 @@ mod tests { control.into_active(control_ring).unwrap(); started.recv().unwrap(); outcome_sender - .send(dispatch_requests( - association, - readiness, - request_source, - response_sink, - notifications, - shutdown, - )) + .send( + dispatch_requests( + broker.clone(), + association, + readiness, + request_source, + response_sink, + notifications, + shutdown, + None, + true, + ) + .result, + ) .unwrap(); }); let (local, notifications, shutdown) = negotiate_local(local_stream); @@ -948,6 +1142,24 @@ mod tests { assert_eq!(error.to_string(), "first failure"); } + #[test] + fn later_protocol_failure_upgrades_abnormal_disposition() { + let association = live_association(); + let failure_coordinator = HostAssociationFailureCoordinator::new(association.shutdown); + + failure_coordinator.report(IoError::new( + ErrorKind::ConnectionAborted, + "process shutdown", + )); + failure_coordinator.report(IoError::new(ErrorKind::InvalidData, "protocol failure")); + + assert!(failure_coordinator.abnormal()); + assert_eq!( + failure_coordinator.take_error().unwrap().kind(), + ErrorKind::ConnectionAborted + ); + } + #[test] fn a_stalled_request_queue_fails_after_its_deadline() { let request = |request_id| BrokerRequest { diff --git a/litebox_broker_userland/src/windows.rs b/litebox_broker_userland/src/windows.rs index 529297faa3..d05f499eb7 100644 --- a/litebox_broker_userland/src/windows.rs +++ b/litebox_broker_userland/src/windows.rs @@ -98,7 +98,7 @@ fn serve_control_stream( ) -> IoResult<()> { let control_channel = WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - litebox_broker_userland::runtime::serve_association( + litebox_broker_userland::runtime::serve_in_process_runner_association( broker, control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), diff --git a/litebox_broker_userland/tests/notification_runtime.rs b/litebox_broker_userland/tests/notification_runtime.rs index 34122571d3..e1f49edb41 100644 --- a/litebox_broker_userland/tests/notification_runtime.rs +++ b/litebox_broker_userland/tests/notification_runtime.rs @@ -53,15 +53,19 @@ fn spawn_host( .build() .unwrap(); let shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); - let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap(); + let shared_buffers = + Arc::new(SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); let control_memory = MemfdSharedMemory::create_control_ring().unwrap(); let control_ring = ControlRing::new(control_memory).unwrap(); let mut control = UnixStreamHostSetupChannel::from_accepted(stream); let association = setup_connection( &broker, + None, + None, &mut control, - &shared_buffers, + Arc::clone(&shared_buffers), Arc::new(ReadinessPublisherRuntime::new()), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), None)?; channel.send_memfd(control_ring.memory(), None) @@ -91,7 +95,7 @@ fn negotiate_local( BrokerLocal, BrokerNotifications, ) { - let (local, notifications) = BrokerLocal::negotiate( + let (local, _startup, notifications) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; @@ -137,7 +141,7 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { let (local_control, host_control) = UnixStream::pair().unwrap(); let host_shared_memory = MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE).unwrap(); let host_shared_buffers = - SharedBufferPool::new(host_shared_memory, SHARED_BUFFER_LAYOUT).unwrap(); + Arc::new(SharedBufferPool::new(host_shared_memory, SHARED_BUFFER_LAYOUT).unwrap()); let host_control_memory = MemfdSharedMemory::create_control_ring().unwrap(); let host_control_ring = ControlRing::new(host_control_memory).unwrap(); let notification = BrokerNotification::Readiness(ReadinessNotification { @@ -150,9 +154,12 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { let mut control = UnixStreamHostSetupChannel::from_accepted(host_control); let association = setup_connection( &broker, + None, + None, &mut control, - &host_shared_buffers, + Arc::clone(&host_shared_buffers), Arc::new(ReadinessPublisherRuntime::new()), + |_| false, |channel| { channel.send_memfd(host_shared_buffers.memory(), None)?; channel.send_memfd(host_control_ring.memory(), None) @@ -176,7 +183,7 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { } }); - let (local, notification_channel) = BrokerLocal::negotiate( + let (local, _startup, notification_channel) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(local_control), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 3c4a438367..78a61d8bf7 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -2,23 +2,32 @@ // Licensed under the MIT license. use std::ffi::{OsStr, OsString}; -use std::io::{ErrorKind, Result}; +use std::io::{ErrorKind, Result, Write as _}; +use std::os::unix::ffi::OsStrExt as _; use std::path::Path; use std::process::{Child, Command}; use std::sync::Arc; use std::time::{Duration, Instant}; use litebox_broker_local::BrokerLocal; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartupData, +}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{ SHARED_BUFFER_POOL_SIZE, SharedBufferSequence, SharedBufferSlotIndex, }; use litebox_broker_protocol::socket::{ReceiveFromFlags, SendFlags, SocketConnectionStatus}; use litebox_broker_transport::control_ring::ControlRing; -use litebox_broker_transport_linux_userland::unix_socket::UnixStreamLocalSetupChannel; +use litebox_broker_transport_linux_userland::unix_socket::{ + UnixControlRingLocalCallChannel, UnixStreamLocalSetupChannel, +}; const RUNNER_ARGUMENT: &str = "broker-userland-test-runner"; const NETWORK_RUNNER_ARGUMENT: &str = "broker-userland-network-test-runner"; +const CHILD_START_RUNNER_ARGUMENT: &str = "broker-userland-child-start-runner"; +const TEST_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x7465_7374); +const FAILING_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x6661_696c); const BROKER_PROCESS_TIMEOUT: Duration = Duration::from_secs(30); fn main() { @@ -50,6 +59,19 @@ fn run_parent_test() { .arg(RUNNER_ARGUMENT); wait_for_broker(event_command); + let child_marker = unique_child_marker_path(); + let mut child_command = Command::new(env!("CARGO_BIN_EXE_litebox-broker-userland")); + child_command + .arg("--runner") + .arg(&test_executable) + .arg(CHILD_START_RUNNER_ARGUMENT) + .arg(&child_marker); + wait_for_broker(child_command); + let child_result = std::fs::read_to_string(&child_marker).unwrap(); + assert!(child_result.starts_with("ready:")); + assert!(child_result.ends_with("\nstarted\nfinished\n")); + std::fs::remove_file(child_marker).unwrap(); + let gateway = std::net::Ipv4Addr::new(10, 0, 2, 1); let tcp_listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); let tcp_port = tcp_listener.local_addr().unwrap().port(); @@ -110,7 +132,7 @@ fn run_fake_runner(args: &[OsString]) { let control_socket_path = args.get(2).unwrap(); let setup_channel = connect_control_with_retry(Path::new(control_socket_path)).unwrap(); - let (local, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { + let (local, startup, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd( SHARED_BUFFER_POOL_SIZE, Some(Instant::now() + Duration::from_secs(5)), @@ -127,6 +149,10 @@ fn run_fake_runner(args: &[OsString]) { Ok((call_channel, Arc::new(shared_memory), ())) }) .unwrap(); + if let Some(bootstrap) = startup { + run_fake_child(local, bootstrap); + return; + } let local = Arc::new(local); if args.get(3).and_then(|argument| argument.to_str()) == Some(NETWORK_RUNNER_ARGUMENT) { @@ -203,6 +229,46 @@ fn run_fake_runner(args: &[OsString]) { local.close_object(handle).unwrap(); return; } + if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_START_RUNNER_ARGUMENT) { + assert_eq!(args.len(), 5, "unexpected runner arguments: {args:?}"); + let marker = Path::new(&args[4]); + let bootstrap = marker.as_os_str().as_encoded_bytes(); + let inherited_event = local.create_event_with_count(1).unwrap(); + let inherited_objects = InheritedProcessObjects::new(&[inherited_event]).unwrap(); + let failed = local + .start_child_process( + FAILING_BOOTSTRAP_FORMAT, + ProcessBootstrapVersion(1), + SharedBufferSequence::new( + &[SharedBufferSlotIndex(0)], + bootstrap.len().try_into().unwrap(), + ) + .unwrap(), + bootstrap, + inherited_objects, + ) + .unwrap(); + assert_ne!(failed.process_id.0, failed.initial_thread_id.0); + let started = local + .start_child_process( + TEST_BOOTSTRAP_FORMAT, + ProcessBootstrapVersion(1), + SharedBufferSequence::new( + &[SharedBufferSlotIndex(0)], + bootstrap.len().try_into().unwrap(), + ) + .unwrap(), + bootstrap, + inherited_objects, + ) + .unwrap(); + assert_ne!(started.process_id.0, started.initial_thread_id.0); + wait_for_marker( + marker, + &format!("ready:{}\nstarted\nfinished\n", started.process_id.0), + ); + return; + } assert_eq!( args.get(3).map(OsString::as_os_str), Some(OsStr::new(RUNNER_ARGUMENT)) @@ -270,6 +336,65 @@ fn run_fake_runner(args: &[OsString]) { drop(local); } +fn run_fake_child( + local: BrokerLocal, + bootstrap: ProcessStartupData, +) { + if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { + return; + } + assert_eq!(bootstrap.format, TEST_BOOTSTRAP_FORMAT); + assert_eq!(bootstrap.version, ProcessBootstrapVersion(1)); + let marker = Path::new(OsStr::from_bytes(&bootstrap.payload)); + let inherited_objects = bootstrap.inherited_objects.as_slice(); + assert_eq!(inherited_objects.len(), 1); + assert_eq!( + local.check_readiness(inherited_objects[0]).unwrap(), + ReadinessFlags::READ | ReadinessFlags::WRITE + ); + std::fs::write(marker, format!("ready:{}\n", local.process_id().0)).unwrap(); + std::fs::OpenOptions::new() + .append(true) + .open(marker) + .unwrap() + .write_all(b"started\n") + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + std::fs::OpenOptions::new() + .append(true) + .open(marker) + .unwrap() + .write_all(b"finished\n") + .unwrap(); +} + +fn wait_for_marker(path: &Path, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match std::fs::read_to_string(path) { + Ok(contents) if contents == expected => return, + Ok(_) | Err(_) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + result => panic!( + "timed out waiting for child marker {}: {result:?}", + path.display() + ), + } + } +} + +fn unique_child_marker_path() -> std::path::PathBuf { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "litebox-child-start-{}-{nonce}", + std::process::id() + )) +} + struct ChildGuard { child: Child, } diff --git a/litebox_runner_linux_on_windows_userland/src/lib.rs b/litebox_runner_linux_on_windows_userland/src/lib.rs index 19ad8cecd4..1561f973d1 100644 --- a/litebox_runner_linux_on_windows_userland/src/lib.rs +++ b/litebox_runner_linux_on_windows_userland/src/lib.rs @@ -68,13 +68,21 @@ pub fn run(cli_args: CliArgs) -> Result<()> { .broker_control_channel .as_deref() .context("file operations require --broker-control-channel")?; + let (connection, startup) = broker::connect(control_pipe)?; + if let Some(bootstrap) = startup { + anyhow::bail!( + "unsupported child Linux process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + ); + } let broker::BrokerConnection { local, notifications, - } = broker::connect(control_pipe)?; - let process_id = - i32::try_from(local.process_id().0).context("process ID does not fit Linux pid_t")?; - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); + } = connection; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = i32::try_from(process_id.0).context("process ID does not fit Linux pid_t")?; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), @@ -118,6 +126,7 @@ pub fn run(cli_args: CliArgs) -> Result<()> { euid: 1000, egid: 1000, }, + initial_thread, prog_path, argv, envp, diff --git a/litebox_runner_linux_userland/Cargo.toml b/litebox_runner_linux_userland/Cargo.toml index 9f3696370e..8187495238 100644 --- a/litebox_runner_linux_userland/Cargo.toml +++ b/litebox_runner_linux_userland/Cargo.toml @@ -9,6 +9,7 @@ clap = { version = "4.5.33", features = ["derive"] } libc = { version = "0.2.169", default-features = false } litebox = { version = "0.1.0", path = "../litebox" } litebox_broker_local_userland = { version = "0.1.0", path = "../litebox_broker_local_userland" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_transport = { version = "0.1.0", path = "../litebox_broker_transport" } litebox_broker_transport_linux_userland = { version = "0.1.0", path = "../litebox_broker_transport_linux_userland" } litebox_common_linux = { version = "0.1.0", path = "../litebox_common_linux" } @@ -24,7 +25,6 @@ glob = "0.3" litebox_broker_core = { version = "0.1.0", path = "../litebox_broker_core", features = ["test-support"] } litebox_broker_host = { version = "0.1.0", path = "../litebox_broker_host" } litebox_broker_platform_linux_userland = { version = "0.1.0", path = "../litebox_broker_platform_linux_userland" } -litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_userland = { version = "0.1.0", path = "../litebox_broker_userland" } [features] diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 1f23069f79..7a0f120032 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -30,7 +30,7 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/usr/bin/python3 --version`). /// /// The program path must be absolute and refer to a file in the broker-owned file system. - #[arg(required = true, trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] + #[arg(trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times) #[arg(long = "env")] @@ -86,7 +86,26 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - let prog_path = &cli_args.program_and_arguments[0]; + let control_socket_path = cli_args + .broker_control_channel + .as_deref() + .context("file operations require --broker-control-channel")?; + let (connection, startup) = + litebox_platform_linux_userland::with_guest_signals_blocked(|| { + broker::connect(control_socket_path) + })?; + if let Some(bootstrap) = startup { + return Err(anyhow!( + "unsupported child Linux process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } + + let prog_path = cli_args + .program_and_arguments + .first() + .context("program path missing")?; if !prog_path.starts_with('/') { anyhow::bail!("program path must be absolute (e.g., /usr/bin/ls), got: {prog_path}"); } @@ -96,24 +115,18 @@ pub fn run(cli_args: CliArgs) -> Result { let mut broker_positional_io_fds = Vec::new(); let mut broker_shutdown_fds = Vec::new(); - let control_socket_path = cli_args - .broker_control_channel - .as_deref() - .context("file operations require --broker-control-channel")?; let broker::BrokerConnection { local: broker_local, notifications: broker_notifications, coordinator: broker_association_coordinator, positional_io_fds, shutdown_fd, - } = litebox_platform_linux_userland::with_guest_signals_blocked(|| { - broker::connect(control_socket_path) - })?; - let process_id = i32::try_from(broker_local.process_id().0) - .context("process ID does not fit Linux pid_t")?; + } = connection; broker_positional_io_fds.extend(positional_io_fds); broker_shutdown_fds.push(shutdown_fd); - let litebox = litebox::LiteBox::new_with_broker_local(platform, broker_local); + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, broker_local); + let process_id = i32::try_from(process_id.0).context("process ID does not fit Linux pid_t")?; broker_association_coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( @@ -156,7 +169,7 @@ pub fn run(cli_args: CliArgs) -> Result { &broker_shutdown_fds, ); - let program = shim.load_program(task_params, prog_path, argv, envp)?; + let program = shim.load_program(task_params, initial_thread, prog_path, argv, envp)?; #[cfg(feature = "lock_tracing")] litebox::sync::start_recording(); @@ -222,6 +235,19 @@ mod tests { ); } + #[test] + fn child_runner_cli_does_not_require_root_program() { + let args = CliArgs::try_parse_from([ + "runner", + "--unstable", + "--broker-control-channel", + "/tmp/broker.sock", + ]) + .unwrap(); + + assert!(args.program_and_arguments.is_empty()); + } + #[test] fn broker_proxy_replaces_proxy_environment() { let mut environment = vec![ diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index 928b77b275..439ee801cc 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -501,11 +501,13 @@ fn run_test_broker_connection( litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE, ) .expect("failed to create broker test shared memory"); - let shared_buffers = litebox_broker_transport::shared_memory::SharedBufferPool::new( - shared_memory, - litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT, - ) - .expect("failed to attach broker test shared-buffer layout"); + let shared_buffers = std::sync::Arc::new( + litebox_broker_transport::shared_memory::SharedBufferPool::new( + shared_memory, + litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT, + ) + .expect("failed to attach broker test shared-buffer layout"), + ); let control_memory = litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory::create_control_ring() .expect("failed to create broker test control ring"); @@ -521,9 +523,12 @@ fn run_test_broker_connection( std::sync::Arc::new(litebox_broker_userland::readiness::ReadinessPublisherRuntime::new()); let association = litebox_broker_host::setup_connection( broker, + None, + None, &mut channel, - &shared_buffers, + std::sync::Arc::clone(&shared_buffers), readiness.clone(), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), Some(setup_deadline))?; channel.send_memfd(control_ring.memory(), Some(setup_deadline)) diff --git a/litebox_runner_macos_userland/src/test_broker.rs b/litebox_runner_macos_userland/src/test_broker.rs index 4e700c9f36..76712d0fee 100644 --- a/litebox_runner_macos_userland/src/test_broker.rs +++ b/litebox_runner_macos_userland/src/test_broker.rs @@ -69,7 +69,7 @@ pub(crate) fn setup( .map_err(|error| anyhow!("test broker: {error:?}"))?; let setup = InProcessBrokerSetup::new(core); let readiness = setup.readiness_sink(); - let (local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_runner_optee_on_linux_userland/src/lib.rs b/litebox_runner_optee_on_linux_userland/src/lib.rs index a917b06fe5..1d75ab40ee 100644 --- a/litebox_runner_optee_on_linux_userland/src/lib.rs +++ b/litebox_runner_optee_on_linux_userland/src/lib.rs @@ -95,12 +95,20 @@ pub fn run(cli_args: CliArgs) -> Result<()> { // Leaked because the shim requires a `'static` session manager. let session_manager: &'static SessionManager = Box::leak(Box::new(SessionManager::new())); + let (connection, startup) = broker::connect(&cli_args.broker_control_channel)?; + if let Some(bootstrap) = startup { + return Err(anyhow::anyhow!( + "unsupported child OP-TEE process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } let broker::BrokerConnection { local, notifications, coordinator, .. - } = broker::connect(&cli_args.broker_control_channel)?; + } = connection; let litebox = litebox::LiteBox::new_with_broker_local(platform, local); coordinator.install_dispatch(litebox.broker_failure_dispatcher()); broker::start_notification_receiver( diff --git a/litebox_runner_windows_on_linux_userland/src/lib.rs b/litebox_runner_windows_on_linux_userland/src/lib.rs index 612f14f323..3c51818765 100644 --- a/litebox_runner_windows_on_linux_userland/src/lib.rs +++ b/litebox_runner_windows_on_linux_userland/src/lib.rs @@ -80,18 +80,27 @@ pub fn run(cli_args: CliArgs) -> Result<()> { .broker_control_channel .as_deref() .context("file operations require --broker-control-channel")?; + let (connection, startup) = + litebox_platform_linux_userland::with_guest_signals_blocked(|| { + broker::connect(control_socket) + })?; + if let Some(bootstrap) = startup { + return Err(anyhow::anyhow!( + "unsupported child Windows process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } let broker::BrokerConnection { local, notifications, coordinator, positional_io_fds: _broker_positional_io_fds, shutdown_fd: _broker_shutdown_fd, - } = litebox_platform_linux_userland::with_guest_signals_blocked(|| { - broker::connect(control_socket) - })?; - let process_id = local.process_id().0 as usize; - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.create_thread()?; + } = connection; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = process_id.0 as usize; coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 40f4d5d191..e9c72b762a 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -20,7 +20,7 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/app/program.exe --help`). /// /// The program path refers to a path inside the broker-owned file system. - #[arg(required = true, trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] + #[arg(trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times). #[arg(long = "env")] @@ -54,19 +54,28 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - let platform = WindowsUserland::new(); - WindowsUserland::set_guest_tls_mode(GuestTlsMode::Windows); let control_pipe = cli_args .broker_control_channel .as_deref() .context("file operations require --broker-control-channel")?; + let (connection, startup) = broker::connect(control_pipe)?; + if let Some(bootstrap) = startup { + anyhow::bail!( + "unsupported child Windows process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + ); + } + + let platform = WindowsUserland::new(); + WindowsUserland::set_guest_tls_mode(GuestTlsMode::Windows); let broker::BrokerConnection { local, notifications, - } = broker::connect(control_pipe)?; - let process_id = local.process_id().0 as usize; - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.create_thread()?; + } = connection; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = process_id.0 as usize; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), @@ -83,7 +92,7 @@ pub fn run(cli_args: CliArgs) -> Result { let (program_path, program_args) = cli_args .program_and_arguments .split_first() - .context("program path missing — clap should have required at least one argument")?; + .context("program path missing")?; let shim = shim_builder.build(); let argv = std::iter::once(program_path.as_str()) diff --git a/litebox_shim_linux/src/lib.rs b/litebox_shim_linux/src/lib.rs index 5534b60331..5b6417294b 100644 --- a/litebox_shim_linux/src/lib.rs +++ b/litebox_shim_linux/src/lib.rs @@ -258,11 +258,11 @@ impl Clone for LinuxShim { } impl LinuxShim { - /// Loads the program at `path` as the shim's initial task, returning the - /// initial register state. + /// Loads a program using the initial thread allocated during broker negotiation. pub fn load_program( &self, task: litebox_common_linux::TaskParams, + initial_thread: litebox::thread::Thread, path: &str, argv: Vec, envp: Vec, @@ -295,7 +295,7 @@ impl LinuxShim { _not_send: core::marker::PhantomData, task: Task { global: self.0.clone(), - litebox_thread: Cell::new(None), + litebox_thread: Cell::new(Some(initial_thread)), thread: syscalls::process::ThreadState::new_process(pid), wait_state: wait::WaitState::new(self.0.platform), pid, diff --git a/litebox_shim_linux/src/syscalls/process.rs b/litebox_shim_linux/src/syscalls/process.rs index 3fd082bac6..86fbef66f7 100644 --- a/litebox_shim_linux/src/syscalls/process.rs +++ b/litebox_shim_linux/src/syscalls/process.rs @@ -387,14 +387,12 @@ impl Task { } /// Transfers a surviving nonleader exec caller to the process leader - /// identity and retires its former broker thread ID. + /// identity while retaining its broker thread ownership. fn rebind_exec_identity(&self) { let Some(old_tid) = self.thread.rebind_for_exec(self.pid) else { return; }; - // Publish the leader identity locally before releasing the old broker - // ID so no local process state can refer to a reusable thread ID. let thread = self .litebox_thread .take() @@ -404,14 +402,7 @@ impl Task { old_tid, "LiteBox thread ownership must match the old Linux thread ID" ); - let thread_id = thread.id(); - if let Err(error) = thread.exit() { - litebox_util_log::error!( - error:% = error, - thread_id; - "failed to record thread exit during exec identity rebinding" - ); - } + self.litebox_thread.set(Some(thread)); } /// Returns true if the task is exiting and should not continue running @@ -1842,7 +1833,7 @@ mod tests { extern crate std; #[test] - fn nonleader_exec_rebinds_identity_and_releases_broker_thread() { + fn nonleader_exec_rebinds_identity_and_retains_broker_thread() { use litebox::thread::CreateError; use litebox_broker_core::BrokerCoreLimits; @@ -1851,14 +1842,14 @@ mod tests { super::super::test_broker::MAX_TEST_BROKER_REFERENCES, BrokerCoreLimits::DEFAULT.max_total_pipe_capacity, ) - .with_thread_quotas(1, 1); + .with_thread_quotas(2, 2); let (litebox, process_id) = crate::syscalls::test_broker::litebox_with_limits(platform, limits); let shim_builder = crate::LinuxShimBuilder::new_with_litebox(platform, litebox, process_id); let leader = shim_builder.build().0.new_test_task(); let task = leader .clone_for_test() - .expect("the sole broker thread slot must be available"); + .expect("the thread slot after the negotiated initial thread must be available"); let old_tid = task.sys_gettid(); let process = task.process().clone(); @@ -1880,10 +1871,16 @@ mod tests { assert!(!inner.threads.contains_key(&old_tid)); } - let replacement = task - .global + assert!(matches!( + task.global.create_thread(), + Err(CreateError::ResourceExhausted) + )); + + let global = task.global.clone(); + drop(task); + let replacement = global .create_thread() - .expect("exec rebinding must release the old broker thread slot"); + .expect("task exit must release the retained broker thread slot"); replacement .exit() .expect("the replacement broker thread must exit"); diff --git a/litebox_shim_linux/src/syscalls/test_broker.rs b/litebox_shim_linux/src/syscalls/test_broker.rs index ec2204f462..804335b504 100644 --- a/litebox_shim_linux/src/syscalls/test_broker.rs +++ b/litebox_shim_linux/src/syscalls/test_broker.rs @@ -45,7 +45,7 @@ pub(crate) fn litebox_with_limits( ) -> (litebox::LiteBox, i32) { let setup = InProcessBrokerSetup::new(test_broker(limits).clone()); let readiness = setup.readiness_sink(); - let (broker_local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (broker_local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_shim_macos/src/loader/tests.rs b/litebox_shim_macos/src/loader/tests.rs index 52ec088a92..341aaee222 100644 --- a/litebox_shim_macos/src/loader/tests.rs +++ b/litebox_shim_macos/src/loader/tests.rs @@ -91,7 +91,7 @@ fn builder(executable: Vec) -> MacosShimBuilder { .unwrap(); let setup = InProcessBrokerSetup::new(core); let readiness = setup.readiness_sink(); - let (local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_shim_macos/src/syscalls/file.rs b/litebox_shim_macos/src/syscalls/file.rs index 203503fa89..4c639893a3 100644 --- a/litebox_shim_macos/src/syscalls/file.rs +++ b/litebox_shim_macos/src/syscalls/file.rs @@ -330,7 +330,7 @@ mod tests { .unwrap(); let setup = InProcessBrokerSetup::new(broker); let readiness = setup.readiness_sink(); - let (local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_shim_windows/src/test_broker.rs b/litebox_shim_windows/src/test_broker.rs index e912f3a32e..ce798b004a 100644 --- a/litebox_shim_windows/src/test_broker.rs +++ b/litebox_shim_windows/src/test_broker.rs @@ -77,7 +77,7 @@ fn connect( ) -> (litebox::LiteBox, usize) { let setup = InProcessBrokerSetup::new(broker); let readiness = setup.readiness_sink(); - let (broker_local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (broker_local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) })