From 14625de134e4fc85f4a2326fb8bf6b48dd3dd407 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 17 Sep 2026 12:56:31 -0700 Subject: [PATCH 01/21] Add dynamic child process startup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/lib.rs | 13 +- litebox_broker_core/src/process.rs | 245 +++++++++ litebox_broker_host/src/lib.rs | 214 +++++++- litebox_broker_local/src/lib.rs | 88 +++- litebox_broker_local/src/process.rs | 86 ++++ litebox_broker_local_userland/src/lib.rs | 5 +- litebox_broker_local_userland/src/linux.rs | 63 ++- litebox_broker_local_userland/src/windows.rs | 37 +- litebox_broker_protocol/src/lib.rs | 1 + litebox_broker_protocol/src/message.rs | 37 +- litebox_broker_protocol/src/process.rs | 101 ++++ litebox_broker_protocol/src/wire.rs | 251 +++++++++- .../src/unix_socket/host.rs | 5 +- .../src/unix_socket/local.rs | 6 +- .../src/host.rs | 5 +- .../src/local.rs | 8 +- litebox_broker_userland/src/runner.rs | 474 +++++++++++++++++- litebox_broker_userland/src/runner/linux.rs | 88 +++- litebox_broker_userland/src/runner/windows.rs | 88 +++- litebox_broker_userland/src/runtime.rs | 232 ++++++++- .../tests/userland_broker.rs | 185 ++++++- litebox_runner_linux_userland/src/lib.rs | 52 +- litebox_runner_windows_userland/src/lib.rs | 30 +- 23 files changed, 2233 insertions(+), 81 deletions(-) create mode 100644 litebox_broker_local/src/process.rs create mode 100644 litebox_broker_protocol/src/process.rs diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 99389bdde4..6c369148ea 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -52,6 +52,7 @@ pub use policy::{ use process::ObjectReference; pub use process::{ AssociationCancellation, BrokerProcess, BrokerThread, CallerCredential, ObjectRights, + PendingProcess, PreparedProcess, }; use random::RandomProvider; use socket::{BrokerSocketPorts, SocketProvider}; @@ -310,6 +311,15 @@ impl BrokerCore { pub fn create_process( &self, caller_credential: CallerCredential, + ) -> Result> { + self.create_process_with_parent(None, caller_credential, process::ProcessLifecycle::Running) + } + + fn create_process_with_parent( + &self, + parent_id: Option, + caller_credential: CallerCredential, + lifecycle: process::ProcessLifecycle, ) -> Result> { let mut processes = self.processes.write(); if processes.len() >= self.limits.max_processes { @@ -323,8 +333,9 @@ impl BrokerCore { let process = Arc::new(BrokerProcess::new( self.clone(), id, - None, + parent_id, caller_credential, + lifecycle, )); assert!( processes.insert(id, Arc::downgrade(&process)).is_none(), diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index bd9c3d3be4..69d57a567d 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -110,6 +110,7 @@ pub struct BrokerProcess { cleaned_up: bool, /// Authoritative parent process ID, absent for a root process. parent_id: Option, + lifecycle: Arc>, /// Broker-entry-authenticated caller credential for this process. pub(crate) caller_credential: CallerCredential, /// Handles of the live object references owned by this process. @@ -124,6 +125,103 @@ pub struct BrokerProcess { pub(crate) cancellation: AssociationCancellation, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ProcessLifecycle { + Attaching, + Running, + Exiting, +} + +/// Commit authority for one broker-reserved child process. +#[derive(Clone)] +pub struct PreparedProcess { + lifecycle: Arc>, +} + +impl PreparedProcess { + /// Commits a prepared child after its start result reaches the parent. + pub fn commit(&self) -> Result<()> { + let mut lifecycle = self.lifecycle.lock(); + match *lifecycle { + ProcessLifecycle::Attaching => { + *lifecycle = ProcessLifecycle::Running; + Ok(()) + } + ProcessLifecycle::Running => Err(BrokerError::Internal), + ProcessLifecycle::Exiting => Err(BrokerError::PeerClosed), + } + } +} + +/// Broker-reserved child process awaiting association setup. +pub struct PendingProcess { + process: Option>, + prepared: PreparedProcess, + inherited_objects: Vec, +} + +impl PendingProcess { + /// Returns the reserved child process ID. + /// + /// # Panics + /// + /// Panics if the pending process was internally transferred twice. + #[must_use] + pub fn id(&self) -> ProcessId { + self.process + .as_ref() + .expect("pending process must remain active") + .id() + } + + /// Returns commit authority without transferring association ownership. + #[must_use] + pub fn prepared(&self) -> PreparedProcess { + self.prepared.clone() + } + + /// Returns child-owned handles in the parent's inheritance-manifest order. + #[must_use] + pub fn inherited_objects(&self) -> &[ObjectHandle] { + &self.inherited_objects + } + + /// Returns the credential inherited from the parent process. + /// + /// # Panics + /// + /// Panics if the pending process was internally transferred twice. + #[must_use] + pub fn caller_credential(&self) -> CallerCredential { + self.process + .as_ref() + .expect("pending process must remain active") + .caller_credential + } + + /// Transfers the reserved process into its authenticated association. + /// + /// # Panics + /// + /// Panics if the pending process was internally transferred twice. + #[must_use] + pub fn attach(mut self) -> (Arc, PreparedProcess) { + let process = self + .process + .take() + .expect("pending process must remain active"); + (process, self.prepared.clone()) + } +} + +impl Drop for PendingProcess { + fn drop(&mut self) { + if let Some(process) = self.process.take() { + BrokerProcess::finish(process); + } + } +} + impl BrokerProcess { /// Creates authenticated broker process state. pub(crate) fn new( @@ -131,12 +229,14 @@ impl BrokerProcess { id: ProcessId, parent_id: Option, caller_credential: CallerCredential, + lifecycle: ProcessLifecycle, ) -> Self { Self { core, id, cleaned_up: false, parent_id, + lifecycle: Arc::new(Mutex::new(lifecycle)), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -161,6 +261,40 @@ impl BrokerProcess { self.parent_id } + /// Returns whether parent acknowledgement committed this process. + #[must_use] + pub fn is_running(&self) -> bool { + *self.lifecycle.lock() == ProcessLifecycle::Running + } + + /// Reserves one child process inheriting this process's authenticated credential. + pub fn prepare_child(&self, inherited_objects: &[ObjectHandle]) -> Result { + let process = self.core.create_process_with_parent( + Some(self.id), + self.caller_credential, + ProcessLifecycle::Attaching, + )?; + let prepared = PreparedProcess { + lifecycle: Arc::clone(&process.lifecycle), + }; + let mut pending = PendingProcess { + process: Some(process), + prepared, + inherited_objects: Vec::new(), + }; + pending + .inherited_objects + .try_reserve_exact(inherited_objects.len()) + .map_err(|_| BrokerError::OutOfMemory)?; + for handle in inherited_objects { + let child = pending.process.as_ref().ok_or(BrokerError::Internal)?; + let child_handle = + self.duplicate_object_reference_to_preserving_rights(*handle, child)?; + pending.inherited_objects.push(child_handle); + } + Ok(pending) + } + /// Creates a broker thread belonging to this process. /// /// # Panics @@ -219,6 +353,12 @@ impl BrokerProcess { self.cancellation.cancel(); } + /// Returns whether association teardown requested cancellation. + #[must_use] + pub fn is_cancellation_requested(&self) -> bool { + self.cancellation.is_cancelled() + } + /// Completes non-unwinding process teardown and releases its IDs. /// /// Dropping a process without calling this method performs authority @@ -345,6 +485,22 @@ impl BrokerProcess { target.create_object_reference_with_rights(object, rights) } + fn duplicate_object_reference_to_preserving_rights( + &self, + handle: ObjectHandle, + target: &BrokerProcess, + ) -> Result { + let rights = { + let references = self.core.references.read(); + let reference = references.get(&handle).ok_or(BrokerError::UnknownObject)?; + if reference.owner != self.id { + return Err(BrokerError::UnknownObject); + } + reference.rights + }; + self.duplicate_object_reference_to(handle, target, rights) + } + pub(crate) fn create_object_reference_pair( &self, first: ObjectEntry, @@ -616,6 +772,7 @@ impl BrokerProcess { return false; } self.cleaned_up = true; + *self.lifecycle.lock() = ProcessLifecycle::Exiting; let mut invariant_fault = self.references.lock().pending_handles != 0; loop { @@ -822,6 +979,94 @@ mod tests { assert_eq!(second.parent_id(), None); } + #[test] + fn prepared_child_is_parented_and_requires_commit() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let source_handle = crate::event::create(&parent, 1).unwrap(); + let pending = parent.prepare_child(&[source_handle]).unwrap(); + let child_id = pending.id(); + let inherited_handle = pending.inherited_objects()[0]; + let (child, prepared) = pending.attach(); + + assert_eq!(child.id(), child_id); + assert_eq!(child.parent_id(), Some(parent.id())); + assert_ne!(inherited_handle, source_handle); + assert_eq!( + child.check_readiness(inherited_handle).unwrap(), + ReadinessFlags::READ | ReadinessFlags::WRITE + ); + assert!(!child.is_running()); + + prepared.commit().unwrap(); + assert!(child.is_running()); + } + + #[test] + fn dropped_prepared_child_releases_process_capacity() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .with_limits(BrokerCoreLimits::DEFAULT.with_process_limit(2)) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let pending = parent.prepare_child(&[]).unwrap(); + + assert_eq!( + parent.prepare_child(&[]).err(), + Some(BrokerError::ResourceExhausted) + ); + drop(pending); + assert!(parent.prepare_child(&[]).is_ok()); + } + + #[test] + fn prepared_child_cannot_commit_after_teardown() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let (child, prepared) = parent.prepare_child(&[]).unwrap().attach(); + + child.finish(); + + assert_eq!(prepared.commit(), Err(BrokerError::PeerClosed)); + } + + #[test] + fn attached_child_releases_process_capacity_after_teardown() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .with_limits(BrokerCoreLimits::DEFAULT.with_process_limit(2)) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let (child, _prepared) = parent.prepare_child(&[]).unwrap().attach(); + + assert_eq!( + parent.prepare_child(&[]).err(), + Some(BrokerError::ResourceExhausted) + ); + child.finish(); + assert!(parent.prepare_child(&[]).is_ok()); + } + #[test] fn released_thread_id_is_reused_after_rotation() { let mut broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index b57ad0415c..332b607217 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -24,7 +24,7 @@ extern crate std; use alloc::{sync::Arc, vec::Vec}; use litebox_broker_core::readiness::ReadinessSink; -use litebox_broker_core::{BrokerCore, BrokerProcess, CallerCredential}; +use litebox_broker_core::{BrokerCore, BrokerProcess, CallerCredential, PendingProcess}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::event::{AddEventResponse, CreateEventResponse}; use litebox_broker_protocol::fs::{ @@ -43,9 +43,14 @@ use litebox_broker_protocol::message::{ use litebox_broker_protocol::pipe::{ CreatePipeResponse, MAX_PIPE_TRANSFER_SIZE, ReadPipeResponse, WritePipeResponse, }; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessBootstrapFormat, + ProcessBootstrapVersion, +}; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ SHARED_BUFFER_LAYOUT, SHARED_BUFFER_SLOT_COUNT, SHARED_BUFFER_SLOT_SIZE, SharedBufferSequence, + SharedBufferSlotIndex, }; use litebox_broker_protocol::socket::{ AcceptSocketResponse, BindSocketResponse, ConnectSocketResponse, CreateSocketResponse, @@ -90,7 +95,32 @@ struct AssociationState { shared_buffer_usage: SharedBufferUsage, } +/// Failure classification for deployment-specific broker operations. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BrokerHostExtensionError { + /// Return an error result and keep serving the association. + Respond(ErrorCode), + /// Fail the association without publishing a response. + Abort(ErrorCode), +} + +/// Opaque bootstrap staged for one broker-reserved prepared child. +pub struct PreparedProcessBootstrap<'a> { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Opaque platform bytes. + pub payload: &'a [u8], +} + impl BrokerHostAssociation<'_, Memory> { + /// Returns the broker-assigned process ID for this association. + #[must_use] + pub fn process_id(&self) -> litebox_broker_protocol::ProcessId { + self.process.id() + } + /// Requests cancellation of provider operations after the peer disconnects. pub fn request_cancellation(&self) { self.process.request_cancellation(); @@ -110,6 +140,30 @@ impl BrokerHostAssociation<'_, Memory> { &self, request: BrokerRequest, send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, + ) -> Result<(), ChannelError> { + self.execute_request_with( + request, + |_process, _operation, _shared_buffers| None, + send_response, + |_result| {}, + ) + } + + /// Executes one active request with an optional deployment-specific operation handler. + /// + /// The extension runs after shared-buffer ownership is validated and while + /// any operation-scoped sequence remains reserved for this request. + pub fn execute_request_with( + &self, + request: BrokerRequest, + extension: impl FnOnce( + &BrokerProcess, + &BrokerOperation, + &SharedBufferPool, + ) + -> Option>, + send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, + response_sent: impl FnOnce(&BrokerResult), ) -> Result<(), ChannelError> { let BrokerRequest { request_id, @@ -134,12 +188,19 @@ impl BrokerHostAssociation<'_, Memory> { } } - let result = match complete_request(handle_request( - &self.process, - operation, - self.shared_buffers, - &self.readiness_sink, - )) { + let request_result = match extension(&self.process, &operation, self.shared_buffers) { + Some(result) => result.map_err(|error| match error { + BrokerHostExtensionError::Respond(error) => RequestFailure::Respond(error), + BrokerHostExtensionError::Abort(error) => RequestFailure::Abort(error), + }), + None => handle_request( + &self.process, + operation, + self.shared_buffers, + &self.readiness_sink, + ), + }; + let result = match complete_request(request_result) { Ok(result) => result, Err(error) => { self.state.lock().failed = true; @@ -152,14 +213,28 @@ impl BrokerHostAssociation<'_, Memory> { .shared_buffer_usage .end(request_id, sequence); } - if let Err(error) = send_response(&BrokerResponse { request_id, result }) { + let response = BrokerResponse { request_id, result }; + if let Err(error) = send_response(&response) { self.state.lock().failed = true; return Err(BrokerHostError::Channel(error)); } + response_sent(&response.result); Ok(()) } } +/// Copies a validated operation-scoped shared-buffer sequence. +pub fn copy_shared_buffer( + shared_buffers: &SharedBufferPool, + buffer: SharedBufferSequence, + maximum_length: u32, +) -> core::result::Result, BrokerHostExtensionError> { + read_shared_buffer(shared_buffers, buffer, maximum_length).map_err(|error| match error { + RequestFailure::Respond(error) => BrokerHostExtensionError::Respond(error), + RequestFailure::Abort(error) => BrokerHostExtensionError::Abort(error), + }) +} + /// Authenticates and negotiates one broker control connection. /// /// `send_shared_memory` runs after version negotiation and before the active @@ -247,15 +322,109 @@ where BrokerProcess::finish(process); return Err(BrokerHostError::Channel(error)); } - return Ok(Ok(BrokerHostAssociation { - process, - shared_buffers, - readiness_sink, - state: SpinMutex::new(AssociationState { - failed: false, - shared_buffer_usage: SharedBufferUsage::new(), - }), - })); + return Ok(Ok(new_association(process, shared_buffers, readiness_sink))); + } +} + +/// Authenticates and negotiates one broker-reserved prepared child connection. +pub fn setup_prepared_connection<'a, SetupChannel, Memory, ChannelError>( + pending: PendingProcess, + setup_channel: &mut SetupChannel, + shared_buffers: &'a SharedBufferPool, + readiness_sink: Arc, + bootstrap: PreparedProcessBootstrap<'_>, + send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, +) -> Result, ChannelError> +where + SetupChannel: HostSetupChannel, + Memory: SharedMemory, +{ + if shared_buffers.layout() != SHARED_BUFFER_LAYOUT { + return Err(BrokerHostError::SharedBufferLayoutMismatch); + } + if bootstrap.payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { + return Err(BrokerHostError::Broker(ErrorCode::ResourceExhausted)); + } + let process_id = pending.id(); + let transport_credential = match setup_channel + .peer_credential() + .map_err(BrokerHostError::Channel)? + { + PeerCredential::HostGuaranteed => CallerCredential::HostGuaranteed, + PeerCredential::Unauthenticated => CallerCredential::Unauthenticated, + _ => return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)), + }; + if transport_credential != pending.caller_credential() { + return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)); + } + let request = match setup_channel + .recv_handshake_request() + .map_err(BrokerHostError::Channel)? + { + HostReceive::Message(request) => request, + HostReceive::ProtocolViolation => { + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::Error(ErrorCode::ProtocolState)) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::ProtocolViolation)); + } + HostReceive::PeerClosed => return Ok(Err(ConnectionTermination::PeerClosed)), + }; + if request.protocol_version != BROKER_PROTOCOL_VERSION { + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::VersionMismatch { + broker_protocol_version: BROKER_PROTOCOL_VERSION, + }) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::Rejected( + ErrorCode::UnsupportedVersion, + ))); + } + + let bootstrap_length = u32::try_from(bootstrap.payload.len()) + .map_err(|_| BrokerHostError::Broker(ErrorCode::ResourceExhausted))?; + let buffer = SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], bootstrap_length) + .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; + write_shared_buffer( + shared_buffers, + buffer, + bootstrap.payload, + MAX_PROCESS_BOOTSTRAP_SIZE, + ) + .map_err(|error| BrokerHostError::Broker(request_failure_error(error)))?; + + let response = BrokerHandshakeResponse::Prepared { + broker_protocol_version: BROKER_PROTOCOL_VERSION, + process_id, + bootstrap: ProcessBootstrap { + format: bootstrap.format, + version: bootstrap.version, + buffer, + }, + inherited_objects: InheritedProcessObjects::new(pending.inherited_objects()) + .ok_or(BrokerHostError::Broker(ErrorCode::Internal))?, + }; + setup_channel + .send_handshake_response(&response) + .map_err(BrokerHostError::Channel)?; + send_shared_memory(setup_channel).map_err(BrokerHostError::Channel)?; + let (process, _prepared) = pending.attach(); + Ok(Ok(new_association(process, shared_buffers, readiness_sink))) +} + +fn new_association( + process: Arc, + shared_buffers: &SharedBufferPool, + readiness_sink: Arc, +) -> BrokerHostAssociation<'_, Memory> { + BrokerHostAssociation { + process, + shared_buffers, + readiness_sink, + state: SpinMutex::new(AssociationState { + failed: false, + shared_buffer_usage: SharedBufferUsage::new(), + }), } } @@ -356,6 +525,12 @@ fn complete_request( } } +const fn request_failure_error(error: RequestFailure) -> ErrorCode { + match error { + RequestFailure::Respond(error) | RequestFailure::Abort(error) => error, + } +} + fn handle_request( process: &BrokerProcess, operation: BrokerOperation, @@ -404,6 +579,11 @@ fn handle_request( BrokerOperation::File(request) => { handle_file_request(process, request, shared_buffers).map(BrokerResult::File) } + BrokerOperation::StartProcess(_) + | BrokerOperation::AcknowledgeProcessStart(_) + | BrokerOperation::ProcessReady(_) => { + Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) + } } } diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 523ff3f590..f719a174ec 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -24,6 +24,7 @@ mod error; mod event; mod fs; mod pipe; +mod process; mod random; mod socket; mod stdio; @@ -31,7 +32,7 @@ mod stdio; #[cfg(any(test, feature = "test-support"))] pub mod test_support; -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use core::sync::atomic::{AtomicU64, Ordering}; use litebox_broker_protocol::error::ErrorCode; @@ -39,6 +40,9 @@ use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, }; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, +}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; use litebox_broker_protocol::{ @@ -67,6 +71,19 @@ pub struct BrokerNotifications { channel: Channel, } +/// Opaque bootstrap delivered to one broker-reserved prepared process. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PreparedProcessBootstrap { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Opaque platform bytes. + pub payload: Vec, + /// Child-owned broker handles in the parent's inheritance-manifest order. + pub inherited_objects: InheritedProcessObjects, +} + impl BrokerLocal { fn new(channel: Channel, process_id: ProcessId, shared_memory: Arc) -> Self { let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) @@ -126,6 +143,9 @@ impl BrokerLocal { activate(setup).map_err(BrokerLocalError::Channel)?; Ok((Self::new(channel, process_id, shared_memory), activated)) } + response @ BrokerHandshakeResponse::Prepared { .. } => { + panic!("broker returned unexpected negotiation response: {response:?}") + } BrokerHandshakeResponse::VersionMismatch { .. } => { Err(BrokerLocalError::Broker(ErrorCode::UnsupportedVersion)) } @@ -142,6 +162,72 @@ impl BrokerLocal { } } + /// Negotiates one broker-reserved prepared process and copies its bootstrap. + /// + /// # Panics + /// + /// Panics if the broker returns a response inconsistent with prepared + /// negotiation or setup returns shared memory with an invalid size. + pub fn negotiate_prepared, Activated>( + mut setup: Setup, + activate: impl FnOnce( + Setup, + ) -> core::result::Result< + (Channel, Arc, Activated), + Channel::Error, + >, + ) -> Result<(Self, PreparedProcessBootstrap, Activated), Channel::Error> { + let requested = BROKER_PROTOCOL_VERSION; + setup + .send_handshake_request(&BrokerHandshakeRequest { + protocol_version: requested, + }) + .map_err(BrokerLocalError::Channel)?; + match setup + .recv_handshake_response() + .map_err(BrokerLocalError::Channel)? + .ok_or(BrokerLocalError::ChannelClosed)? + { + response @ BrokerHandshakeResponse::Prepared { + broker_protocol_version, + process_id, + bootstrap, + inherited_objects, + } => { + assert_eq!( + requested, broker_protocol_version, + "broker returned unexpected prepared negotiation response: {response:?}" + ); + let (channel, shared_memory, activated) = + activate(setup).map_err(BrokerLocalError::Channel)?; + let local = Self::new(channel, process_id, shared_memory); + let mut payload = Vec::new(); + payload + .try_reserve_exact(bootstrap.buffer.length() as usize) + .map_err(|_| BrokerLocalError::Broker(ErrorCode::OutOfMemory))?; + payload.resize(bootstrap.buffer.length() as usize, 0); + local.read_shared_buffer(bootstrap.buffer, &mut payload); + Ok(( + local, + PreparedProcessBootstrap { + format: bootstrap.format, + version: bootstrap.version, + payload, + inherited_objects, + }, + activated, + )) + } + response @ BrokerHandshakeResponse::Negotiated { .. } => { + panic!("broker returned unexpected prepared negotiation response: {response:?}") + } + BrokerHandshakeResponse::VersionMismatch { .. } => { + Err(BrokerLocalError::Broker(ErrorCode::UnsupportedVersion)) + } + BrokerHandshakeResponse::Error(error) => Err(BrokerLocalError::Broker(error)), + } + } + /// Returns the assigned process ID. #[must_use] pub const fn process_id(&self) -> ProcessId { diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs new file mode 100644 index 0000000000..a53258cd47 --- /dev/null +++ b/litebox_broker_local/src/process.rs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +use litebox_broker_protocol::ThreadId; +use litebox_broker_protocol::error::ErrorCode; +use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessReadyRequest, ProcessStartToken, StartProcessRequest, + StartedProcess, +}; +use litebox_broker_protocol::shared_buffer::SharedBufferSequence; +use litebox_broker_transport::channel::LocalCallChannel; + +use crate::{BrokerLocal, BrokerLocalError, Result}; + +impl BrokerLocal { + /// Requests materialization of one child process. + /// + /// The returned token must be acknowledged before the child may begin + /// guest execution. The caller must retain exclusive ownership of the + /// bootstrap sequence until this method returns. + /// + /// # Panics + /// + /// Panics if the bootstrap length differs from the shared-buffer sequence + /// or the broker returns a response for another operation. + pub fn request_process_start( + &self, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + buffer: SharedBufferSequence, + bootstrap: &[u8], + inherited_objects: InheritedProcessObjects, + ) -> Result { + if buffer.length() > MAX_PROCESS_BOOTSTRAP_SIZE { + return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); + } + self.write_shared_buffer(buffer, bootstrap); + match self.request(BrokerOperation::StartProcess(StartProcessRequest { + bootstrap: ProcessBootstrap { + format, + version, + buffer, + }, + inherited_objects, + }))? { + BrokerResult::ProcessStarted(started) => Ok(started), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => panic!("broker returned unexpected process-start response: {response:?}"), + } + } + + /// Acknowledges a successful process-start result and releases the child. + /// + /// # Panics + /// + /// Panics if the broker returns a response for another operation. + pub fn acknowledge_process_start( + &self, + token: ProcessStartToken, + ) -> Result<(), Channel::Error> { + match self.request(BrokerOperation::AcknowledgeProcessStart(token))? { + BrokerResult::ProcessStartAcknowledged => Ok(()), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => { + panic!("broker returned unexpected process-start acknowledgement: {response:?}") + } + } + } + + /// Reports that this prepared process is ready and waits for parent acknowledgement. + /// + /// # Panics + /// + /// Panics if the broker returns a response for another operation. + pub fn process_ready(&self, initial_thread_id: Option) -> Result<(), Channel::Error> { + match self.request(BrokerOperation::ProcessReady(ProcessReadyRequest { + initial_thread_id, + }))? { + BrokerResult::ProcessReady => Ok(()), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => panic!("broker returned unexpected process-ready response: {response:?}"), + } + } +} diff --git a/litebox_broker_local_userland/src/lib.rs b/litebox_broker_local_userland/src/lib.rs index 90c69d2bf5..c7eff35fbe 100644 --- a/litebox_broker_local_userland/src/lib.rs +++ b/litebox_broker_local_userland/src/lib.rs @@ -9,10 +9,11 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::{ - BrokerAssociationFailureCoordinator, BrokerConnection, connect, start_notification_receiver, + BrokerAssociationFailureCoordinator, BrokerConnection, connect, connect_prepared, + start_notification_receiver, }; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(all(windows, target_arch = "x86_64"))] -pub use windows::{BrokerConnection, connect, start_notification_receiver}; +pub use windows::{BrokerConnection, connect, connect_prepared, start_notification_receiver}; diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index 01b4b54ad3..e442dbd7a7 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -12,7 +12,7 @@ use std::{ }; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications, PreparedProcessBootstrap}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; @@ -94,6 +94,67 @@ pub fn connect(control_socket_path: &Path) -> Result { }) } +/// Connects to and negotiates a broker-reserved prepared Linux process. +pub fn connect_prepared( + control_socket_path: &Path, +) -> Result<(BrokerConnection, PreparedProcessBootstrap)> { + let setup_deadline = Instant::now() + SETUP_TIMEOUT; + let setup_channel = connect_with_retry( + control_socket_path, + setup_deadline, + "timed out connecting to broker", + |path, deadline| UnixStreamLocalSetupChannel::connect_with_setup_deadline(path, deadline), + ) + .with_context(|| { + format!( + "failed to connect to broker at {}", + control_socket_path.display() + ) + })?; + let association_coordinator = Arc::new(BrokerAssociationFailureCoordinator::new()); + let (local, bootstrap, (notification_channel, positional_io_fds, shutdown_fd)) = + BrokerLocal::negotiate_prepared(setup_channel, |mut setup| { + let shared_memory = + setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; + let control_memory = setup.receive_control_ring(Some(setup_deadline))?; + let positional_io_fds = [ + shared_memory.as_fd().as_raw_fd(), + control_memory.as_fd().as_raw_fd(), + ]; + let control_ring = ControlRing::new(control_memory).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid broker control ring: {error:?}"), + ) + })?; + let weak_association_coordinator = Arc::downgrade(&association_coordinator); + let (call_channel, notification_channel, association_shutdown) = + setup.into_active(control_ring, move || { + if let Some(association_coordinator) = weak_association_coordinator.upgrade() { + association_coordinator.report_failure(); + } + })?; + let shutdown_fd = association_shutdown.as_fd().as_raw_fd(); + association_coordinator.install_shutdown(association_shutdown)?; + Ok(( + call_channel, + Arc::new(shared_memory), + (notification_channel, positional_io_fds, shutdown_fd), + )) + }) + .context("prepared broker negotiation failed")?; + Ok(( + BrokerConnection { + local, + notifications: BrokerNotifications::new(notification_channel), + coordinator: association_coordinator, + positional_io_fds, + shutdown_fd, + }, + bootstrap, + )) +} + /// Starts the broker notification receiver for an active association. pub fn start_notification_receiver( mut notifications: BrokerNotifications, diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index 8ef679fd65..8dc3bae983 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications, PreparedProcessBootstrap}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; @@ -55,6 +55,41 @@ pub fn connect(control_pipe: &OsStr) -> Result { }) } +/// Connects to and negotiates a broker-reserved prepared Windows process. +pub fn connect_prepared( + control_pipe: &OsStr, +) -> Result<(BrokerConnection, PreparedProcessBootstrap)> { + let deadline = Instant::now() + SETUP_TIMEOUT; + let setup = + WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(control_pipe, deadline) + .with_context(|| { + format!( + "failed to connect to broker at {}", + std::path::Path::new(control_pipe).display() + ) + })?; + let (local, bootstrap, notifications) = BrokerLocal::negotiate_prepared(setup, |mut setup| { + let shared_memory = Arc::new(setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?); + let control_memory = setup.receive_control_ring()?; + let control_ring = ControlRing::new(control_memory).map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid broker control ring: {error:?}"), + ) + })?; + let (calls, notifications) = setup.into_active(control_ring)?; + Ok((calls, shared_memory, notifications)) + }) + .context("prepared broker negotiation failed")?; + Ok(( + BrokerConnection { + local, + notifications: BrokerNotifications::new(notifications), + }, + bootstrap, + )) +} + /// Starts the broker notification receiver for an active association. pub fn start_notification_receiver( mut notifications: BrokerNotifications, diff --git a/litebox_broker_protocol/src/lib.rs b/litebox_broker_protocol/src/lib.rs index 681b4f71eb..8806d1c297 100644 --- a/litebox_broker_protocol/src/lib.rs +++ b/litebox_broker_protocol/src/lib.rs @@ -21,6 +21,7 @@ pub mod event; pub mod fs; pub mod message; pub mod pipe; +pub mod process; pub mod random; pub mod readiness; pub mod shared_buffer; diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index 56b778f114..d051c085c2 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -17,6 +17,10 @@ use crate::pipe::{ CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; +use crate::process::{ + InheritedProcessObjects, ProcessBootstrap, ProcessReadyRequest, ProcessStartToken, + StartProcessRequest, StartedProcess, +}; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; use crate::socket::{ @@ -64,6 +68,12 @@ pub enum BrokerOperation { Stdio(StdioRequest), /// File request family. File(FileRequest), + /// Start one child process from an opaque platform bootstrap. + StartProcess(StartProcessRequest), + /// Commit a prepared process after its start result reaches the parent. + AcknowledgeProcessStart(ProcessStartToken), + /// Report that this prepared process is ready to begin guest execution. + ProcessReady(ProcessReadyRequest), } impl BrokerOperation { @@ -97,7 +107,11 @@ impl BrokerOperation { | FileRequest::Unlink(UnlinkFileRequest { path: buffer, .. }) | FileRequest::Mkdir(MkdirFileRequest { path: buffer, .. }) | FileRequest::Rmdir(RmdirFileRequest { path: buffer, .. }), - ) => Some(*buffer), + ) + | Self::StartProcess(StartProcessRequest { + bootstrap: ProcessBootstrap { buffer, .. }, + .. + }) => Some(*buffer), Self::CreateThread | Self::ExitThread(_) | Self::CloseObject(_) @@ -118,7 +132,9 @@ impl BrokerOperation { | Self::Stdio(StdioRequest::IsTerminal(_)) | Self::File( FileRequest::Seek(_) | FileRequest::Truncate(_) | FileRequest::HandleStatus(_), - ) => None, + ) + | Self::AcknowledgeProcessStart(_) + | Self::ProcessReady(_) => None, } } } @@ -145,6 +161,17 @@ pub enum BrokerHandshakeResponse { /// Assigned process ID. process_id: ProcessId, }, + /// Negotiation result for a broker-reserved prepared child. + Prepared { + /// Broker protocol version supported by this endpoint. + broker_protocol_version: ProtocolVersion, + /// Broker-assigned child process ID. + process_id: ProcessId, + /// Opaque platform bootstrap staged in the child's shared-buffer pool. + bootstrap: ProcessBootstrap, + /// Child-owned broker handles corresponding to the parent's inheritance manifest. + inherited_objects: InheritedProcessObjects, + }, /// Negotiation failed because the requested version is unsupported. /// /// The connection remains in negotiation state and the local peer may retry @@ -234,6 +261,12 @@ pub enum BrokerResult { Stdio(StdioResponse), /// File response family. File(FileResponse), + /// A child was materialized and is ready for parent acknowledgement. + ProcessStarted(StartedProcess), + /// Parent acknowledgement committed the prepared child. + ProcessStartAcknowledged, + /// Parent acknowledgement released this prepared child. + ProcessReady, /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), } diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs new file mode 100644 index 0000000000..a770a582be --- /dev/null +++ b/litebox_broker_protocol/src/process.rs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +use crate::shared_buffer::SharedBufferSequence; +use crate::{ObjectHandle, ProcessId, ThreadId}; + +/// Maximum size of one versioned process bootstrap carried through the broker. +pub const MAX_PROCESS_BOOTSTRAP_SIZE: u32 = 64 * 1024; + +/// Maximum number of broker objects inherited by the bounded initial contract. +pub const MAX_INHERITED_PROCESS_OBJECTS: usize = 4; + +/// Platform-defined process-bootstrap format. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProcessBootstrapFormat(pub u32); + +/// Version of a platform-defined process-bootstrap format. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProcessBootstrapVersion(pub u16); + +/// Association-scoped token for acknowledging a prepared process start. +#[repr(transparent)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProcessStartToken(pub u64); + +/// Ordered broker-object handles inherited by a prepared child. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct InheritedProcessObjects { + handles: [ObjectHandle; MAX_INHERITED_PROCESS_OBJECTS], + count: u8, +} + +impl InheritedProcessObjects { + /// Empty inherited-object list. + pub const EMPTY: Self = Self { + handles: [ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS], + count: 0, + }; + + /// Creates a bounded ordered inherited-object list. + pub fn new(handles: &[ObjectHandle]) -> Option { + if handles.len() > MAX_INHERITED_PROCESS_OBJECTS { + return None; + } + let Ok(count) = u8::try_from(handles.len()) else { + return None; + }; + let mut stored = [ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS]; + stored[..handles.len()].copy_from_slice(handles); + Some(Self { + handles: stored, + count, + }) + } + + /// Returns inherited handles in manifest order. + #[must_use] + pub fn as_slice(&self) -> &[ObjectHandle] { + &self.handles[..usize::from(self.count)] + } +} + +/// Opaque bootstrap descriptor supplied when starting a process. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessBootstrap { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Operation-scoped shared-buffer sequence containing the bootstrap bytes. + pub buffer: SharedBufferSequence, +} + +/// Starts one child process from an opaque platform bootstrap. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StartProcessRequest { + /// Opaque bootstrap staged in the parent's shared-buffer pool. + pub bootstrap: ProcessBootstrap, + /// Parent-owned broker handles inherited in manifest order. + pub inherited_objects: InheritedProcessObjects, +} + +/// Reports a materialized child that is ready for parent acknowledgement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StartedProcess { + /// Token that must be acknowledged before the child may begin guest execution. + pub token: ProcessStartToken, + /// Broker-assigned child process ID. + pub process_id: ProcessId, + /// Broker-assigned initial thread ID when it differs from the process ID. + pub initial_thread_id: Option, +} + +/// Reports that a prepared child finished restoring its initial state. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessReadyRequest { + /// Broker-assigned initial thread ID when it differs from the process ID. + pub initial_thread_id: Option, +} diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 6e2087c683..d6e75bf176 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -23,6 +23,11 @@ use crate::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, ReadinessNotification, }; +use crate::process::{ + InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrap, + ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessReadyRequest, ProcessStartToken, + StartProcessRequest, StartedProcess, +}; use crate::readiness::ReadinessFlags; use primitive::{Decoder, Encoder}; @@ -45,6 +50,9 @@ const REQUEST_TAG_STDIO: u8 = 7; const REQUEST_TAG_FILE: u8 = 8; const REQUEST_TAG_CREATE_THREAD: u8 = 9; const REQUEST_TAG_EXIT_THREAD: u8 = 10; +const REQUEST_TAG_START_PROCESS: u8 = 11; +const REQUEST_TAG_ACKNOWLEDGE_PROCESS_START: u8 = 12; +const REQUEST_TAG_PROCESS_READY: u8 = 13; // Paired request and successful-response tags intentionally share values. const RESPONSE_TAG_NEGOTIATED: u8 = 0; @@ -58,16 +66,20 @@ const RESPONSE_TAG_STDIO: u8 = 7; const RESPONSE_TAG_FILE: u8 = 8; const RESPONSE_TAG_THREAD_CREATED: u8 = 9; const RESPONSE_TAG_THREAD_EXITED: u8 = 10; +const RESPONSE_TAG_PROCESS_STARTED: u8 = 11; +const RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED: u8 = 12; +const RESPONSE_TAG_PROCESS_READY: u8 = 13; // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; +const RESPONSE_TAG_PREPARED: u8 = 252; const RESPONSE_TAG_HANDSHAKE_ERROR: u8 = 254; const RESPONSE_TAG_VERSION_MISMATCH: u8 = 255; const NOTIFICATION_TAG_READINESS: u8 = 0; /// Maximum byte length of any encoded active request or response. -pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 67; +pub const MAX_ENCODED_ACTIVE_MESSAGE_SIZE: usize = 85; /// Maximum byte length of any encoded broker notification. pub const MAX_ENCODED_NOTIFICATION_SIZE: usize = 13; @@ -116,7 +128,10 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_EXIT_THREAD + | REQUEST_TAG_START_PROCESS + | REQUEST_TAG_ACKNOWLEDGE_PROCESS_START + | REQUEST_TAG_PROCESS_READY => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -185,6 +200,32 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); fs::encode_fs_request(&mut encoder, request); } + BrokerOperation::StartProcess(StartProcessRequest { + bootstrap: + ProcessBootstrap { + format, + version, + buffer, + }, + inherited_objects, + }) => { + encoder.u8(REQUEST_TAG_START_PROCESS); + encoder.request_id(request_id); + encoder.u32(format.0); + encoder.u16(version.0); + encoder.shared_buffer_sequence(buffer); + encode_inherited_objects(&mut encoder, inherited_objects); + } + BrokerOperation::AcknowledgeProcessStart(token) => { + encoder.u8(REQUEST_TAG_ACKNOWLEDGE_PROCESS_START); + encoder.request_id(request_id); + encoder.u64(token.0); + } + BrokerOperation::ProcessReady(ProcessReadyRequest { initial_thread_id }) => { + encoder.u8(REQUEST_TAG_PROCESS_READY); + encoder.request_id(request_id); + encode_optional_thread_id(&mut encoder, initial_thread_id); + } } encoder.finish() } @@ -204,7 +245,10 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_STDIO | REQUEST_TAG_FILE | REQUEST_TAG_CREATE_THREAD - | REQUEST_TAG_EXIT_THREAD => {} + | REQUEST_TAG_EXIT_THREAD + | REQUEST_TAG_START_PROCESS + | REQUEST_TAG_ACKNOWLEDGE_PROCESS_START + | REQUEST_TAG_PROCESS_READY => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -219,6 +263,20 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_FILL_RANDOM => BrokerOperation::FillRandom(decoder.shared_buffer_sequence()?), REQUEST_TAG_STDIO => BrokerOperation::Stdio(stdio::decode_stdio_request(&mut decoder)?), REQUEST_TAG_FILE => BrokerOperation::File(fs::decode_fs_request(&mut decoder)?), + REQUEST_TAG_START_PROCESS => BrokerOperation::StartProcess(StartProcessRequest { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + }, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }), + REQUEST_TAG_ACKNOWLEDGE_PROCESS_START => { + BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(decoder.u64()?)) + } + REQUEST_TAG_PROCESS_READY => BrokerOperation::ProcessReady(ProcessReadyRequest { + initial_thread_id: decode_optional_thread_id(&mut decoder)?, + }), _ => unreachable!("active request tag was validated"), }; decoder.finish()?; @@ -243,6 +301,25 @@ pub fn encode_handshake_response(response: BrokerHandshakeResponse) -> Vec { encoder.protocol_version(broker_protocol_version); encoder.process_id(process_id); } + BrokerHandshakeResponse::Prepared { + broker_protocol_version, + process_id, + bootstrap: + ProcessBootstrap { + format, + version, + buffer, + }, + inherited_objects, + } => { + encoder.u8(RESPONSE_TAG_PREPARED); + encoder.protocol_version(broker_protocol_version); + encoder.process_id(process_id); + encoder.u32(format.0); + encoder.u16(version.0); + encoder.shared_buffer_sequence(buffer); + encode_inherited_objects(&mut encoder, inherited_objects); + } BrokerHandshakeResponse::VersionMismatch { broker_protocol_version, } => { @@ -266,6 +343,16 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result BrokerHandshakeResponse::Prepared { + broker_protocol_version: decoder.protocol_version()?, + process_id: decoder.process_id()?, + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + }, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }, RESPONSE_TAG_EVENT | RESPONSE_TAG_OBJECT_CLOSED | RESPONSE_TAG_PIPE @@ -276,7 +363,10 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { + | RESPONSE_TAG_THREAD_EXITED + | RESPONSE_TAG_PROCESS_STARTED + | RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED + | RESPONSE_TAG_PROCESS_READY => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_VERSION_MISMATCH => BrokerHandshakeResponse::VersionMismatch { @@ -346,6 +436,25 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.request_id(request_id); fs::encode_fs_response(&mut encoder, response); } + BrokerResult::ProcessStarted(StartedProcess { + token, + process_id, + initial_thread_id, + }) => { + encoder.u8(RESPONSE_TAG_PROCESS_STARTED); + encoder.request_id(request_id); + encoder.u64(token.0); + encoder.process_id(process_id); + encode_optional_thread_id(&mut encoder, initial_thread_id); + } + BrokerResult::ProcessStartAcknowledged => { + encoder.u8(RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED); + encoder.request_id(request_id); + } + BrokerResult::ProcessReady => { + encoder.u8(RESPONSE_TAG_PROCESS_READY); + encoder.request_id(request_id); + } BrokerResult::Error(error) => { encoder.u8(RESPONSE_TAG_ERROR); encoder.request_id(request_id); @@ -360,7 +469,10 @@ pub fn decode_response(frame: &[u8]) -> Result { let mut decoder = Decoder::new(frame); let tag = decoder.u8()?; match tag { - RESPONSE_TAG_NEGOTIATED | RESPONSE_TAG_HANDSHAKE_ERROR | RESPONSE_TAG_VERSION_MISMATCH => { + RESPONSE_TAG_NEGOTIATED + | RESPONSE_TAG_PREPARED + | RESPONSE_TAG_HANDSHAKE_ERROR + | RESPONSE_TAG_VERSION_MISMATCH => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_EVENT @@ -373,7 +485,10 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_STDIO | RESPONSE_TAG_FILE | RESPONSE_TAG_THREAD_CREATED - | RESPONSE_TAG_THREAD_EXITED => {} + | RESPONSE_TAG_THREAD_EXITED + | RESPONSE_TAG_PROCESS_STARTED + | RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED + | RESPONSE_TAG_PROCESS_READY => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -389,12 +504,58 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_RANDOM_FILLED => BrokerResult::RandomFilled, RESPONSE_TAG_STDIO => BrokerResult::Stdio(stdio::decode_stdio_response(&mut decoder)?), RESPONSE_TAG_FILE => BrokerResult::File(fs::decode_fs_response(&mut decoder)?), + RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(StartedProcess { + token: ProcessStartToken(decoder.u64()?), + process_id: decoder.process_id()?, + initial_thread_id: decode_optional_thread_id(&mut decoder)?, + }), + RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED => BrokerResult::ProcessStartAcknowledged, + RESPONSE_TAG_PROCESS_READY => BrokerResult::ProcessReady, _ => unreachable!("active response tag was validated"), }; decoder.finish()?; Ok(BrokerResponse { request_id, result }) } +fn encode_optional_thread_id(encoder: &mut Encoder, thread_id: Option) { + encoder.u8(u8::from(thread_id.is_some())); + if let Some(thread_id) = thread_id { + encoder.thread_id(thread_id); + } +} + +fn decode_optional_thread_id( + decoder: &mut Decoder<'_>, +) -> Result, WireError> { + match decoder.u8()? { + 0 => Ok(None), + 1 => Ok(Some(decoder.thread_id()?)), + _ => Err(WireError::InvalidTag), + } +} + +fn encode_inherited_objects(encoder: &mut Encoder, objects: InheritedProcessObjects) { + encoder.u8(u8::try_from(objects.as_slice().len()) + .expect("bounded inherited-object count must fit in u8")); + for handle in objects.as_slice() { + encoder.handle(*handle); + } +} + +fn decode_inherited_objects( + decoder: &mut Decoder<'_>, +) -> Result { + let count = usize::from(decoder.u8()?); + if count > MAX_INHERITED_PROCESS_OBJECTS { + return Err(WireError::InvalidTag); + } + let mut handles = [crate::ObjectHandle(0); MAX_INHERITED_PROCESS_OBJECTS]; + for handle in &mut handles[..count] { + *handle = decoder.handle()?; + } + InheritedProcessObjects::new(&handles[..count]).ok_or(WireError::InvalidTag) +} + fn encode_error_code(encoder: &mut Encoder, error: ErrorCode) { encoder.u16(match error { ErrorCode::UnsupportedVersion => 1, @@ -484,6 +645,10 @@ mod tests { CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; + use crate::process::{ + InheritedProcessObjects, ProcessBootstrap, ProcessBootstrapFormat, ProcessBootstrapVersion, + ProcessReadyRequest, ProcessStartToken, StartProcessRequest, StartedProcess, + }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ AcceptSocketRequest, AcceptSocketResponse, AddressFamily, BindSocketRequest, @@ -539,6 +704,11 @@ mod tests { RESPONSE_TAG_RANDOM_FILLED, RESPONSE_TAG_STDIO, RESPONSE_TAG_FILE, + RESPONSE_TAG_THREAD_CREATED, + RESPONSE_TAG_THREAD_EXITED, + RESPONSE_TAG_PROCESS_STARTED, + RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED, + RESPONSE_TAG_PROCESS_READY, ], [ REQUEST_TAG_NEGOTIATE, @@ -550,15 +720,21 @@ mod tests { REQUEST_TAG_FILL_RANDOM, REQUEST_TAG_STDIO, REQUEST_TAG_FILE, + REQUEST_TAG_CREATE_THREAD, + REQUEST_TAG_EXIT_THREAD, + REQUEST_TAG_START_PROCESS, + REQUEST_TAG_ACKNOWLEDGE_PROCESS_START, + REQUEST_TAG_PROCESS_READY, ] ); assert_eq!( [ RESPONSE_TAG_ERROR, + RESPONSE_TAG_PREPARED, RESPONSE_TAG_HANDSHAKE_ERROR, RESPONSE_TAG_VERSION_MISMATCH, ], - [253, 254, 255] + [253, 252, 254, 255] ); } @@ -818,6 +994,27 @@ mod tests { name: TcpOptionName::KeepAlive, })), BrokerOperation::Socket(SocketRequest::Status(SocketStatusRequest { handle })), + BrokerOperation::StartProcess(StartProcessRequest { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(u32::MAX), + version: ProcessBootstrapVersion(u16::MAX), + buffer: largest_sequence, + }, + inherited_objects: InheritedProcessObjects::new(&[ + ObjectHandle(1), + ObjectHandle(2), + ObjectHandle(3), + ObjectHandle(4), + ]) + .unwrap(), + }), + BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(u64::MAX)), + BrokerOperation::ProcessReady(ProcessReadyRequest { + initial_thread_id: None, + }), + BrokerOperation::ProcessReady(ProcessReadyRequest { + initial_thread_id: Some(thread_id(19)), + }), ]; let mut maximum_encoded_size = 0; @@ -996,6 +1193,20 @@ mod tests { broker_protocol_version: ProtocolVersion(1), process_id: process_id(7), }, + BrokerHandshakeResponse::Prepared { + broker_protocol_version: ProtocolVersion(1), + process_id: process_id(9), + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(0x7465_7374), + version: ProcessBootstrapVersion(1), + buffer: sequence(0, 37), + }, + inherited_objects: InheritedProcessObjects::new(&[ + ObjectHandle(5), + ObjectHandle(6), + ]) + .unwrap(), + }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), }, @@ -1150,6 +1361,18 @@ mod tests { BrokerResult::File(FileResponse::Mkdir), BrokerResult::File(FileResponse::Rmdir), BrokerResult::File(FileResponse::Failed(FileError::Io)), + BrokerResult::ProcessStarted(StartedProcess { + token: ProcessStartToken(u64::MAX), + process_id: process_id(u32::MAX), + initial_thread_id: None, + }), + BrokerResult::ProcessStarted(StartedProcess { + token: ProcessStartToken(7), + process_id: process_id(9), + initial_thread_id: Some(thread_id(11)), + }), + BrokerResult::ProcessStartAcknowledged, + BrokerResult::ProcessReady, BrokerResult::Error(ErrorCode::PolicyDenied), BrokerResult::Error(ErrorCode::WouldBlock), BrokerResult::Error(ErrorCode::PeerClosed), @@ -1617,7 +1840,7 @@ mod tests { #[test] fn decode_rejects_malformed_handshake_response_frames() { assert_eq!( - decode_handshake_response(&[0xfc, 1, 2, 3]), + decode_handshake_response(&[0xfb, 1, 2, 3]), Err(WireError::InvalidTag) ); assert_eq!( @@ -1666,7 +1889,7 @@ mod tests { #[test] fn decode_rejects_malformed_response_frames() { assert_eq!( - decode_response(&[0xfc, 1, 2, 3]), + decode_response(&[0xfb, 1, 2, 3]), Err(WireError::InvalidTag) ); for response in [ @@ -1674,6 +1897,16 @@ mod tests { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), }, + BrokerHandshakeResponse::Prepared { + broker_protocol_version: ProtocolVersion(1), + process_id: process_id(2), + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(3), + version: ProcessBootstrapVersion(4), + buffer: sequence(0, 5), + }, + inherited_objects: InheritedProcessObjects::EMPTY, + }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), }, diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 1ddfb23d17..501eefd431 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -286,7 +286,10 @@ impl HostSetupChannel for UnixStreamHostSetupChannel { &encode_handshake_response(response.clone()), self.setup_deadline, )?; - self.negotiated = matches!(response, BrokerHandshakeResponse::Negotiated { .. }); + self.negotiated = matches!( + response, + BrokerHandshakeResponse::Negotiated { .. } | BrokerHandshakeResponse::Prepared { .. } + ); Ok(()) } } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 7ea2f9f291..3de72a6c75 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -307,7 +307,11 @@ impl LocalSetupChannel for UnixStreamLocalSetupChannel { match frame { Some(frame) => { let response = decode_handshake_response(&frame).map_err(wire_error)?; - self.negotiated = matches!(&response, BrokerHandshakeResponse::Negotiated { .. }); + self.negotiated = matches!( + &response, + BrokerHandshakeResponse::Negotiated { .. } + | BrokerHandshakeResponse::Prepared { .. } + ); Ok(Some(response)) } None => Ok(None), diff --git a/litebox_broker_transport_windows_userland/src/host.rs b/litebox_broker_transport_windows_userland/src/host.rs index deff03c523..8f1487fa4a 100644 --- a/litebox_broker_transport_windows_userland/src/host.rs +++ b/litebox_broker_transport_windows_userland/src/host.rs @@ -158,7 +158,10 @@ impl HostSetupChannel for WindowsNamedPipeHostSetupChannel { &encode_handshake_response(response.clone()), self.setup_deadline, )?; - self.negotiated = matches!(response, BrokerHandshakeResponse::Negotiated { .. }); + self.negotiated = matches!( + response, + BrokerHandshakeResponse::Negotiated { .. } | BrokerHandshakeResponse::Prepared { .. } + ); Ok(()) } } diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 96d263d6ae..b474177ff0 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -158,7 +158,13 @@ impl LocalSetupChannel for WindowsNamedPipeLocalSetupChannel { let response = read_frame(file_handle(&self.stream), self.setup_deadline)? .map(|frame| decode_handshake_response(&frame).map_err(wire_error)) .transpose()?; - self.negotiated = matches!(response, Some(BrokerHandshakeResponse::Negotiated { .. })); + self.negotiated = matches!( + response, + Some( + BrokerHandshakeResponse::Negotiated { .. } + | BrokerHandshakeResponse::Prepared { .. } + ) + ); Ok(response) } } diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 61a184d561..afa3e5993a 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -7,9 +7,19 @@ use std::ffi::{OsStr, OsString}; use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::path::PathBuf; use std::process::{Child, Command, ExitStatus}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; -use litebox_broker_core::BrokerCore; +use litebox_broker_core::{BrokerCore, BrokerProcess, PendingProcess, PreparedProcess}; +use litebox_broker_host::{BrokerHostExtensionError, copy_shared_buffer}; +use litebox_broker_protocol::error::ErrorCode; +use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; +use litebox_broker_protocol::process::{ + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartToken, + StartedProcess, +}; +use litebox_broker_protocol::{ProcessId, ThreadId}; +use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; #[cfg(target_os = "linux")] mod linux; @@ -23,8 +33,15 @@ use windows::PlatformRunnerEndpoint; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); +const PREPARED_CHILD_ARGUMENT: &str = "--prepared-child"; +const MAX_PENDING_CHILD_STARTS: usize = crate::WORKER_COUNT - 1; +const _: () = assert!(MAX_PENDING_CHILD_STARTS > 0); /// Configuration for starting one out-of-process runner. +/// +/// Dynamically started descendants use the same executable with the hidden +/// `--prepared-child` argument instead of the root arguments. +#[derive(Clone)] pub struct RunnerConfig { executable: PathBuf, arguments: Vec, @@ -63,6 +80,14 @@ impl RunnerConfig { arguments.extend(self.arguments.iter().cloned()); arguments } + + fn prepared_child(&self) -> Self { + Self { + executable: self.executable.clone(), + arguments: vec![OsString::from(PREPARED_CHILD_ARGUMENT)], + proxy_url: self.proxy_url.clone(), + } + } } /// One out-of-process runner and its dedicated broker control endpoint. @@ -72,6 +97,7 @@ impl RunnerConfig { pub struct RunnerInstance { runner: Child, endpoint: PlatformRunnerEndpoint, + child_config: RunnerConfig, } impl RunnerInstance { @@ -81,7 +107,12 @@ impl RunnerInstance { let runner = Command::new(&config.executable) .args(config.arguments(endpoint.control_channel())) .spawn()?; - Ok(Self { runner, endpoint }) + let child_config = config.prepared_child(); + Ok(Self { + runner, + endpoint, + child_config, + }) } /// Serves the runner's broker association and waits for its host process. @@ -90,7 +121,29 @@ impl RunnerInstance { /// non-successful runner exit is returned as ordinary instance data for the /// caller to interpret. pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { - let association_result = self.endpoint.serve(broker, &mut self.runner); + let children = RunnerChildren::new(self.child_config.clone()); + let association_result = + self.endpoint + .serve(broker, &mut self.runner, Arc::clone(&children)); + self.endpoint.close(); + if association_result.is_err() { + let _ = self.runner.kill(); + } + let runner_status = self.runner.wait(); + children.wait_for_drain(); + let runner_status = runner_status?; + association_result?; + Ok(runner_status) + } + + fn run_prepared_to_completion( + mut self, + prepared: PreparedRunner, + children: Arc, + ) -> IoResult { + let association_result = self + .endpoint + .serve_prepared(&mut self.runner, prepared, children); self.endpoint.close(); if association_result.is_err() { let _ = self.runner.kill(); @@ -111,6 +164,421 @@ impl Drop for RunnerInstance { } } +pub(crate) struct RunnerChildren { + config: RunnerConfig, + state: Mutex, + drained: Condvar, +} + +pub(crate) struct PreparedRunner { + pub(crate) pending: PendingProcess, + pub(crate) format: ProcessBootstrapFormat, + pub(crate) version: ProcessBootstrapVersion, + pub(crate) bootstrap: Vec, +} + +struct RunnerChildrenState { + launches: Vec<(ProcessStartToken, Arc)>, + active_instances: usize, +} + +struct ChildLaunch { + parent_id: ProcessId, + child_id: ProcessId, + prepared: PreparedProcess, + state: Mutex, + changed: Condvar, +} + +#[derive(Clone, Copy)] +enum ChildLaunchState { + Starting, + Ready { + initial_thread_id: Option, + start_result_delivered: bool, + }, + Committed, + Aborted(ErrorCode), +} + +impl RunnerChildren { + fn new(config: RunnerConfig) -> Arc { + Arc::new(Self { + config, + state: Mutex::new(RunnerChildrenState { + launches: Vec::new(), + active_instances: 0, + }), + drained: Condvar::new(), + }) + } + + pub(crate) fn handle_operation( + self: &Arc, + process: &BrokerProcess, + operation: &BrokerOperation, + shared_buffers: &SharedBufferPool, + ) -> Option> { + match operation { + BrokerOperation::StartProcess(request) => Some( + copy_shared_buffer( + shared_buffers, + request.bootstrap.buffer, + MAX_PROCESS_BOOTSTRAP_SIZE, + ) + .and_then(|bootstrap| { + self.start_process( + process, + request.bootstrap.format, + request.bootstrap.version, + bootstrap, + request.inherited_objects.as_slice(), + ) + .map_err(process_extension_error) + }) + .map(BrokerResult::ProcessStarted), + ), + BrokerOperation::AcknowledgeProcessStart(token) => Some( + self.acknowledge_process_start(process.id(), *token) + .map(|()| BrokerResult::ProcessStartAcknowledged) + .map_err(process_extension_error), + ), + BrokerOperation::ProcessReady(request) => Some( + self.process_ready(process.id(), request.initial_thread_id) + .map(|()| BrokerResult::ProcessReady) + .map_err(process_extension_error), + ), + _ => None, + } + } + + pub(crate) fn response_sent(&self, parent_id: ProcessId, result: &BrokerResult) { + let BrokerResult::ProcessStarted(started) = result else { + return; + }; + let Some(launch) = self.find_launch(started.token) else { + return; + }; + if launch.parent_id == parent_id { + launch.mark_start_result_delivered(); + } + } + + fn start_process( + self: &Arc, + parent: &BrokerProcess, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + bootstrap: Vec, + requested_inherited_objects: &[litebox_broker_protocol::ObjectHandle], + ) -> Result { + if !parent.is_running() { + return Err(ErrorCode::ProtocolState); + } + let pending = parent + .prepare_child(requested_inherited_objects) + .map_err(ErrorCode::from)?; + let child_id = pending.id(); + let launch = Arc::new(ChildLaunch { + parent_id: parent.id(), + child_id, + prepared: pending.prepared(), + state: Mutex::new(ChildLaunchState::Starting), + changed: Condvar::new(), + }); + let token = loop { + let mut token = [0; 8]; + getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; + let token = ProcessStartToken(u64::from_ne_bytes(token)); + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state + .launches + .try_reserve(1) + .map_err(|_| ErrorCode::OutOfMemory)?; + if parent.is_cancellation_requested() { + return Err(ErrorCode::PeerClosed); + } + if state.launches.len() >= MAX_PENDING_CHILD_STARTS { + return Err(ErrorCode::ResourceExhausted); + } + if state + .launches + .iter() + .any(|(candidate, _)| *candidate == token) + { + continue; + } + let active_instances = state + .active_instances + .checked_add(1) + .ok_or(ErrorCode::ResourceExhausted)?; + state.launches.push((token, Arc::clone(&launch))); + state.active_instances = active_instances; + break token; + }; + + let children = Arc::clone(self); + let config = self.config.clone(); + let thread_launch = Arc::clone(&launch); + let thread = std::thread::Builder::new() + .name(format!("litebox-runner-{}", child_id.0)) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + RunnerInstance::start(config).and_then(|instance| { + instance.run_prepared_to_completion( + PreparedRunner { + pending, + format, + version, + bootstrap, + }, + Arc::clone(&children), + ) + }) + })) + .unwrap_or_else(|_| Err(IoError::other("prepared runner thread panicked"))); + children.child_finished(token, &thread_launch, result); + }); + if thread.is_err() { + self.remove_launch(token); + self.finish_instance(); + return Err(ErrorCode::OutOfMemory); + } + drop(thread); + + let initial_thread_id = launch.wait_until_ready()?; + Ok(StartedProcess { + token, + process_id: child_id, + initial_thread_id, + }) + } + + fn acknowledge_process_start( + &self, + parent_id: ProcessId, + token: ProcessStartToken, + ) -> Result<(), ErrorCode> { + let launch = self.find_launch(token).ok_or(ErrorCode::UnknownObject)?; + if launch.parent_id != parent_id { + return Err(ErrorCode::UnknownObject); + } + launch.commit()?; + self.remove_launch(token); + Ok(()) + } + + fn process_ready( + &self, + child_id: ProcessId, + initial_thread_id: Option, + ) -> Result<(), ErrorCode> { + let launch = self + .find_child_launch(child_id) + .ok_or(ErrorCode::ProtocolState)?; + launch.ready_and_wait(initial_thread_id) + } + + pub(crate) fn association_ended(&self, process_id: ProcessId) { + loop { + let launch = { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state + .launches + .iter() + .position(|(_, launch)| { + launch.parent_id == process_id || launch.child_id == process_id + }) + .map(|index| state.launches.swap_remove(index).1) + }; + let Some(launch) = launch else { + break; + }; + launch.abort(ErrorCode::PeerClosed); + } + } + + fn find_launch(&self, token: ProcessStartToken) -> Option> { + self.state + .lock() + .expect("runner child state mutex poisoned") + .launches + .iter() + .find_map(|(candidate, launch)| (*candidate == token).then(|| Arc::clone(launch))) + } + + fn find_child_launch(&self, child_id: ProcessId) -> Option> { + self.state + .lock() + .expect("runner child state mutex poisoned") + .launches + .iter() + .find_map(|(_, launch)| (launch.child_id == child_id).then(|| Arc::clone(launch))) + } + + fn remove_launch(&self, token: ProcessStartToken) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + if let Some(index) = state + .launches + .iter() + .position(|(candidate, _)| *candidate == token) + { + state.launches.swap_remove(index); + } + } + + fn child_finished( + &self, + token: ProcessStartToken, + launch: &ChildLaunch, + result: IoResult, + ) { + if result.is_err() || result.is_ok_and(|status| !status.success()) { + launch.abort(ErrorCode::PeerClosed); + } + self.remove_launch(token); + self.finish_instance(); + } + + fn finish_instance(&self) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state.active_instances = state + .active_instances + .checked_sub(1) + .expect("runner child count must remain balanced"); + if state.active_instances == 0 { + self.drained.notify_all(); + } + } + + fn wait_for_drain(&self) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + while state.active_instances != 0 { + state = self + .drained + .wait(state) + .expect("runner child state mutex poisoned"); + } + } +} + +const fn process_extension_error(error: ErrorCode) -> BrokerHostExtensionError { + match error { + ErrorCode::PolicyDenied + | ErrorCode::UnknownObject + | ErrorCode::InvalidRights + | ErrorCode::ResourceExhausted + | ErrorCode::WouldBlock + | ErrorCode::PeerClosed + | ErrorCode::OutOfMemory + | ErrorCode::UnsupportedOperation => BrokerHostExtensionError::Respond(error), + _ => BrokerHostExtensionError::Abort(error), + } +} + +impl ChildLaunch { + fn wait_until_ready(&self) -> Result, ErrorCode> { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match *state { + ChildLaunchState::Starting => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + ChildLaunchState::Ready { + initial_thread_id, .. + } => return Ok(initial_thread_id), + ChildLaunchState::Committed => return Err(ErrorCode::ProtocolState), + ChildLaunchState::Aborted(error) => return Err(error), + } + } + } + + fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if !matches!(*state, ChildLaunchState::Starting) { + return Err(ErrorCode::ProtocolState); + } + *state = ChildLaunchState::Ready { + initial_thread_id, + start_result_delivered: false, + }; + self.changed.notify_all(); + loop { + match *state { + ChildLaunchState::Ready { .. } => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + ChildLaunchState::Committed => return Ok(()), + ChildLaunchState::Aborted(error) => return Err(error), + ChildLaunchState::Starting => unreachable!("ready state cannot regress"), + } + } + } + + fn commit(&self) -> Result<(), ErrorCode> { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if !matches!( + *state, + ChildLaunchState::Ready { + start_result_delivered: true, + .. + } + ) { + return Err(match *state { + ChildLaunchState::Aborted(error) => error, + _ => ErrorCode::ProtocolState, + }); + } + self.prepared.commit().map_err(ErrorCode::from)?; + *state = ChildLaunchState::Committed; + self.changed.notify_all(); + Ok(()) + } + + fn mark_start_result_delivered(&self) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if let ChildLaunchState::Ready { + start_result_delivered, + .. + } = &mut *state + { + *start_result_delivered = true; + } + } + + fn abort(&self, error: ErrorCode) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if matches!( + *state, + ChildLaunchState::Starting | ChildLaunchState::Ready { .. } + ) { + *state = ChildLaunchState::Aborted(error); + self.changed.notify_all(); + } + } +} + fn accept_runner_channel( deadline: Instant, channel_name: &'static str, diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index c3f687fd8f..a48242dc82 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -6,6 +6,7 @@ use std::io::Result as IoResult; use std::os::unix::net::UnixListener; use std::path::PathBuf; use std::process::Child; +use std::sync::Arc; use std::time::Instant; use litebox_broker_core::BrokerCore; @@ -15,7 +16,7 @@ use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; -use super::{SETUP_TIMEOUT, accept_runner_channel}; +use super::{PreparedRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, @@ -42,13 +43,35 @@ impl PlatformRunnerEndpoint { self.socket_path.as_os_str() } - pub(super) fn serve(&mut self, broker: &BrokerCore, runner: &mut Child) -> IoResult<()> { + pub(super) fn serve( + &mut self, + broker: &BrokerCore, + runner: &mut Child, + children: Arc, + ) -> IoResult<()> { serve_runner_process( broker, self.listener .as_ref() .expect("a live runner instance must own its control listener"), runner, + children, + ) + } + + pub(super) fn serve_prepared( + &mut self, + runner: &mut Child, + prepared: PreparedRunner, + children: Arc, + ) -> IoResult<()> { + serve_prepared_runner_process( + self.listener + .as_ref() + .expect("a live runner instance must own its control listener"), + runner, + prepared, + children, ) } @@ -62,7 +85,50 @@ fn serve_runner_process( broker: &BrokerCore, control_listener: &UnixListener, runner: &mut Child, + children: Arc, ) -> IoResult<()> { + let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; + crate::runtime::serve_runner_association( + broker, + control_channel, + || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), + MemfdSharedMemory::create_control_ring, + |channel, shared_memory, control_memory| { + channel.send_memfd(shared_memory, Some(setup_deadline))?; + channel.send_memfd(control_memory, Some(setup_deadline))?; + Ok(()) + }, + UnixStreamHostSetupChannel::into_active, + children, + ) +} + +fn serve_prepared_runner_process( + control_listener: &UnixListener, + runner: &mut Child, + prepared: PreparedRunner, + children: Arc, +) -> IoResult<()> { + let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; + crate::runtime::serve_prepared_association( + prepared, + control_channel, + || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), + MemfdSharedMemory::create_control_ring, + |channel, shared_memory, control_memory| { + channel.send_memfd(shared_memory, Some(setup_deadline))?; + channel.send_memfd(control_memory, Some(setup_deadline))?; + Ok(()) + }, + UnixStreamHostSetupChannel::into_active, + children, + ) +} + +fn accept_control_channel( + control_listener: &UnixListener, + runner: &mut Child, +) -> IoResult<(UnixStreamHostSetupChannel, Instant)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let control_stream = accept_runner_channel( setup_deadline, @@ -75,18 +141,8 @@ fn serve_runner_process( || control_listener.accept().map(|(stream, _)| stream), )?; validate_peer_process(&control_stream, runner.id())?; - let control_channel = - UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - crate::runtime::serve_association( - broker, - control_channel, - || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), - MemfdSharedMemory::create_control_ring, - |channel, shared_memory, control_memory| { - channel.send_memfd(shared_memory, Some(setup_deadline))?; - channel.send_memfd(control_memory, Some(setup_deadline))?; - Ok(()) - }, - UnixStreamHostSetupChannel::into_active, - ) + Ok(( + UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline), + setup_deadline, + )) } diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 62d8e47adc..5b3ec0b158 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -5,6 +5,7 @@ use std::ffi::OsString; use std::io::Result as IoResult; use std::os::windows::io::AsRawHandle; use std::process::Child; +use std::sync::Arc; use std::time::Instant; use litebox_broker_core::BrokerCore; @@ -14,7 +15,7 @@ use litebox_broker_transport_windows_userland::named_pipe::{ }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; -use super::{SETUP_TIMEOUT, accept_runner_channel}; +use super::{PreparedRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; pub(super) struct PlatformRunnerEndpoint { pipe_name: OsString, @@ -35,13 +36,35 @@ impl PlatformRunnerEndpoint { &self.pipe_name } - pub(super) fn serve(&mut self, broker: &BrokerCore, runner: &mut Child) -> IoResult<()> { + pub(super) fn serve( + &mut self, + broker: &BrokerCore, + runner: &mut Child, + children: Arc, + ) -> IoResult<()> { serve_runner_process( broker, self.listener .as_mut() .expect("a live runner instance must own its control listener"), runner, + children, + ) + } + + pub(super) fn serve_prepared( + &mut self, + runner: &mut Child, + prepared: PreparedRunner, + children: Arc, + ) -> IoResult<()> { + serve_prepared_runner_process( + self.listener + .as_mut() + .expect("a live runner instance must own its control listener"), + runner, + prepared, + children, ) } @@ -54,7 +77,50 @@ fn serve_runner_process( broker: &BrokerCore, control_listener: &mut WindowsNamedPipeListener, runner: &mut Child, + children: Arc, ) -> IoResult<()> { + let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; + let runner_process = runner.as_raw_handle(); + crate::runtime::serve_runner_association( + broker, + control_channel, + || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), + WindowsSharedMemory::create_control_ring, + |channel, shared_memory, control_memory| { + channel.send_shared_memory(shared_memory, runner_process)?; + channel.send_shared_memory(control_memory, runner_process) + }, + WindowsNamedPipeHostSetupChannel::into_active, + children, + ) +} + +fn serve_prepared_runner_process( + control_listener: &mut WindowsNamedPipeListener, + runner: &mut Child, + prepared: PreparedRunner, + children: Arc, +) -> IoResult<()> { + let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; + let runner_process = runner.as_raw_handle(); + crate::runtime::serve_prepared_association( + prepared, + control_channel, + || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), + WindowsSharedMemory::create_control_ring, + |channel, shared_memory, control_memory| { + channel.send_shared_memory(shared_memory, runner_process)?; + channel.send_shared_memory(control_memory, runner_process) + }, + WindowsNamedPipeHostSetupChannel::into_active, + children, + ) +} + +fn accept_control_channel( + control_listener: &mut WindowsNamedPipeListener, + runner: &mut Child, +) -> IoResult<(WindowsNamedPipeHostSetupChannel, Instant)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let control_stream = accept_runner_channel( setup_deadline, @@ -67,20 +133,10 @@ fn serve_runner_process( || control_listener.try_accept(), )?; validate_client_process(&control_stream, runner.id())?; - let runner_process = runner.as_raw_handle(); - let control_channel = - WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - crate::runtime::serve_association( - broker, - control_channel, - || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), - WindowsSharedMemory::create_control_ring, - |channel, shared_memory, control_memory| { - channel.send_shared_memory(shared_memory, runner_process)?; - channel.send_shared_memory(control_memory, runner_process) - }, - WindowsNamedPipeHostSetupChannel::into_active, - ) + Ok(( + WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline), + setup_deadline, + )) } fn unique_control_pipe_name() -> OsString { diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 554e968908..f8f0686dbd 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -27,7 +27,8 @@ use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; use litebox_broker_host::{ - BrokerHostAssociation, BrokerHostError, ConnectionTermination, setup_connection, + BrokerHostAssociation, BrokerHostError, ConnectionTermination, PreparedProcessBootstrap, + setup_connection, setup_prepared_connection, }; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::BrokerRequest; @@ -40,6 +41,7 @@ use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; use crate::readiness::ReadinessPublisherRuntime; +use crate::runner::{PreparedRunner, RunnerChildren}; const REQUEST_QUEUE_CAPACITY: usize = 64; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); @@ -65,6 +67,81 @@ pub fn serve_association< ResponseSink, NotificationChannel, Shutdown, +>( + broker: &BrokerCore, + control_channel: SetupChannel, + create_shared_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce() -> IoResult, + send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, + activate: impl FnOnce( + SetupChannel, + ControlRing, + ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, +) -> IoResult<()> +where + Memory: ControlRingMemory, + SetupChannel: HostSetupChannel, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync, +{ + serve_association_with_children( + broker, + control_channel, + create_shared_memory, + create_control_memory, + send_shared_memory, + activate, + None, + ) +} + +pub(crate) fn serve_runner_association< + Memory, + SetupChannel, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, +>( + broker: &BrokerCore, + control_channel: SetupChannel, + create_shared_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce() -> IoResult, + send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, + activate: impl FnOnce( + SetupChannel, + ControlRing, + ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + children: Arc, +) -> IoResult<()> +where + Memory: ControlRingMemory, + SetupChannel: HostSetupChannel, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync, +{ + serve_association_with_children( + broker, + control_channel, + create_shared_memory, + create_control_memory, + send_shared_memory, + activate, + Some(children), + ) +} + +fn serve_association_with_children< + Memory, + SetupChannel, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, >( broker: &BrokerCore, mut control_channel: SetupChannel, @@ -75,6 +152,7 @@ pub fn serve_association< SetupChannel, ControlRing, ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + children: Option>, ) -> IoResult<()> where Memory: ControlRingMemory, @@ -128,13 +206,107 @@ where return Err(error); } }; - dispatch_requests( + dispatch_requests_with_children( + association, + readiness, + request_source, + response_sink, + notification_channel, + shutdown, + children, + ) +} + +pub(crate) fn serve_prepared_association< + Memory, + SetupChannel, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, +>( + prepared: PreparedRunner, + mut control_channel: SetupChannel, + create_shared_memory: impl FnOnce() -> IoResult, + create_control_memory: impl FnOnce() -> IoResult, + send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, + activate: impl FnOnce( + SetupChannel, + ControlRing, + ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, + children: Arc, +) -> IoResult<()> +where + Memory: ControlRingMemory, + SetupChannel: HostSetupChannel, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync, +{ + let PreparedRunner { + pending, + format, + version, + bootstrap, + } = prepared; + let shared_memory = create_shared_memory()?; + let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) + .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?; + let control_memory = create_control_memory()?; + let control_ring = ControlRing::new(control_memory) + .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; + let readiness = Arc::new(ReadinessPublisherRuntime::new()); + let association = match setup_prepared_connection( + pending, + &mut control_channel, + &shared_buffers, + readiness.clone(), + PreparedProcessBootstrap { + format, + version, + payload: &bootstrap, + }, + |channel| send_shared_memory(channel, shared_buffers.memory(), control_ring.memory()), + ) + .map_err(map_host_error)? + { + Ok(association) => association, + Err(ConnectionTermination::PeerClosed) => { + return Err(IoError::new( + ErrorKind::UnexpectedEof, + "prepared runner closed before completing broker setup", + )); + } + Err(ConnectionTermination::ProtocolViolation) => { + return Err(IoError::new( + ErrorKind::InvalidData, + "prepared runner violated the broker protocol during setup", + )); + } + Err(_) => { + return Err(IoError::new( + ErrorKind::InvalidData, + "prepared runner ended broker setup unexpectedly", + )); + } + }; + let (request_source, response_sink, notification_channel, shutdown) = + match activate(control_channel, control_ring) { + Ok(active) => active, + Err(error) => { + association.finish(); + return Err(error); + } + }; + dispatch_requests_with_children( association, readiness, request_source, response_sink, notification_channel, shutdown, + Some(children), ) } @@ -296,13 +468,47 @@ impl Drop for AssociationCancellationGuard<'_, '_, Memory> /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network /// reactor is currently its production source. +#[cfg(all(test, target_os = "linux"))] fn dispatch_requests( + association: BrokerHostAssociation<'_, Memory>, + readiness: Arc, + request_source: RequestSource, + response_sink: ResponseSink, + notification_channel: NotificationChannel, + shutdown: Shutdown, +) -> IoResult<()> +where + Memory: SharedMemory, + RequestSource: HostRequestSource, + ResponseSink: HostResponseSink + Clone + Send, + NotificationChannel: HostNotificationChannel + Send, + Shutdown: HostAssociationShutdown + Send + Sync, +{ + dispatch_requests_with_children( + association, + readiness, + request_source, + response_sink, + notification_channel, + shutdown, + None, + ) +} + +fn dispatch_requests_with_children< + Memory, + RequestSource, + ResponseSink, + NotificationChannel, + Shutdown, +>( association: BrokerHostAssociation<'_, Memory>, readiness: Arc, mut request_source: RequestSource, response_sink: ResponseSink, mut notification_channel: NotificationChannel, shutdown: Shutdown, + children: Option>, ) -> IoResult<()> where Memory: SharedMemory, @@ -359,6 +565,7 @@ where let request_receiver = Arc::clone(&request_receiver); let response_sink = response_sink.clone(); let worker_failure_coordinator = Arc::clone(&failure_coordinator); + let worker_children = children.clone(); match std::thread::Builder::new() .name(format!("litebox-broker-worker-{worker_id}")) .spawn_scoped(scope, move || { @@ -367,6 +574,7 @@ where &request_receiver, &response_sink, &worker_failure_coordinator, + worker_children.as_ref(), ); }) { Ok(worker) => workers.push(worker), @@ -379,6 +587,9 @@ where read_requests(&mut request_source, request_sender, &failure_coordinator); drop(cancellation); + if let Some(children) = &children { + children.association_ended(association.process_id()); + } for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -497,6 +708,7 @@ fn run_worker( request_receiver: &Mutex>, response_sink: &ResponseSink, failure_coordinator: &HostAssociationFailureCoordinator, + children: Option<&Arc>, ) where Memory: SharedMemory, ResponseSink: HostResponseSink, @@ -514,7 +726,21 @@ fn run_worker( continue; } match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - association.execute_request(request, |response| response_sink.send_response(response)) + let process_id = association.process_id(); + association.execute_request_with( + request, + |process, operation, shared_buffers| { + children.and_then(|children| { + children.handle_operation(process, operation, shared_buffers) + }) + }, + |response| response_sink.send_response(response), + |result| { + if let Some(children) = children { + children.response_sent(process_id, result); + } + }, + ) })) { Ok(Ok(())) => {} Ok(Err(error)) => failure_coordinator.report(map_host_error(error)), diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 3c4a438367..6b85193906 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -2,13 +2,18 @@ // Licensed under the MIT license. use std::ffi::{OsStr, OsString}; -use std::io::{ErrorKind, Result}; +use std::io::{ErrorKind, Result, Write as _}; +use std::os::unix::ffi::OsStrExt as _; use std::path::Path; use std::process::{Child, Command}; use std::sync::Arc; use std::time::{Duration, Instant}; -use litebox_broker_local::BrokerLocal; +use litebox_broker_local::{BrokerLocal, BrokerLocalError}; +use litebox_broker_protocol::error::ErrorCode; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, +}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{ SHARED_BUFFER_POOL_SIZE, SharedBufferSequence, SharedBufferSlotIndex, @@ -19,6 +24,11 @@ use litebox_broker_transport_linux_userland::unix_socket::UnixStreamLocalSetupCh const RUNNER_ARGUMENT: &str = "broker-userland-test-runner"; const NETWORK_RUNNER_ARGUMENT: &str = "broker-userland-network-test-runner"; +const CHILD_START_RUNNER_ARGUMENT: &str = "broker-userland-child-start-runner"; +const CHILD_CANCEL_RUNNER_ARGUMENT: &str = "broker-userland-child-cancel-runner"; +const PREPARED_CHILD_ARGUMENT: &str = "--prepared-child"; +const TEST_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x7465_7374); +const FAILING_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x6661_696c); const BROKER_PROCESS_TIMEOUT: Duration = Duration::from_secs(30); fn main() { @@ -50,6 +60,32 @@ fn run_parent_test() { .arg(RUNNER_ARGUMENT); wait_for_broker(event_command); + let child_marker = unique_child_marker_path(); + let mut child_command = Command::new(env!("CARGO_BIN_EXE_litebox-broker-userland")); + child_command + .arg("--runner") + .arg(&test_executable) + .arg(CHILD_START_RUNNER_ARGUMENT) + .arg(&child_marker); + wait_for_broker(child_command); + let child_result = std::fs::read_to_string(&child_marker).unwrap(); + assert!(child_result.starts_with("ready:")); + assert!(child_result.ends_with("\nstarted\nfinished\n")); + std::fs::remove_file(child_marker).unwrap(); + + let cancelled_marker = unique_child_marker_path(); + let mut cancel_command = Command::new(env!("CARGO_BIN_EXE_litebox-broker-userland")); + cancel_command + .arg("--runner") + .arg(&test_executable) + .arg(CHILD_CANCEL_RUNNER_ARGUMENT) + .arg(&cancelled_marker); + wait_for_broker(cancel_command); + let cancelled_result = std::fs::read_to_string(&cancelled_marker).unwrap(); + assert!(cancelled_result.starts_with("ready:")); + assert_eq!(cancelled_result.lines().count(), 1); + std::fs::remove_file(cancelled_marker).unwrap(); + let gateway = std::net::Ipv4Addr::new(10, 0, 2, 1); let tcp_listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); let tcp_port = tcp_listener.local_addr().unwrap().port(); @@ -109,6 +145,10 @@ fn run_fake_runner(args: &[OsString]) { ); let control_socket_path = args.get(2).unwrap(); + if args.get(3).and_then(|argument| argument.to_str()) == Some(PREPARED_CHILD_ARGUMENT) { + run_fake_prepared_child(Path::new(control_socket_path)); + return; + } let setup_channel = connect_control_with_retry(Path::new(control_socket_path)).unwrap(); let (local, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd( @@ -203,6 +243,68 @@ fn run_fake_runner(args: &[OsString]) { local.close_object(handle).unwrap(); return; } + if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_START_RUNNER_ARGUMENT) { + assert_eq!(args.len(), 5, "unexpected runner arguments: {args:?}"); + let marker = Path::new(&args[4]); + let bootstrap = marker.as_os_str().as_encoded_bytes(); + let inherited_event = local.create_event_with_count(1).unwrap(); + let inherited_objects = InheritedProcessObjects::new(&[inherited_event]).unwrap(); + assert!(matches!( + local.request_process_start( + FAILING_BOOTSTRAP_FORMAT, + ProcessBootstrapVersion(1), + SharedBufferSequence::new( + &[SharedBufferSlotIndex(0)], + bootstrap.len().try_into().unwrap(), + ) + .unwrap(), + bootstrap, + inherited_objects, + ), + Err(BrokerLocalError::Broker(ErrorCode::PeerClosed)) + )); + let started = local + .request_process_start( + TEST_BOOTSTRAP_FORMAT, + ProcessBootstrapVersion(1), + SharedBufferSequence::new( + &[SharedBufferSlotIndex(0)], + bootstrap.len().try_into().unwrap(), + ) + .unwrap(), + bootstrap, + inherited_objects, + ) + .unwrap(); + assert_eq!(started.initial_thread_id, None); + let ready = format!("ready:{}\n", started.process_id.0); + wait_for_marker(marker, &ready); + std::thread::sleep(Duration::from_millis(100)); + assert_eq!(std::fs::read_to_string(marker).unwrap(), ready); + local.acknowledge_process_start(started.token).unwrap(); + return; + } + if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_CANCEL_RUNNER_ARGUMENT) { + assert_eq!(args.len(), 5, "unexpected runner arguments: {args:?}"); + let marker = Path::new(&args[4]); + let bootstrap = marker.as_os_str().as_encoded_bytes(); + let inherited_event = local.create_event_with_count(1).unwrap(); + let started = local + .request_process_start( + TEST_BOOTSTRAP_FORMAT, + ProcessBootstrapVersion(1), + SharedBufferSequence::new( + &[SharedBufferSlotIndex(0)], + bootstrap.len().try_into().unwrap(), + ) + .unwrap(), + bootstrap, + InheritedProcessObjects::new(&[inherited_event]).unwrap(), + ) + .unwrap(); + wait_for_marker(marker, &format!("ready:{}\n", started.process_id.0)); + return; + } assert_eq!( args.get(3).map(OsString::as_os_str), Some(OsStr::new(RUNNER_ARGUMENT)) @@ -270,6 +372,85 @@ fn run_fake_runner(args: &[OsString]) { drop(local); } +fn run_fake_prepared_child(control_socket_path: &Path) { + let setup_channel = connect_control_with_retry(control_socket_path).unwrap(); + let (local, bootstrap, ()) = BrokerLocal::negotiate_prepared(setup_channel, |mut setup| { + let shared_memory = setup.receive_memfd( + SHARED_BUFFER_POOL_SIZE, + Some(Instant::now() + Duration::from_secs(5)), + )?; + let control_memory = + setup.receive_control_ring(Some(Instant::now() + Duration::from_secs(5)))?; + let control_ring = ControlRing::new(control_memory).map_err(|error| { + std::io::Error::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; + let (call_channel, _notifications, _shutdown) = setup.into_active(control_ring, || {})?; + Ok((call_channel, Arc::new(shared_memory), ())) + }) + .unwrap(); + if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { + return; + } + assert_eq!(bootstrap.format, TEST_BOOTSTRAP_FORMAT); + assert_eq!(bootstrap.version, ProcessBootstrapVersion(1)); + let marker = Path::new(OsStr::from_bytes(&bootstrap.payload)); + let inherited_objects = bootstrap.inherited_objects.as_slice(); + assert_eq!(inherited_objects.len(), 1); + assert_eq!( + local.check_readiness(inherited_objects[0]).unwrap(), + ReadinessFlags::READ | ReadinessFlags::WRITE + ); + std::fs::write(marker, format!("ready:{}\n", local.process_id().0)).unwrap(); + match local.process_ready(None) { + Ok(()) => {} + Err(BrokerLocalError::Broker(ErrorCode::PeerClosed)) => return, + Err(error) => panic!("prepared child readiness failed: {error}"), + } + std::fs::OpenOptions::new() + .append(true) + .open(marker) + .unwrap() + .write_all(b"started\n") + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + std::fs::OpenOptions::new() + .append(true) + .open(marker) + .unwrap() + .write_all(b"finished\n") + .unwrap(); +} + +fn wait_for_marker(path: &Path, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match std::fs::read_to_string(path) { + Ok(contents) if contents == expected => return, + Ok(_) | Err(_) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + result => panic!( + "timed out waiting for child marker {}: {result:?}", + path.display() + ), + } + } +} + +fn unique_child_marker_path() -> std::path::PathBuf { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "litebox-child-start-{}-{nonce}", + std::process::id() + )) +} + struct ChildGuard { child: Child, } diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 1f23069f79..836845ed86 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -30,7 +30,11 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/usr/bin/python3 --version`). /// /// The program path must be absolute and refer to a file in the broker-owned file system. - #[arg(required = true, trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] + #[arg( + required_unless_present = "prepared_child", + trailing_var_arg = true, + value_hint = clap::ValueHint::CommandWithArguments + )] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times) #[arg(long = "env")] @@ -41,6 +45,14 @@ pub struct CliArgs { /// Allow using unstable options #[arg(short = 'Z', long = "unstable")] pub unstable: bool, + /// Connect as a broker-reserved prepared child. + #[arg( + long = "prepared-child", + hide = true, + requires_all = ["unstable", "broker_control_channel"], + help_heading = "Unstable Options" + )] + pub prepared_child: bool, /// Broker-supplied Unix socket path for the local control channel. #[arg( long = "broker-control-channel", @@ -86,7 +98,28 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - let prog_path = &cli_args.program_and_arguments[0]; + if cli_args.prepared_child { + if !cli_args.program_and_arguments.is_empty() { + return Err(anyhow!( + "--prepared-child does not accept a root program argument" + )); + } + let control_socket_path = cli_args + .broker_control_channel + .as_deref() + .context("--prepared-child requires --broker-control-channel")?; + let (_connection, bootstrap) = broker::connect_prepared(control_socket_path)?; + return Err(anyhow!( + "unsupported prepared Linux process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } + + let prog_path = cli_args + .program_and_arguments + .first() + .context("program path missing")?; if !prog_path.starts_with('/') { anyhow::bail!("program path must be absolute (e.g., /usr/bin/ls), got: {prog_path}"); } @@ -222,6 +255,21 @@ mod tests { ); } + #[test] + fn prepared_child_does_not_require_a_root_program() { + let args = CliArgs::try_parse_from([ + "runner", + "--unstable", + "--broker-control-channel", + "/tmp/broker.sock", + "--prepared-child", + ]) + .unwrap(); + + assert!(args.prepared_child); + assert!(args.program_and_arguments.is_empty()); + } + #[test] fn broker_proxy_replaces_proxy_environment() { let mut environment = vec![ diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 40f4d5d191..b665ca3099 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -20,7 +20,11 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/app/program.exe --help`). /// /// The program path refers to a path inside the broker-owned file system. - #[arg(required = true, trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] + #[arg( + required_unless_present = "prepared_child", + trailing_var_arg = true, + value_hint = clap::ValueHint::CommandWithArguments + )] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times). #[arg(long = "env")] @@ -31,6 +35,14 @@ pub struct CliArgs { /// Allow using unstable options. #[arg(short = 'Z', long = "unstable")] pub unstable: bool, + /// Connect as a broker-reserved prepared child. + #[arg( + long = "prepared-child", + hide = true, + requires_all = ["unstable", "broker_control_channel"], + help_heading = "Unstable Options" + )] + pub prepared_child: bool, /// Broker-supplied Windows named-pipe path for the local control channel. #[arg( long = "broker-control-channel", @@ -54,6 +66,22 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); + if cli_args.prepared_child { + if !cli_args.program_and_arguments.is_empty() { + anyhow::bail!("--prepared-child does not accept a root program argument"); + } + let control_pipe = cli_args + .broker_control_channel + .as_deref() + .context("--prepared-child requires --broker-control-channel")?; + let (_connection, bootstrap) = broker::connect_prepared(control_pipe)?; + anyhow::bail!( + "unsupported prepared Windows process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + ); + } + let platform = WindowsUserland::new(); WindowsUserland::set_guest_tls_mode(GuestTlsMode::Windows); let control_pipe = cli_args From 774c402ee9da114de664e49195b5e65f65cc2fd1 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 17 Sep 2026 14:06:37 -0700 Subject: [PATCH 02/21] Clarify child process state and commit naming Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/lib.rs | 8 ++-- litebox_broker_core/src/process.rs | 65 +++++++++++++-------------- litebox_broker_host/src/lib.rs | 2 +- litebox_broker_userland/src/runner.rs | 8 ++-- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 6c369148ea..5bc698828a 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -52,7 +52,7 @@ pub use policy::{ use process::ObjectReference; pub use process::{ AssociationCancellation, BrokerProcess, BrokerThread, CallerCredential, ObjectRights, - PendingProcess, PreparedProcess, + PendingProcess, ProcessStartCommit, }; use random::RandomProvider; use socket::{BrokerSocketPorts, SocketProvider}; @@ -312,14 +312,14 @@ impl BrokerCore { &self, caller_credential: CallerCredential, ) -> Result> { - self.create_process_with_parent(None, caller_credential, process::ProcessLifecycle::Running) + self.create_process_with_parent(None, caller_credential, process::ProcessState::Running) } fn create_process_with_parent( &self, parent_id: Option, caller_credential: CallerCredential, - lifecycle: process::ProcessLifecycle, + state: process::ProcessState, ) -> Result> { let mut processes = self.processes.write(); if processes.len() >= self.limits.max_processes { @@ -335,7 +335,7 @@ impl BrokerCore { id, parent_id, caller_credential, - lifecycle, + state, )); assert!( processes.insert(id, Arc::downgrade(&process)).is_none(), diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index 69d57a567d..4c3bb985b7 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -110,7 +110,7 @@ pub struct BrokerProcess { cleaned_up: bool, /// Authoritative parent process ID, absent for a root process. parent_id: Option, - lifecycle: Arc>, + state: Arc>, /// Broker-entry-authenticated caller credential for this process. pub(crate) caller_credential: CallerCredential, /// Handles of the live object references owned by this process. @@ -126,7 +126,7 @@ pub struct BrokerProcess { } #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum ProcessLifecycle { +pub(crate) enum ProcessState { Attaching, Running, Exiting, @@ -134,21 +134,19 @@ pub(crate) enum ProcessLifecycle { /// Commit authority for one broker-reserved child process. #[derive(Clone)] -pub struct PreparedProcess { - lifecycle: Arc>, -} +pub struct ProcessStartCommit(Arc>); -impl PreparedProcess { +impl ProcessStartCommit { /// Commits a prepared child after its start result reaches the parent. pub fn commit(&self) -> Result<()> { - let mut lifecycle = self.lifecycle.lock(); - match *lifecycle { - ProcessLifecycle::Attaching => { - *lifecycle = ProcessLifecycle::Running; + let mut state = self.0.lock(); + match *state { + ProcessState::Attaching => { + *state = ProcessState::Running; Ok(()) } - ProcessLifecycle::Running => Err(BrokerError::Internal), - ProcessLifecycle::Exiting => Err(BrokerError::PeerClosed), + ProcessState::Running => Err(BrokerError::Internal), + ProcessState::Exiting => Err(BrokerError::PeerClosed), } } } @@ -156,7 +154,7 @@ impl PreparedProcess { /// Broker-reserved child process awaiting association setup. pub struct PendingProcess { process: Option>, - prepared: PreparedProcess, + start_commit: ProcessStartCommit, inherited_objects: Vec, } @@ -176,8 +174,8 @@ impl PendingProcess { /// Returns commit authority without transferring association ownership. #[must_use] - pub fn prepared(&self) -> PreparedProcess { - self.prepared.clone() + pub fn start_commit(&self) -> ProcessStartCommit { + self.start_commit.clone() } /// Returns child-owned handles in the parent's inheritance-manifest order. @@ -205,12 +203,10 @@ impl PendingProcess { /// /// Panics if the pending process was internally transferred twice. #[must_use] - pub fn attach(mut self) -> (Arc, PreparedProcess) { - let process = self - .process + pub fn attach(mut self) -> Arc { + self.process .take() - .expect("pending process must remain active"); - (process, self.prepared.clone()) + .expect("pending process must remain active") } } @@ -229,14 +225,14 @@ impl BrokerProcess { id: ProcessId, parent_id: Option, caller_credential: CallerCredential, - lifecycle: ProcessLifecycle, + state: ProcessState, ) -> Self { Self { core, id, cleaned_up: false, parent_id, - lifecycle: Arc::new(Mutex::new(lifecycle)), + state: Arc::new(Mutex::new(state)), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -264,7 +260,7 @@ impl BrokerProcess { /// Returns whether parent acknowledgement committed this process. #[must_use] pub fn is_running(&self) -> bool { - *self.lifecycle.lock() == ProcessLifecycle::Running + *self.state.lock() == ProcessState::Running } /// Reserves one child process inheriting this process's authenticated credential. @@ -272,14 +268,12 @@ impl BrokerProcess { let process = self.core.create_process_with_parent( Some(self.id), self.caller_credential, - ProcessLifecycle::Attaching, + ProcessState::Attaching, )?; - let prepared = PreparedProcess { - lifecycle: Arc::clone(&process.lifecycle), - }; + let start_commit = ProcessStartCommit(Arc::clone(&process.state)); let mut pending = PendingProcess { process: Some(process), - prepared, + start_commit, inherited_objects: Vec::new(), }; pending @@ -772,7 +766,7 @@ impl BrokerProcess { return false; } self.cleaned_up = true; - *self.lifecycle.lock() = ProcessLifecycle::Exiting; + *self.state.lock() = ProcessState::Exiting; let mut invariant_fault = self.references.lock().pending_handles != 0; loop { @@ -993,7 +987,8 @@ mod tests { let pending = parent.prepare_child(&[source_handle]).unwrap(); let child_id = pending.id(); let inherited_handle = pending.inherited_objects()[0]; - let (child, prepared) = pending.attach(); + let start_commit = pending.start_commit(); + let child = pending.attach(); assert_eq!(child.id(), child_id); assert_eq!(child.parent_id(), Some(parent.id())); @@ -1004,7 +999,7 @@ mod tests { ); assert!(!child.is_running()); - prepared.commit().unwrap(); + start_commit.commit().unwrap(); assert!(child.is_running()); } @@ -1039,11 +1034,13 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let (child, prepared) = parent.prepare_child(&[]).unwrap().attach(); + let pending = parent.prepare_child(&[]).unwrap(); + let start_commit = pending.start_commit(); + let child = pending.attach(); child.finish(); - assert_eq!(prepared.commit(), Err(BrokerError::PeerClosed)); + assert_eq!(start_commit.commit(), Err(BrokerError::PeerClosed)); } #[test] @@ -1057,7 +1054,7 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let (child, _prepared) = parent.prepare_child(&[]).unwrap().attach(); + let child = parent.prepare_child(&[]).unwrap().attach(); assert_eq!( parent.prepare_child(&[]).err(), diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 332b607217..d921079a84 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -408,7 +408,7 @@ where .send_handshake_response(&response) .map_err(BrokerHostError::Channel)?; send_shared_memory(setup_channel).map_err(BrokerHostError::Channel)?; - let (process, _prepared) = pending.attach(); + let process = pending.attach(); Ok(Ok(new_association(process, shared_buffers, readiness_sink))) } diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index afa3e5993a..4c01eb6282 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -10,7 +10,7 @@ use std::process::{Child, Command, ExitStatus}; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; -use litebox_broker_core::{BrokerCore, BrokerProcess, PendingProcess, PreparedProcess}; +use litebox_broker_core::{BrokerCore, BrokerProcess, PendingProcess, ProcessStartCommit}; use litebox_broker_host::{BrokerHostExtensionError, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; @@ -185,7 +185,7 @@ struct RunnerChildrenState { struct ChildLaunch { parent_id: ProcessId, child_id: ProcessId, - prepared: PreparedProcess, + start_commit: ProcessStartCommit, state: Mutex, changed: Condvar, } @@ -282,7 +282,7 @@ impl RunnerChildren { let launch = Arc::new(ChildLaunch { parent_id: parent.id(), child_id, - prepared: pending.prepared(), + start_commit: pending.start_commit(), state: Mutex::new(ChildLaunchState::Starting), changed: Condvar::new(), }); @@ -550,7 +550,7 @@ impl ChildLaunch { _ => ErrorCode::ProtocolState, }); } - self.prepared.commit().map_err(ErrorCode::from)?; + self.start_commit.commit().map_err(ErrorCode::from)?; *state = ChildLaunchState::Committed; self.changed.notify_all(); Ok(()) From b242864ca6ba228b8ca9e744c020e6f2427f0e47 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 17 Sep 2026 16:35:50 -0700 Subject: [PATCH 03/21] Unify child process negotiation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/lib.rs | 1 - litebox_broker_core/src/process.rs | 219 ++++++------------ litebox_broker_host/src/lib.rs | 169 ++++++-------- litebox_broker_host/src/test_support.rs | 2 + litebox_broker_local/src/lib.rs | 136 +++++------ litebox_broker_local/src/process.rs | 2 +- litebox_broker_local_userland/src/lib.rs | 4 +- litebox_broker_local_userland/src/linux.rs | 81 ++----- litebox_broker_local_userland/src/windows.rs | 52 ++--- litebox_broker_protocol/src/message.rs | 25 +- litebox_broker_protocol/src/process.rs | 15 +- litebox_broker_protocol/src/wire.rs | 132 ++++++----- .../src/unix_socket/host.rs | 6 +- .../src/unix_socket/local.rs | 7 +- .../src/host.rs | 5 +- .../src/local.rs | 8 +- .../src/named_pipe.rs | 5 + litebox_broker_userland/src/runner.rs | 150 ++++++------ litebox_broker_userland/src/runner/linux.rs | 24 +- litebox_broker_userland/src/runner/windows.rs | 24 +- litebox_broker_userland/src/runtime.rs | 179 +++++--------- .../tests/notification_runtime.rs | 8 +- .../tests/userland_broker.rs | 16 +- litebox_runner_linux_userland/src/lib.rs | 26 +-- litebox_runner_linux_userland/tests/run.rs | 2 + .../src/test_broker.rs | 2 +- litebox_runner_windows_userland/src/lib.rs | 18 +- .../src/syscalls/test_broker.rs | 2 +- litebox_shim_macos/src/syscalls/file.rs | 2 +- litebox_shim_windows/src/test_broker.rs | 2 +- 30 files changed, 560 insertions(+), 764 deletions(-) diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 5bc698828a..d9d3f7ebb2 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -52,7 +52,6 @@ pub use policy::{ use process::ObjectReference; pub use process::{ AssociationCancellation, BrokerProcess, BrokerThread, CallerCredential, ObjectRights, - PendingProcess, ProcessStartCommit, }; use random::RandomProvider; use socket::{BrokerSocketPorts, SocketProvider}; diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index 4c3bb985b7..c8efd0d289 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -107,10 +107,9 @@ pub struct BrokerProcess { pub(crate) core: BrokerCore, /// Assigned process ID and internal authority. pub(crate) id: ProcessId, - cleaned_up: bool, /// Authoritative parent process ID, absent for a root process. parent_id: Option, - state: Arc>, + state: Mutex, /// Broker-entry-authenticated caller credential for this process. pub(crate) caller_credential: CallerCredential, /// Handles of the live object references owned by this process. @@ -132,92 +131,6 @@ pub(crate) enum ProcessState { Exiting, } -/// Commit authority for one broker-reserved child process. -#[derive(Clone)] -pub struct ProcessStartCommit(Arc>); - -impl ProcessStartCommit { - /// Commits a prepared child after its start result reaches the parent. - pub fn commit(&self) -> Result<()> { - let mut state = self.0.lock(); - match *state { - ProcessState::Attaching => { - *state = ProcessState::Running; - Ok(()) - } - ProcessState::Running => Err(BrokerError::Internal), - ProcessState::Exiting => Err(BrokerError::PeerClosed), - } - } -} - -/// Broker-reserved child process awaiting association setup. -pub struct PendingProcess { - process: Option>, - start_commit: ProcessStartCommit, - inherited_objects: Vec, -} - -impl PendingProcess { - /// Returns the reserved child process ID. - /// - /// # Panics - /// - /// Panics if the pending process was internally transferred twice. - #[must_use] - pub fn id(&self) -> ProcessId { - self.process - .as_ref() - .expect("pending process must remain active") - .id() - } - - /// Returns commit authority without transferring association ownership. - #[must_use] - pub fn start_commit(&self) -> ProcessStartCommit { - self.start_commit.clone() - } - - /// Returns child-owned handles in the parent's inheritance-manifest order. - #[must_use] - pub fn inherited_objects(&self) -> &[ObjectHandle] { - &self.inherited_objects - } - - /// Returns the credential inherited from the parent process. - /// - /// # Panics - /// - /// Panics if the pending process was internally transferred twice. - #[must_use] - pub fn caller_credential(&self) -> CallerCredential { - self.process - .as_ref() - .expect("pending process must remain active") - .caller_credential - } - - /// Transfers the reserved process into its authenticated association. - /// - /// # Panics - /// - /// Panics if the pending process was internally transferred twice. - #[must_use] - pub fn attach(mut self) -> Arc { - self.process - .take() - .expect("pending process must remain active") - } -} - -impl Drop for PendingProcess { - fn drop(&mut self) { - if let Some(process) = self.process.take() { - BrokerProcess::finish(process); - } - } -} - impl BrokerProcess { /// Creates authenticated broker process state. pub(crate) fn new( @@ -230,9 +143,8 @@ impl BrokerProcess { Self { core, id, - cleaned_up: false, parent_id, - state: Arc::new(Mutex::new(state)), + state: Mutex::new(state), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -257,36 +169,64 @@ impl BrokerProcess { self.parent_id } + /// Returns the credential authenticated for this process association. + #[must_use] + pub const fn caller_credential(&self) -> CallerCredential { + self.caller_credential + } + /// Returns whether parent acknowledgement committed this process. #[must_use] pub fn is_running(&self) -> bool { *self.state.lock() == ProcessState::Running } - /// Reserves one child process inheriting this process's authenticated credential. - pub fn prepare_child(&self, inherited_objects: &[ObjectHandle]) -> Result { + /// Commits this child after its start result reaches the parent. + pub fn commit_start(&self) -> Result<()> { + let mut state = self.state.lock(); + match *state { + ProcessState::Attaching => { + *state = ProcessState::Running; + Ok(()) + } + ProcessState::Running => Err(BrokerError::Internal), + ProcessState::Exiting => Err(BrokerError::PeerClosed), + } + } + + /// Creates one child process inheriting this process's authenticated credential. + /// + /// Returned handles follow the requested inheritance order. If later host + /// launch or association setup fails normally, the caller must finish the + /// returned process; dropping it preserves its IDs as unwind protection. + pub fn create_child( + &self, + inherited_objects: &[ObjectHandle], + ) -> Result<(Arc, Vec)> { let process = self.core.create_process_with_parent( Some(self.id), self.caller_credential, ProcessState::Attaching, )?; - let start_commit = ProcessStartCommit(Arc::clone(&process.state)); - let mut pending = PendingProcess { - process: Some(process), - start_commit, - inherited_objects: Vec::new(), - }; - pending - .inherited_objects - .try_reserve_exact(inherited_objects.len()) - .map_err(|_| BrokerError::OutOfMemory)?; - for handle in inherited_objects { - let child = pending.process.as_ref().ok_or(BrokerError::Internal)?; - let child_handle = - self.duplicate_object_reference_to_preserving_rights(*handle, child)?; - pending.inherited_objects.push(child_handle); + let inherited_result = (|| { + let mut child_handles = Vec::new(); + child_handles + .try_reserve_exact(inherited_objects.len()) + .map_err(|_| BrokerError::OutOfMemory)?; + for handle in inherited_objects { + let child_handle = + self.duplicate_object_reference_to_preserving_rights(*handle, &process)?; + child_handles.push(child_handle); + } + Ok(child_handles) + })(); + match inherited_result { + Ok(child_handles) => Ok((process, child_handles)), + Err(error) => { + BrokerProcess::finish(process); + Err(error) + } } - Ok(pending) } /// Creates a broker thread belonging to this process. @@ -357,15 +297,9 @@ impl BrokerProcess { /// /// Dropping a process without calling this method performs authority /// cleanup but leaves its numeric IDs occupied so they cannot be reused - /// after an unwind. - /// # Panics - /// - /// Panics if another owner still holds this process. + /// after an unwind. Calling this method more than once is harmless. pub fn finish(self: Arc) { - let Ok(mut process) = Arc::try_unwrap(self) else { - panic!("all broker process owners must be released before teardown"); - }; - process.cleanup(true); + self.cleanup(true); } pub(crate) fn create_object_reference(&self, object: ObjectEntry) -> Result { @@ -761,12 +695,13 @@ impl BrokerProcess { Ok(reference) } - fn cleanup(&mut self, release_ids: bool) -> bool { - if self.cleaned_up { + fn cleanup(&self, release_ids: bool) -> bool { + let mut process_state = self.state.lock(); + if *process_state == ProcessState::Exiting { return false; } - self.cleaned_up = true; - *self.state.lock() = ProcessState::Exiting; + *process_state = ProcessState::Exiting; + drop(process_state); let mut invariant_fault = self.references.lock().pending_handles != 0; loop { @@ -974,7 +909,7 @@ mod tests { } #[test] - fn prepared_child_is_parented_and_requires_commit() { + fn child_is_parented_and_requires_commit() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -984,11 +919,9 @@ mod tests { .create_process(CallerCredential::Unauthenticated) .unwrap(); let source_handle = crate::event::create(&parent, 1).unwrap(); - let pending = parent.prepare_child(&[source_handle]).unwrap(); - let child_id = pending.id(); - let inherited_handle = pending.inherited_objects()[0]; - let start_commit = pending.start_commit(); - let child = pending.attach(); + let (child, inherited_objects) = parent.create_child(&[source_handle]).unwrap(); + let child_id = child.id(); + let inherited_handle = inherited_objects[0]; assert_eq!(child.id(), child_id); assert_eq!(child.parent_id(), Some(parent.id())); @@ -999,12 +932,12 @@ mod tests { ); assert!(!child.is_running()); - start_commit.commit().unwrap(); + child.commit_start().unwrap(); assert!(child.is_running()); } #[test] - fn dropped_prepared_child_releases_process_capacity() { + fn finished_uncommitted_child_releases_process_capacity() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1014,37 +947,39 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let pending = parent.prepare_child(&[]).unwrap(); + let (child, _) = parent.create_child(&[]).unwrap(); assert_eq!( - parent.prepare_child(&[]).err(), + parent.create_child(&[]).err(), Some(BrokerError::ResourceExhausted) ); - drop(pending); - assert!(parent.prepare_child(&[]).is_ok()); + child.finish(); + assert!(parent.create_child(&[]).is_ok()); } #[test] - fn prepared_child_cannot_commit_after_teardown() { + fn child_cannot_commit_after_teardown() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) + .with_limits(BrokerCoreLimits::DEFAULT.with_process_limit(2)) .build() .unwrap(); let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let pending = parent.prepare_child(&[]).unwrap(); - let start_commit = pending.start_commit(); - let child = pending.attach(); + let (child, _) = parent.create_child(&[]).unwrap(); - child.finish(); + Arc::clone(&child).finish(); + Arc::clone(&child).finish(); - assert_eq!(start_commit.commit(), Err(BrokerError::PeerClosed)); + assert_eq!(child.commit_start(), Err(BrokerError::PeerClosed)); + let (replacement, _) = parent.create_child(&[]).unwrap(); + replacement.finish(); } #[test] - fn attached_child_releases_process_capacity_after_teardown() { + fn child_releases_process_capacity_after_teardown() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1054,14 +989,14 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let child = parent.prepare_child(&[]).unwrap().attach(); + let (child, _) = parent.create_child(&[]).unwrap(); assert_eq!( - parent.prepare_child(&[]).err(), + parent.create_child(&[]).err(), Some(BrokerError::ResourceExhausted) ); child.finish(); - assert!(parent.prepare_child(&[]).is_ok()); + assert!(parent.create_child(&[]).is_ok()); } #[test] diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index d921079a84..438f988eb2 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -24,7 +24,7 @@ extern crate std; use alloc::{sync::Arc, vec::Vec}; use litebox_broker_core::readiness::ReadinessSink; -use litebox_broker_core::{BrokerCore, BrokerProcess, CallerCredential, PendingProcess}; +use litebox_broker_core::{BrokerCore, BrokerProcess, CallerCredential}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::event::{AddEventResponse, CreateEventResponse}; use litebox_broker_protocol::fs::{ @@ -45,7 +45,7 @@ use litebox_broker_protocol::pipe::{ }; use litebox_broker_protocol::process::{ InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessBootstrapFormat, - ProcessBootstrapVersion, + ProcessBootstrapVersion, ProcessStartup, }; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ @@ -62,7 +62,7 @@ use litebox_broker_protocol::stdio::{ IsTerminalStdioRequest, IsTerminalStdioResponse, MAX_STDIO_TRANSFER_SIZE, ReadStdioRequest, ReadStdioResponse, WriteStdioRequest, WriteStdioResponse, }; -use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId}; +use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, ObjectHandle, RequestId}; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use spin::mutex::SpinMutex; @@ -104,14 +104,16 @@ pub enum BrokerHostExtensionError { Abort(ErrorCode), } -/// Opaque bootstrap staged for one broker-reserved prepared child. -pub struct PreparedProcessBootstrap<'a> { +/// Child startup data staged during broker negotiation. +pub struct ProcessStartupData { /// Platform-defined format. pub format: ProcessBootstrapFormat, /// Version within the platform-defined format. pub version: ProcessBootstrapVersion, /// Opaque platform bytes. - pub payload: &'a [u8], + pub payload: Vec, + /// Child-owned broker handles in the parent's inheritance-manifest order. + pub inherited_objects: Vec, } impl BrokerHostAssociation<'_, Memory> { @@ -241,6 +243,8 @@ pub fn copy_shared_buffer( /// association is returned. pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( core: &BrokerCore, + process: Option>, + startup: Option, setup_channel: &mut SetupChannel, shared_buffers: &'a SharedBufferPool, readiness_sink: Arc, @@ -253,6 +257,37 @@ where if shared_buffers.layout() != SHARED_BUFFER_LAYOUT { return Err(BrokerHostError::SharedBufferLayoutMismatch); } + if process.is_some() != startup.is_some() { + return Err(BrokerHostError::Broker(ErrorCode::Internal)); + } + let startup = match startup { + Some(ProcessStartupData { + format, + version, + payload, + inherited_objects, + }) => { + if payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { + return Err(BrokerHostError::Broker(ErrorCode::ResourceExhausted)); + } + let bootstrap_length = u32::try_from(payload.len()) + .map_err(|_| BrokerHostError::Broker(ErrorCode::ResourceExhausted))?; + let buffer = SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], bootstrap_length) + .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; + write_shared_buffer(shared_buffers, buffer, &payload, MAX_PROCESS_BOOTSTRAP_SIZE) + .map_err(|error| BrokerHostError::Broker(request_failure_error(error)))?; + Some(ProcessStartup { + bootstrap: ProcessBootstrap { + format, + version, + buffer, + }, + inherited_objects: InheritedProcessObjects::new(&inherited_objects) + .ok_or(BrokerHostError::Broker(ErrorCode::Internal))?, + }) + } + None => None, + }; let limits = core.limits(); // Sockets are currently the only externally backed objects. Add future // resource limits here so every live registration fits in the @@ -270,6 +305,7 @@ where PeerCredential::Unauthenticated => CallerCredential::Unauthenticated, _ => return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)), }; + let mut process = process; loop { let request = match setup_channel .recv_handshake_request() @@ -299,119 +335,49 @@ where continue; } - let process = match core.create_process(caller_credential) { - Ok(process) => process, - Err(litebox_broker_core::BrokerError::ResourceExhausted) => { - let error = ErrorCode::ResourceExhausted; + let finish_on_setup_error = process.is_none(); + let process = match process.take() { + Some(process) if process.caller_credential() == caller_credential => process, + Some(_) => { + let error = ErrorCode::PolicyDenied; setup_channel .send_handshake_response(&BrokerHandshakeResponse::Error(error)) .map_err(BrokerHostError::Channel)?; return Ok(Err(ConnectionTermination::Rejected(error))); } - Err(error) => return Err(BrokerHostError::from(error)), + None => match core.create_process(caller_credential) { + Ok(process) => process, + Err(litebox_broker_core::BrokerError::ResourceExhausted) => { + let error = ErrorCode::ResourceExhausted; + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::Error(error)) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::Rejected(error))); + } + Err(error) => return Err(BrokerHostError::from(error)), + }, }; let response = BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: process.id(), + startup, }; if let Err(error) = setup_channel.send_handshake_response(&response) { - BrokerProcess::finish(process); + if finish_on_setup_error { + BrokerProcess::finish(process); + } return Err(BrokerHostError::Channel(error)); } if let Err(error) = send_shared_memory(setup_channel) { - BrokerProcess::finish(process); + if finish_on_setup_error { + BrokerProcess::finish(process); + } return Err(BrokerHostError::Channel(error)); } return Ok(Ok(new_association(process, shared_buffers, readiness_sink))); } } -/// Authenticates and negotiates one broker-reserved prepared child connection. -pub fn setup_prepared_connection<'a, SetupChannel, Memory, ChannelError>( - pending: PendingProcess, - setup_channel: &mut SetupChannel, - shared_buffers: &'a SharedBufferPool, - readiness_sink: Arc, - bootstrap: PreparedProcessBootstrap<'_>, - send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, -) -> Result, ChannelError> -where - SetupChannel: HostSetupChannel, - Memory: SharedMemory, -{ - if shared_buffers.layout() != SHARED_BUFFER_LAYOUT { - return Err(BrokerHostError::SharedBufferLayoutMismatch); - } - if bootstrap.payload.len() > MAX_PROCESS_BOOTSTRAP_SIZE as usize { - return Err(BrokerHostError::Broker(ErrorCode::ResourceExhausted)); - } - let process_id = pending.id(); - let transport_credential = match setup_channel - .peer_credential() - .map_err(BrokerHostError::Channel)? - { - PeerCredential::HostGuaranteed => CallerCredential::HostGuaranteed, - PeerCredential::Unauthenticated => CallerCredential::Unauthenticated, - _ => return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)), - }; - if transport_credential != pending.caller_credential() { - return Err(BrokerHostError::Broker(ErrorCode::PolicyDenied)); - } - let request = match setup_channel - .recv_handshake_request() - .map_err(BrokerHostError::Channel)? - { - HostReceive::Message(request) => request, - HostReceive::ProtocolViolation => { - setup_channel - .send_handshake_response(&BrokerHandshakeResponse::Error(ErrorCode::ProtocolState)) - .map_err(BrokerHostError::Channel)?; - return Ok(Err(ConnectionTermination::ProtocolViolation)); - } - HostReceive::PeerClosed => return Ok(Err(ConnectionTermination::PeerClosed)), - }; - if request.protocol_version != BROKER_PROTOCOL_VERSION { - setup_channel - .send_handshake_response(&BrokerHandshakeResponse::VersionMismatch { - broker_protocol_version: BROKER_PROTOCOL_VERSION, - }) - .map_err(BrokerHostError::Channel)?; - return Ok(Err(ConnectionTermination::Rejected( - ErrorCode::UnsupportedVersion, - ))); - } - - let bootstrap_length = u32::try_from(bootstrap.payload.len()) - .map_err(|_| BrokerHostError::Broker(ErrorCode::ResourceExhausted))?; - let buffer = SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], bootstrap_length) - .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; - write_shared_buffer( - shared_buffers, - buffer, - bootstrap.payload, - MAX_PROCESS_BOOTSTRAP_SIZE, - ) - .map_err(|error| BrokerHostError::Broker(request_failure_error(error)))?; - - let response = BrokerHandshakeResponse::Prepared { - broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id, - bootstrap: ProcessBootstrap { - format: bootstrap.format, - version: bootstrap.version, - buffer, - }, - inherited_objects: InheritedProcessObjects::new(pending.inherited_objects()) - .ok_or(BrokerHostError::Broker(ErrorCode::Internal))?, - }; - setup_channel - .send_handshake_response(&response) - .map_err(BrokerHostError::Channel)?; - send_shared_memory(setup_channel).map_err(BrokerHostError::Channel)?; - let process = pending.attach(); - Ok(Ok(new_association(process, shared_buffers, readiness_sink))) -} - fn new_association( process: Arc, shared_buffers: &SharedBufferPool, @@ -1914,6 +1880,7 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: root_process_id(1), + startup: None, } ); let handle = match &channel.results[0] { @@ -1949,6 +1916,7 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: root_process_id(2), + startup: None, } ] ); @@ -2022,6 +1990,7 @@ mod tests { [BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: root_process_id(4), + startup: None, }] ); assert!(channel.results.is_empty()); @@ -2807,6 +2776,8 @@ mod tests { ) -> Result { let association = match setup_connection( broker, + None, + None, control_channel, shared_buffers, test_readiness_sink(), diff --git a/litebox_broker_host/src/test_support.rs b/litebox_broker_host/src/test_support.rs index 5a9911b4da..f79398c0cd 100644 --- a/litebox_broker_host/src/test_support.rs +++ b/litebox_broker_host/src/test_support.rs @@ -98,6 +98,8 @@ impl LocalSetupChannel for InProcessBrokerSetup { let readiness: Arc = self.readiness.clone(); let association = crate::setup_connection( &self.broker, + None, + None, &mut host_setup, shared_buffers, readiness, diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index f719a174ec..b4bd1b5eeb 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -71,9 +71,9 @@ pub struct BrokerNotifications { channel: Channel, } -/// Opaque bootstrap delivered to one broker-reserved prepared process. +/// Child startup data delivered during broker negotiation. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct PreparedProcessBootstrap { +pub struct ProcessStartupData { /// Platform-defined format. pub format: ProcessBootstrapFormat, /// Version within the platform-defined format. @@ -104,6 +104,8 @@ impl BrokerLocal { /// after negotiation, such as receiving shared memory and starting the /// active transport. Any additional endpoints activation produces, such as /// a notification receiver, are returned to the caller as `Activated`. + /// Child startup data is copied from the negotiated shared-buffer pool and + /// returned when present. /// /// # Panics /// @@ -118,7 +120,7 @@ impl BrokerLocal { (Channel, Arc, Activated), Channel::Error, >, - ) -> Result<(Self, Activated), Channel::Error> { + ) -> Result<(Self, Option, Activated), Channel::Error> { let requested = BROKER_PROTOCOL_VERSION; let request = BrokerHandshakeRequest { protocol_version: requested, @@ -134,6 +136,7 @@ impl BrokerLocal { response @ BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + startup, } => { assert_eq!( requested, broker_protocol_version, @@ -141,10 +144,25 @@ impl BrokerLocal { ); let (channel, shared_memory, activated) = activate(setup).map_err(BrokerLocalError::Channel)?; - Ok((Self::new(channel, process_id, shared_memory), activated)) - } - response @ BrokerHandshakeResponse::Prepared { .. } => { - panic!("broker returned unexpected negotiation response: {response:?}") + let local = Self::new(channel, process_id, shared_memory); + let startup = match startup { + Some(startup) => { + let mut payload = Vec::new(); + payload + .try_reserve_exact(startup.bootstrap.buffer.length() as usize) + .map_err(|_| BrokerLocalError::Broker(ErrorCode::OutOfMemory))?; + payload.resize(startup.bootstrap.buffer.length() as usize, 0); + local.read_shared_buffer(startup.bootstrap.buffer, &mut payload); + Some(ProcessStartupData { + format: startup.bootstrap.format, + version: startup.bootstrap.version, + payload, + inherited_objects: startup.inherited_objects, + }) + } + None => None, + }; + Ok((local, startup, activated)) } BrokerHandshakeResponse::VersionMismatch { .. } => { Err(BrokerLocalError::Broker(ErrorCode::UnsupportedVersion)) @@ -162,72 +180,6 @@ impl BrokerLocal { } } - /// Negotiates one broker-reserved prepared process and copies its bootstrap. - /// - /// # Panics - /// - /// Panics if the broker returns a response inconsistent with prepared - /// negotiation or setup returns shared memory with an invalid size. - pub fn negotiate_prepared, Activated>( - mut setup: Setup, - activate: impl FnOnce( - Setup, - ) -> core::result::Result< - (Channel, Arc, Activated), - Channel::Error, - >, - ) -> Result<(Self, PreparedProcessBootstrap, Activated), Channel::Error> { - let requested = BROKER_PROTOCOL_VERSION; - setup - .send_handshake_request(&BrokerHandshakeRequest { - protocol_version: requested, - }) - .map_err(BrokerLocalError::Channel)?; - match setup - .recv_handshake_response() - .map_err(BrokerLocalError::Channel)? - .ok_or(BrokerLocalError::ChannelClosed)? - { - response @ BrokerHandshakeResponse::Prepared { - broker_protocol_version, - process_id, - bootstrap, - inherited_objects, - } => { - assert_eq!( - requested, broker_protocol_version, - "broker returned unexpected prepared negotiation response: {response:?}" - ); - let (channel, shared_memory, activated) = - activate(setup).map_err(BrokerLocalError::Channel)?; - let local = Self::new(channel, process_id, shared_memory); - let mut payload = Vec::new(); - payload - .try_reserve_exact(bootstrap.buffer.length() as usize) - .map_err(|_| BrokerLocalError::Broker(ErrorCode::OutOfMemory))?; - payload.resize(bootstrap.buffer.length() as usize, 0); - local.read_shared_buffer(bootstrap.buffer, &mut payload); - Ok(( - local, - PreparedProcessBootstrap { - format: bootstrap.format, - version: bootstrap.version, - payload, - inherited_objects, - }, - activated, - )) - } - response @ BrokerHandshakeResponse::Negotiated { .. } => { - panic!("broker returned unexpected prepared negotiation response: {response:?}") - } - BrokerHandshakeResponse::VersionMismatch { .. } => { - Err(BrokerLocalError::Broker(ErrorCode::UnsupportedVersion)) - } - BrokerHandshakeResponse::Error(error) => Err(BrokerLocalError::Broker(error)), - } - } - /// Returns the assigned process ID. #[must_use] pub const fn process_id(&self) -> ProcessId { @@ -414,6 +366,7 @@ mod tests { use core::cell::{Cell, RefCell}; use core::convert::Infallible; use litebox_broker_protocol::message::{ReadinessNotification, StdioRequest, StdioResponse}; + use litebox_broker_protocol::process::{ProcessBootstrap, ProcessStartup}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use litebox_broker_protocol::stdio::{ @@ -436,11 +389,12 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + startup: None, }), None, ); let setup_calls = Cell::new(0); - let (local, ()) = BrokerLocal::negotiate(channel, |channel| { + let (local, startup, ()) = BrokerLocal::negotiate(channel, |channel| { assert!(channel.sent_handshake_request.is_some()); assert!(channel.handshake_response.is_none()); assert!(channel.sent_request.borrow().is_none()); @@ -456,9 +410,40 @@ mod tests { }) ); assert_eq!(setup_calls.get(), 1); + assert!(startup.is_none()); assert_eq!(local.process_id(), test_process_id()); } + #[test] + fn negotiate_returns_child_startup_data() { + let inherited_objects = InheritedProcessObjects::new(&[ObjectHandle(7)]).unwrap(); + let channel = FakeControlChannel::new( + Some(BrokerHandshakeResponse::Negotiated { + broker_protocol_version: BROKER_PROTOCOL_VERSION, + process_id: test_process_id(), + startup: Some(ProcessStartup { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(3), + version: ProcessBootstrapVersion(4), + buffer: SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], 5).unwrap(), + }, + inherited_objects, + }), + }), + None, + ); + + let (_local, startup, ()) = + BrokerLocal::negotiate(channel, |channel| Ok((channel, noop_shared_memory(), ()))) + .unwrap(); + let startup = startup.unwrap(); + + assert_eq!(startup.format, ProcessBootstrapFormat(3)); + assert_eq!(startup.version, ProcessBootstrapVersion(4)); + assert_eq!(startup.payload, [0; 5]); + assert_eq!(startup.inherited_objects, inherited_objects); + } + #[test] fn close_object_sends_close_object_request() { let handle = ObjectHandle(7); @@ -698,6 +683,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id: test_process_id(), + startup: None, }), None, ); @@ -790,6 +776,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + startup: None, }), None, ); @@ -813,6 +800,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + startup: None, }), None, ); diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs index a53258cd47..6641a76ef7 100644 --- a/litebox_broker_local/src/process.rs +++ b/litebox_broker_local/src/process.rs @@ -69,7 +69,7 @@ impl BrokerLocal { } } - /// Reports that this prepared process is ready and waits for parent acknowledgement. + /// Reports that this child process is ready and waits for parent acknowledgement. /// /// # Panics /// diff --git a/litebox_broker_local_userland/src/lib.rs b/litebox_broker_local_userland/src/lib.rs index c7eff35fbe..ec30a0f860 100644 --- a/litebox_broker_local_userland/src/lib.rs +++ b/litebox_broker_local_userland/src/lib.rs @@ -9,11 +9,11 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::{ - BrokerAssociationFailureCoordinator, BrokerConnection, connect, connect_prepared, + BrokerAssociationFailureCoordinator, BrokerConnection, connect, connect_child, start_notification_receiver, }; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(all(windows, target_arch = "x86_64"))] -pub use windows::{BrokerConnection, connect, connect_prepared, start_notification_receiver}; +pub use windows::{BrokerConnection, connect, connect_child, start_notification_receiver}; diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index e442dbd7a7..84b8f53cc9 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -12,7 +12,7 @@ use std::{ }; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications, PreparedProcessBootstrap}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications, ProcessStartupData}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; @@ -40,64 +40,24 @@ pub struct BrokerConnection { /// Connects to and negotiates an association with a Linux-userland broker. pub fn connect(control_socket_path: &Path) -> Result { - let setup_deadline = Instant::now() + SETUP_TIMEOUT; - let setup_channel = connect_with_retry( - control_socket_path, - setup_deadline, - "timed out connecting to broker", - |path, deadline| UnixStreamLocalSetupChannel::connect_with_setup_deadline(path, deadline), - ) - .with_context(|| { - format!( - "failed to connect to broker at {}", - control_socket_path.display() - ) - })?; - let association_coordinator = Arc::new(BrokerAssociationFailureCoordinator::new()); - let (local, (notification_channel, positional_io_fds, shutdown_fd)) = - BrokerLocal::negotiate(setup_channel, |mut setup| { - let shared_memory = - setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; - let control_memory = setup.receive_control_ring(Some(setup_deadline))?; - let positional_io_fds = [ - shared_memory.as_fd().as_raw_fd(), - control_memory.as_fd().as_raw_fd(), - ]; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("invalid broker control ring: {error:?}"), - ) - })?; - let weak_association_coordinator = Arc::downgrade(&association_coordinator); - let (call_channel, notification_channel, association_shutdown) = - setup.into_active(control_ring, move || { - if let Some(association_coordinator) = weak_association_coordinator.upgrade() { - association_coordinator.report_failure(); - } - })?; - let shutdown_fd = association_shutdown.as_fd().as_raw_fd(); - association_coordinator.install_shutdown(association_shutdown)?; - Ok(( - call_channel, - Arc::new(shared_memory), - (notification_channel, positional_io_fds, shutdown_fd), - )) - }) - .context("broker negotiation failed")?; - Ok(BrokerConnection { - local, - notifications: BrokerNotifications::new(notification_channel), - coordinator: association_coordinator, - positional_io_fds, - shutdown_fd, - }) + let (connection, startup) = connect_with_startup(control_socket_path)?; + if startup.is_some() { + anyhow::bail!("initial broker association returned child startup data"); + } + Ok(connection) +} + +/// Connects a child runner and receives its startup data. +pub fn connect_child(control_socket_path: &Path) -> Result<(BrokerConnection, ProcessStartupData)> { + let (connection, startup) = connect_with_startup(control_socket_path)?; + let startup = + startup.ok_or_else(|| anyhow::anyhow!("child broker association omitted startup data"))?; + Ok((connection, startup)) } -/// Connects to and negotiates a broker-reserved prepared Linux process. -pub fn connect_prepared( +fn connect_with_startup( control_socket_path: &Path, -) -> Result<(BrokerConnection, PreparedProcessBootstrap)> { +) -> Result<(BrokerConnection, Option)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let setup_channel = connect_with_retry( control_socket_path, @@ -112,8 +72,8 @@ pub fn connect_prepared( ) })?; let association_coordinator = Arc::new(BrokerAssociationFailureCoordinator::new()); - let (local, bootstrap, (notification_channel, positional_io_fds, shutdown_fd)) = - BrokerLocal::negotiate_prepared(setup_channel, |mut setup| { + let (local, startup, (notification_channel, positional_io_fds, shutdown_fd)) = + BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, Some(setup_deadline))?; let control_memory = setup.receive_control_ring(Some(setup_deadline))?; @@ -142,7 +102,7 @@ pub fn connect_prepared( (notification_channel, positional_io_fds, shutdown_fd), )) }) - .context("prepared broker negotiation failed")?; + .context("broker negotiation failed")?; Ok(( BrokerConnection { local, @@ -151,7 +111,7 @@ pub fn connect_prepared( positional_io_fds, shutdown_fd, }, - bootstrap, + startup, )) } @@ -323,6 +283,7 @@ mod tests { &litebox_broker_protocol::message::BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }, ) .unwrap(); diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index 8dc3bae983..e122be3506 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications, PreparedProcessBootstrap}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications, ProcessStartupData}; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; @@ -27,38 +27,24 @@ pub struct BrokerConnection { /// Connects to and negotiates an association with a Windows-userland broker. pub fn connect(control_pipe: &OsStr) -> Result { - let deadline = Instant::now() + SETUP_TIMEOUT; - let setup = - WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(control_pipe, deadline) - .with_context(|| { - format!( - "failed to connect to broker at {}", - std::path::Path::new(control_pipe).display() - ) - })?; - let (local, notifications) = BrokerLocal::negotiate(setup, |mut setup| { - let shared_memory = Arc::new(setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?); - let control_memory = setup.receive_control_ring()?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("invalid broker control ring: {error:?}"), - ) - })?; - let (calls, notifications) = setup.into_active(control_ring)?; - Ok((calls, shared_memory, notifications)) - }) - .context("broker negotiation failed")?; - Ok(BrokerConnection { - local, - notifications: BrokerNotifications::new(notifications), - }) + let (connection, startup) = connect_with_startup(control_pipe)?; + if startup.is_some() { + anyhow::bail!("initial broker association returned child startup data"); + } + Ok(connection) } -/// Connects to and negotiates a broker-reserved prepared Windows process. -pub fn connect_prepared( +/// Connects a child runner and receives its startup data. +pub fn connect_child(control_pipe: &OsStr) -> Result<(BrokerConnection, ProcessStartupData)> { + let (connection, startup) = connect_with_startup(control_pipe)?; + let startup = + startup.ok_or_else(|| anyhow::anyhow!("child broker association omitted startup data"))?; + Ok((connection, startup)) +} + +fn connect_with_startup( control_pipe: &OsStr, -) -> Result<(BrokerConnection, PreparedProcessBootstrap)> { +) -> Result<(BrokerConnection, Option)> { let deadline = Instant::now() + SETUP_TIMEOUT; let setup = WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(control_pipe, deadline) @@ -68,7 +54,7 @@ pub fn connect_prepared( std::path::Path::new(control_pipe).display() ) })?; - let (local, bootstrap, notifications) = BrokerLocal::negotiate_prepared(setup, |mut setup| { + let (local, startup, notifications) = BrokerLocal::negotiate(setup, |mut setup| { let shared_memory = Arc::new(setup.receive_shared_memory(SHARED_BUFFER_POOL_SIZE)?); let control_memory = setup.receive_control_ring()?; let control_ring = ControlRing::new(control_memory).map_err(|error| { @@ -80,13 +66,13 @@ pub fn connect_prepared( let (calls, notifications) = setup.into_active(control_ring)?; Ok((calls, shared_memory, notifications)) }) - .context("prepared broker negotiation failed")?; + .context("broker negotiation failed")?; Ok(( BrokerConnection { local, notifications: BrokerNotifications::new(notifications), }, - bootstrap, + startup, )) } diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index d051c085c2..ed23c41d45 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -18,8 +18,8 @@ use crate::pipe::{ WritePipeResponse, }; use crate::process::{ - InheritedProcessObjects, ProcessBootstrap, ProcessReadyRequest, ProcessStartToken, - StartProcessRequest, StartedProcess, + ProcessBootstrap, ProcessReadyRequest, ProcessStartToken, ProcessStartup, StartProcessRequest, + StartedProcess, }; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; @@ -70,9 +70,9 @@ pub enum BrokerOperation { File(FileRequest), /// Start one child process from an opaque platform bootstrap. StartProcess(StartProcessRequest), - /// Commit a prepared process after its start result reaches the parent. + /// Commit a child process after its start result reaches the parent. AcknowledgeProcessStart(ProcessStartToken), - /// Report that this prepared process is ready to begin guest execution. + /// Report that this child process is ready to begin guest execution. ProcessReady(ProcessReadyRequest), } @@ -160,17 +160,8 @@ pub enum BrokerHandshakeResponse { broker_protocol_version: ProtocolVersion, /// Assigned process ID. process_id: ProcessId, - }, - /// Negotiation result for a broker-reserved prepared child. - Prepared { - /// Broker protocol version supported by this endpoint. - broker_protocol_version: ProtocolVersion, - /// Broker-assigned child process ID. - process_id: ProcessId, - /// Opaque platform bootstrap staged in the child's shared-buffer pool. - bootstrap: ProcessBootstrap, - /// Child-owned broker handles corresponding to the parent's inheritance manifest. - inherited_objects: InheritedProcessObjects, + /// Child startup data, absent for the initial process. + startup: Option, }, /// Negotiation failed because the requested version is unsupported. /// @@ -263,9 +254,9 @@ pub enum BrokerResult { File(FileResponse), /// A child was materialized and is ready for parent acknowledgement. ProcessStarted(StartedProcess), - /// Parent acknowledgement committed the prepared child. + /// Parent acknowledgement committed the child. ProcessStartAcknowledged, - /// Parent acknowledgement released this prepared child. + /// Parent acknowledgement released the child. ProcessReady, /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index a770a582be..b25952a41b 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -20,12 +20,12 @@ pub struct ProcessBootstrapFormat(pub u32); #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ProcessBootstrapVersion(pub u16); -/// Association-scoped token for acknowledging a prepared process start. +/// Association-scoped token for acknowledging a child process start. #[repr(transparent)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ProcessStartToken(pub u64); -/// Ordered broker-object handles inherited by a prepared child. +/// Ordered broker-object handles inherited by a child. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct InheritedProcessObjects { handles: [ObjectHandle; MAX_INHERITED_PROCESS_OBJECTS], @@ -73,6 +73,15 @@ pub struct ProcessBootstrap { pub buffer: SharedBufferSequence, } +/// Child startup data delivered during broker negotiation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessStartup { + /// Opaque platform bootstrap staged in the child's shared-buffer pool. + pub bootstrap: ProcessBootstrap, + /// Child-owned broker handles in the parent's inheritance-manifest order. + pub inherited_objects: InheritedProcessObjects, +} + /// Starts one child process from an opaque platform bootstrap. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartProcessRequest { @@ -93,7 +102,7 @@ pub struct StartedProcess { pub initial_thread_id: Option, } -/// Reports that a prepared child finished restoring its initial state. +/// Reports that a child finished restoring its initial state. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ProcessReadyRequest { /// Broker-assigned initial thread ID when it differs from the process ID. diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index d6e75bf176..b45448baa2 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -12,8 +12,7 @@ //! //! New object families should add a top-level broker message tag and a private //! family codec module instead of adding flat helpers here. Existing payloads -//! are positional; changing fields is an ABI change, so prefer a new operation -//! tag or explicit negotiated-version gate for payload evolution. +//! are positional, so update both endpoints and their codec tests together. use alloc::vec::Vec; use thiserror::Error; @@ -26,7 +25,7 @@ use crate::message::{ use crate::process::{ InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrap, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessReadyRequest, ProcessStartToken, - StartProcessRequest, StartedProcess, + ProcessStartup, StartProcessRequest, StartedProcess, }; use crate::readiness::ReadinessFlags; @@ -72,7 +71,6 @@ const RESPONSE_TAG_PROCESS_READY: u8 = 13; // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; -const RESPONSE_TAG_PREPARED: u8 = 252; const RESPONSE_TAG_HANDSHAKE_ERROR: u8 = 254; const RESPONSE_TAG_VERSION_MISMATCH: u8 = 255; @@ -296,29 +294,29 @@ pub fn encode_handshake_response(response: BrokerHandshakeResponse) -> Vec { BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + startup, } => { encoder.u8(RESPONSE_TAG_NEGOTIATED); encoder.protocol_version(broker_protocol_version); encoder.process_id(process_id); - } - BrokerHandshakeResponse::Prepared { - broker_protocol_version, - process_id, - bootstrap: - ProcessBootstrap { - format, - version, - buffer, - }, - inherited_objects, - } => { - encoder.u8(RESPONSE_TAG_PREPARED); - encoder.protocol_version(broker_protocol_version); - encoder.process_id(process_id); - encoder.u32(format.0); - encoder.u16(version.0); - encoder.shared_buffer_sequence(buffer); - encode_inherited_objects(&mut encoder, inherited_objects); + match startup { + Some(ProcessStartup { + bootstrap: + ProcessBootstrap { + format, + version, + buffer, + }, + inherited_objects, + }) => { + encoder.u8(1); + encoder.u32(format.0); + encoder.u16(version.0); + encoder.shared_buffer_sequence(buffer); + encode_inherited_objects(&mut encoder, inherited_objects); + } + None => encoder.u8(0), + } } BrokerHandshakeResponse::VersionMismatch { broker_protocol_version, @@ -342,16 +340,18 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result BrokerHandshakeResponse::Negotiated { broker_protocol_version: decoder.protocol_version()?, process_id: decoder.process_id()?, - }, - RESPONSE_TAG_PREPARED => BrokerHandshakeResponse::Prepared { - broker_protocol_version: decoder.protocol_version()?, - process_id: decoder.process_id()?, - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(decoder.u32()?), - version: ProcessBootstrapVersion(decoder.u16()?), - buffer: decoder.shared_buffer_sequence()?, + startup: match decoder.u8()? { + 0 => None, + 1 => Some(ProcessStartup { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + }, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }), + _ => return Err(WireError::InvalidTag), }, - inherited_objects: decode_inherited_objects(&mut decoder)?, }, RESPONSE_TAG_EVENT | RESPONSE_TAG_OBJECT_CLOSED @@ -469,10 +469,7 @@ pub fn decode_response(frame: &[u8]) -> Result { let mut decoder = Decoder::new(frame); let tag = decoder.u8()?; match tag { - RESPONSE_TAG_NEGOTIATED - | RESPONSE_TAG_PREPARED - | RESPONSE_TAG_HANDSHAKE_ERROR - | RESPONSE_TAG_VERSION_MISMATCH => { + RESPONSE_TAG_NEGOTIATED | RESPONSE_TAG_HANDSHAKE_ERROR | RESPONSE_TAG_VERSION_MISMATCH => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_EVENT @@ -647,7 +644,8 @@ mod tests { }; use crate::process::{ InheritedProcessObjects, ProcessBootstrap, ProcessBootstrapFormat, ProcessBootstrapVersion, - ProcessReadyRequest, ProcessStartToken, StartProcessRequest, StartedProcess, + ProcessReadyRequest, ProcessStartToken, ProcessStartup, StartProcessRequest, + StartedProcess, }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ @@ -730,11 +728,10 @@ mod tests { assert_eq!( [ RESPONSE_TAG_ERROR, - RESPONSE_TAG_PREPARED, RESPONSE_TAG_HANDSHAKE_ERROR, RESPONSE_TAG_VERSION_MISMATCH, ], - [253, 252, 254, 255] + [253, 254, 255] ); } @@ -1188,24 +1185,23 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + startup: None, }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(7), - }, - BrokerHandshakeResponse::Prepared { - broker_protocol_version: ProtocolVersion(1), - process_id: process_id(9), - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(0x7465_7374), - version: ProcessBootstrapVersion(1), - buffer: sequence(0, 37), - }, - inherited_objects: InheritedProcessObjects::new(&[ - ObjectHandle(5), - ObjectHandle(6), - ]) - .unwrap(), + startup: Some(ProcessStartup { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(0x7465_7374), + version: ProcessBootstrapVersion(1), + buffer: sequence(0, 37), + }, + inherited_objects: InheritedProcessObjects::new(&[ + ObjectHandle(5), + ObjectHandle(6), + ]) + .unwrap(), + }), }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), @@ -1875,10 +1871,19 @@ mod tests { Err(WireError::WrongMessagePhase) ); - let mut frame = encode_handshake_response(BrokerHandshakeResponse::Negotiated { + let negotiated = BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), - }); + startup: None, + }; + let mut invalid_startup = encode_handshake_response(negotiated.clone()); + *invalid_startup.last_mut().unwrap() = 2; + assert_eq!( + decode_handshake_response(&invalid_startup), + Err(WireError::InvalidTag) + ); + + let mut frame = encode_handshake_response(negotiated); frame.push(0xff); assert_eq!( decode_handshake_response(&frame), @@ -1896,16 +1901,19 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + startup: None, }, - BrokerHandshakeResponse::Prepared { + BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(2), - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(3), - version: ProcessBootstrapVersion(4), - buffer: sequence(0, 5), - }, - inherited_objects: InheritedProcessObjects::EMPTY, + startup: Some(ProcessStartup { + bootstrap: ProcessBootstrap { + format: ProcessBootstrapFormat(3), + version: ProcessBootstrapVersion(4), + buffer: sequence(0, 5), + }, + inherited_objects: InheritedProcessObjects::EMPTY, + }), }, BrokerHandshakeResponse::VersionMismatch { broker_protocol_version: ProtocolVersion(1), diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 501eefd431..36e8cb8264 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -286,10 +286,7 @@ impl HostSetupChannel for UnixStreamHostSetupChannel { &encode_handshake_response(response.clone()), self.setup_deadline, )?; - self.negotiated = matches!( - response, - BrokerHandshakeResponse::Negotiated { .. } | BrokerHandshakeResponse::Prepared { .. } - ); + self.negotiated = matches!(response, BrokerHandshakeResponse::Negotiated { .. }); Ok(()) } } @@ -616,6 +613,7 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); assert!(matches!( diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 3de72a6c75..03b5df7bd1 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -307,11 +307,7 @@ impl LocalSetupChannel for UnixStreamLocalSetupChannel { match frame { Some(frame) => { let response = decode_handshake_response(&frame).map_err(wire_error)?; - self.negotiated = matches!( - &response, - BrokerHandshakeResponse::Negotiated { .. } - | BrokerHandshakeResponse::Prepared { .. } - ); + self.negotiated = matches!(&response, BrokerHandshakeResponse::Negotiated { .. }); Ok(Some(response)) } None => Ok(None), @@ -712,6 +708,7 @@ mod control_ring_tests { &encode_handshake_response(BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }), None, ) diff --git a/litebox_broker_transport_windows_userland/src/host.rs b/litebox_broker_transport_windows_userland/src/host.rs index 8f1487fa4a..deff03c523 100644 --- a/litebox_broker_transport_windows_userland/src/host.rs +++ b/litebox_broker_transport_windows_userland/src/host.rs @@ -158,10 +158,7 @@ impl HostSetupChannel for WindowsNamedPipeHostSetupChannel { &encode_handshake_response(response.clone()), self.setup_deadline, )?; - self.negotiated = matches!( - response, - BrokerHandshakeResponse::Negotiated { .. } | BrokerHandshakeResponse::Prepared { .. } - ); + self.negotiated = matches!(response, BrokerHandshakeResponse::Negotiated { .. }); Ok(()) } } diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index b474177ff0..96d263d6ae 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -158,13 +158,7 @@ impl LocalSetupChannel for WindowsNamedPipeLocalSetupChannel { let response = read_frame(file_handle(&self.stream), self.setup_deadline)? .map(|frame| decode_handshake_response(&frame).map_err(wire_error)) .transpose()?; - self.negotiated = matches!( - response, - Some( - BrokerHandshakeResponse::Negotiated { .. } - | BrokerHandshakeResponse::Prepared { .. } - ) - ); + self.negotiated = matches!(response, Some(BrokerHandshakeResponse::Negotiated { .. })); Ok(response) } } diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index a02f987ec2..a098d6437a 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -295,6 +295,7 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); let memory = WindowsSharedMemory::create(4096).unwrap(); @@ -337,6 +338,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); let (mut requests, responses, mut notifications, _shutdown) = @@ -413,6 +415,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); let (mut requests, responses, _notifications, _shutdown) = @@ -485,6 +488,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); let (mut requests, responses, _notifications, shutdown) = @@ -563,6 +567,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); let (mut requests, _responses, _notifications, shutdown) = diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 4c01eb6282..4880ec301f 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -10,7 +10,7 @@ use std::process::{Child, Command, ExitStatus}; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; -use litebox_broker_core::{BrokerCore, BrokerProcess, PendingProcess, ProcessStartCommit}; +use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{BrokerHostExtensionError, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; @@ -18,7 +18,7 @@ use litebox_broker_protocol::process::{ MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartToken, StartedProcess, }; -use litebox_broker_protocol::{ProcessId, ThreadId}; +use litebox_broker_protocol::{ObjectHandle, ProcessId, ThreadId}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; #[cfg(target_os = "linux")] @@ -33,14 +33,14 @@ use windows::PlatformRunnerEndpoint; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); -const PREPARED_CHILD_ARGUMENT: &str = "--prepared-child"; +const CHILD_ARGUMENT: &str = "--child"; const MAX_PENDING_CHILD_STARTS: usize = crate::WORKER_COUNT - 1; const _: () = assert!(MAX_PENDING_CHILD_STARTS > 0); /// Configuration for starting one out-of-process runner. /// /// Dynamically started descendants use the same executable with the hidden -/// `--prepared-child` argument instead of the root arguments. +/// `--child` argument instead of the root arguments. #[derive(Clone)] pub struct RunnerConfig { executable: PathBuf, @@ -81,10 +81,10 @@ impl RunnerConfig { arguments } - fn prepared_child(&self) -> Self { + fn child(&self) -> Self { Self { executable: self.executable.clone(), - arguments: vec![OsString::from(PREPARED_CHILD_ARGUMENT)], + arguments: vec![OsString::from(CHILD_ARGUMENT)], proxy_url: self.proxy_url.clone(), } } @@ -107,7 +107,7 @@ impl RunnerInstance { let runner = Command::new(&config.executable) .args(config.arguments(endpoint.control_channel())) .spawn()?; - let child_config = config.prepared_child(); + let child_config = config.child(); Ok(Self { runner, endpoint, @@ -121,10 +121,8 @@ impl RunnerInstance { /// non-successful runner exit is returned as ordinary instance data for the /// caller to interpret. pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { - let children = RunnerChildren::new(self.child_config.clone()); - let association_result = - self.endpoint - .serve(broker, &mut self.runner, Arc::clone(&children)); + let children = RunnerChildren::new(self.child_config.clone(), broker.clone()); + let association_result = self.endpoint.serve(&mut self.runner, Arc::clone(&children)); self.endpoint.close(); if association_result.is_err() { let _ = self.runner.kill(); @@ -136,14 +134,12 @@ impl RunnerInstance { Ok(runner_status) } - fn run_prepared_to_completion( + fn run_child_to_completion( mut self, - prepared: PreparedRunner, + child: ChildRunner, children: Arc, ) -> IoResult { - let association_result = self - .endpoint - .serve_prepared(&mut self.runner, prepared, children); + let association_result = self.endpoint.serve_child(&mut self.runner, child, children); self.endpoint.close(); if association_result.is_err() { let _ = self.runner.kill(); @@ -165,13 +161,15 @@ impl Drop for RunnerInstance { } pub(crate) struct RunnerChildren { + pub(crate) broker: BrokerCore, config: RunnerConfig, state: Mutex, drained: Condvar, } -pub(crate) struct PreparedRunner { - pub(crate) pending: PendingProcess, +pub(crate) struct ChildRunner { + pub(crate) process: Arc, + pub(crate) inherited_objects: Vec, pub(crate) format: ProcessBootstrapFormat, pub(crate) version: ProcessBootstrapVersion, pub(crate) bootstrap: Vec, @@ -184,8 +182,7 @@ struct RunnerChildrenState { struct ChildLaunch { parent_id: ProcessId, - child_id: ProcessId, - start_commit: ProcessStartCommit, + process: Arc, state: Mutex, changed: Condvar, } @@ -202,8 +199,9 @@ enum ChildLaunchState { } impl RunnerChildren { - fn new(config: RunnerConfig) -> Arc { + fn new(config: RunnerConfig, broker: BrokerCore) -> Arc { Arc::new(Self { + broker, config, state: Mutex::new(RunnerChildrenState { launches: Vec::new(), @@ -275,49 +273,56 @@ impl RunnerChildren { if !parent.is_running() { return Err(ErrorCode::ProtocolState); } - let pending = parent - .prepare_child(requested_inherited_objects) + let (process, inherited_objects) = parent + .create_child(requested_inherited_objects) .map_err(ErrorCode::from)?; - let child_id = pending.id(); + let child_id = process.id(); let launch = Arc::new(ChildLaunch { parent_id: parent.id(), - child_id, - start_commit: pending.start_commit(), + process: Arc::clone(&process), state: Mutex::new(ChildLaunchState::Starting), changed: Condvar::new(), }); - let token = loop { - let mut token = [0; 8]; - getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; - let token = ProcessStartToken(u64::from_ne_bytes(token)); - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state - .launches - .try_reserve(1) - .map_err(|_| ErrorCode::OutOfMemory)?; - if parent.is_cancellation_requested() { - return Err(ErrorCode::PeerClosed); - } - if state.launches.len() >= MAX_PENDING_CHILD_STARTS { - return Err(ErrorCode::ResourceExhausted); + let token = match (|| { + loop { + let mut token = [0; 8]; + getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; + let token = ProcessStartToken(u64::from_ne_bytes(token)); + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state + .launches + .try_reserve(1) + .map_err(|_| ErrorCode::OutOfMemory)?; + if parent.is_cancellation_requested() { + return Err(ErrorCode::PeerClosed); + } + if state.launches.len() >= MAX_PENDING_CHILD_STARTS { + return Err(ErrorCode::ResourceExhausted); + } + if state + .launches + .iter() + .any(|(candidate, _)| *candidate == token) + { + continue; + } + let active_instances = state + .active_instances + .checked_add(1) + .ok_or(ErrorCode::ResourceExhausted)?; + state.launches.push((token, Arc::clone(&launch))); + state.active_instances = active_instances; + return Ok(token); } - if state - .launches - .iter() - .any(|(candidate, _)| *candidate == token) - { - continue; + })() { + Ok(token) => token, + Err(error) => { + BrokerProcess::finish(process); + return Err(error); } - let active_instances = state - .active_instances - .checked_add(1) - .ok_or(ErrorCode::ResourceExhausted)?; - state.launches.push((token, Arc::clone(&launch))); - state.active_instances = active_instances; - break token; }; let children = Arc::clone(self); @@ -326,11 +331,12 @@ impl RunnerChildren { let thread = std::thread::Builder::new() .name(format!("litebox-runner-{}", child_id.0)) .spawn(move || { - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { RunnerInstance::start(config).and_then(|instance| { - instance.run_prepared_to_completion( - PreparedRunner { - pending, + instance.run_child_to_completion( + ChildRunner { + process, + inherited_objects, format, version, bootstrap, @@ -338,12 +344,20 @@ impl RunnerChildren { Arc::clone(&children), ) }) - })) - .unwrap_or_else(|_| Err(IoError::other("prepared runner thread panicked"))); - children.child_finished(token, &thread_launch, result); + })); + match outcome { + Ok(result) => children.child_finished(token, &thread_launch, result, false), + Err(_) => children.child_finished( + token, + &thread_launch, + Err(IoError::other("child runner thread panicked")), + true, + ), + } }); if thread.is_err() { self.remove_launch(token); + BrokerProcess::finish(Arc::clone(&launch.process)); self.finish_instance(); return Err(ErrorCode::OutOfMemory); } @@ -393,7 +407,7 @@ impl RunnerChildren { .launches .iter() .position(|(_, launch)| { - launch.parent_id == process_id || launch.child_id == process_id + launch.parent_id == process_id || launch.process.id() == process_id }) .map(|index| state.launches.swap_remove(index).1) }; @@ -419,7 +433,7 @@ impl RunnerChildren { .expect("runner child state mutex poisoned") .launches .iter() - .find_map(|(_, launch)| (launch.child_id == child_id).then(|| Arc::clone(launch))) + .find_map(|(_, launch)| (launch.process.id() == child_id).then(|| Arc::clone(launch))) } fn remove_launch(&self, token: ProcessStartToken) { @@ -441,11 +455,15 @@ impl RunnerChildren { token: ProcessStartToken, launch: &ChildLaunch, result: IoResult, + panicked: bool, ) { if result.is_err() || result.is_ok_and(|status| !status.success()) { launch.abort(ErrorCode::PeerClosed); } self.remove_launch(token); + if !panicked { + BrokerProcess::finish(Arc::clone(&launch.process)); + } self.finish_instance(); } @@ -550,7 +568,7 @@ impl ChildLaunch { _ => ErrorCode::ProtocolState, }); } - self.start_commit.commit().map_err(ErrorCode::from)?; + self.process.commit_start().map_err(ErrorCode::from)?; *state = ChildLaunchState::Committed; self.changed.notify_all(); Ok(()) diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index a48242dc82..7fcdced7f4 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -9,14 +9,13 @@ use std::process::Child; use std::sync::Arc; use std::time::Instant; -use litebox_broker_core::BrokerCore; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport_linux_userland::memfd::MemfdSharedMemory; use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; -use super::{PreparedRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, @@ -45,12 +44,10 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, - broker: &BrokerCore, runner: &mut Child, children: Arc, ) -> IoResult<()> { serve_runner_process( - broker, self.listener .as_ref() .expect("a live runner instance must own its control listener"), @@ -59,18 +56,18 @@ impl PlatformRunnerEndpoint { ) } - pub(super) fn serve_prepared( + pub(super) fn serve_child( &mut self, runner: &mut Child, - prepared: PreparedRunner, + child: ChildRunner, children: Arc, ) -> IoResult<()> { - serve_prepared_runner_process( + serve_child_runner_process( self.listener .as_ref() .expect("a live runner instance must own its control listener"), runner, - prepared, + child, children, ) } @@ -82,14 +79,13 @@ impl PlatformRunnerEndpoint { } fn serve_runner_process( - broker: &BrokerCore, control_listener: &UnixListener, runner: &mut Child, children: Arc, ) -> IoResult<()> { let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; crate::runtime::serve_runner_association( - broker, + None, control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), MemfdSharedMemory::create_control_ring, @@ -103,15 +99,15 @@ fn serve_runner_process( ) } -fn serve_prepared_runner_process( +fn serve_child_runner_process( control_listener: &UnixListener, runner: &mut Child, - prepared: PreparedRunner, + child: ChildRunner, children: Arc, ) -> IoResult<()> { let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; - crate::runtime::serve_prepared_association( - prepared, + crate::runtime::serve_runner_association( + Some(child), control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), MemfdSharedMemory::create_control_ring, diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 5b3ec0b158..14dee40d5b 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -8,14 +8,13 @@ use std::process::Child; use std::sync::Arc; use std::time::Instant; -use litebox_broker_core::BrokerCore; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport_windows_userland::named_pipe::{ WindowsNamedPipeHostSetupChannel, WindowsNamedPipeListener, validate_client_process, }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; -use super::{PreparedRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; pub(super) struct PlatformRunnerEndpoint { pipe_name: OsString, @@ -38,12 +37,10 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, - broker: &BrokerCore, runner: &mut Child, children: Arc, ) -> IoResult<()> { serve_runner_process( - broker, self.listener .as_mut() .expect("a live runner instance must own its control listener"), @@ -52,18 +49,18 @@ impl PlatformRunnerEndpoint { ) } - pub(super) fn serve_prepared( + pub(super) fn serve_child( &mut self, runner: &mut Child, - prepared: PreparedRunner, + child: ChildRunner, children: Arc, ) -> IoResult<()> { - serve_prepared_runner_process( + serve_child_runner_process( self.listener .as_mut() .expect("a live runner instance must own its control listener"), runner, - prepared, + child, children, ) } @@ -74,7 +71,6 @@ impl PlatformRunnerEndpoint { } fn serve_runner_process( - broker: &BrokerCore, control_listener: &mut WindowsNamedPipeListener, runner: &mut Child, children: Arc, @@ -82,7 +78,7 @@ fn serve_runner_process( let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; let runner_process = runner.as_raw_handle(); crate::runtime::serve_runner_association( - broker, + None, control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), WindowsSharedMemory::create_control_ring, @@ -95,16 +91,16 @@ fn serve_runner_process( ) } -fn serve_prepared_runner_process( +fn serve_child_runner_process( control_listener: &mut WindowsNamedPipeListener, runner: &mut Child, - prepared: PreparedRunner, + child: ChildRunner, children: Arc, ) -> IoResult<()> { let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; let runner_process = runner.as_raw_handle(); - crate::runtime::serve_prepared_association( - prepared, + crate::runtime::serve_runner_association( + Some(child), control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), WindowsSharedMemory::create_control_ring, diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index f8f0686dbd..b6123d0de6 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -27,8 +27,8 @@ use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; use litebox_broker_host::{ - BrokerHostAssociation, BrokerHostError, ConnectionTermination, PreparedProcessBootstrap, - setup_connection, setup_prepared_connection, + BrokerHostAssociation, BrokerHostError, ConnectionTermination, ProcessStartupData, + setup_connection, }; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::BrokerRequest; @@ -41,7 +41,7 @@ use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; use crate::readiness::ReadinessPublisherRuntime; -use crate::runner::{PreparedRunner, RunnerChildren}; +use crate::runner::{ChildRunner, RunnerChildren}; const REQUEST_QUEUE_CAPACITY: usize = 64; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); @@ -94,6 +94,7 @@ where send_shared_memory, activate, None, + None, ) } @@ -105,7 +106,7 @@ pub(crate) fn serve_runner_association< NotificationChannel, Shutdown, >( - broker: &BrokerCore, + child: Option, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, create_control_memory: impl FnOnce() -> IoResult, @@ -124,17 +125,20 @@ where NotificationChannel: HostNotificationChannel + Send, Shutdown: HostAssociationShutdown + Send + Sync, { + let broker = children.broker.clone(); serve_association_with_children( - broker, + &broker, control_channel, create_shared_memory, create_control_memory, send_shared_memory, activate, Some(children), + child, ) } +#[allow(clippy::too_many_arguments)] fn serve_association_with_children< Memory, SetupChannel, @@ -153,6 +157,7 @@ fn serve_association_with_children< ControlRing, ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, children: Option>, + child: Option, ) -> IoResult<()> where Memory: ControlRingMemory, @@ -162,6 +167,25 @@ where NotificationChannel: HostNotificationChannel + Send, Shutdown: HostAssociationShutdown + Send + Sync, { + let is_child = child.is_some(); + let (process, startup) = match child { + Some(ChildRunner { + process, + inherited_objects, + format, + version, + bootstrap, + }) => ( + Some(process), + Some(ProcessStartupData { + format, + version, + payload: bootstrap, + inherited_objects, + }), + ), + None => (None, None), + }; let shared_memory = create_shared_memory()?; let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?; @@ -171,6 +195,8 @@ where let readiness = Arc::new(ReadinessPublisherRuntime::new()); let association = match setup_connection( broker, + process, + startup, &mut control_channel, &shared_buffers, readiness.clone(), @@ -202,7 +228,9 @@ where match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { - association.finish(); + if !is_child { + association.finish(); + } return Err(error); } }; @@ -217,99 +245,6 @@ where ) } -pub(crate) fn serve_prepared_association< - Memory, - SetupChannel, - RequestSource, - ResponseSink, - NotificationChannel, - Shutdown, ->( - prepared: PreparedRunner, - mut control_channel: SetupChannel, - create_shared_memory: impl FnOnce() -> IoResult, - create_control_memory: impl FnOnce() -> IoResult, - send_shared_memory: impl FnOnce(&mut SetupChannel, &Memory, &Memory) -> IoResult<()>, - activate: impl FnOnce( - SetupChannel, - ControlRing, - ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, - children: Arc, -) -> IoResult<()> -where - Memory: ControlRingMemory, - SetupChannel: HostSetupChannel, - RequestSource: HostRequestSource, - ResponseSink: HostResponseSink + Clone + Send, - NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, -{ - let PreparedRunner { - pending, - format, - version, - bootstrap, - } = prepared; - let shared_memory = create_shared_memory()?; - let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) - .map_err(|error| IoError::new(ErrorKind::InvalidData, error.to_string()))?; - let control_memory = create_control_memory()?; - let control_ring = ControlRing::new(control_memory) - .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; - let readiness = Arc::new(ReadinessPublisherRuntime::new()); - let association = match setup_prepared_connection( - pending, - &mut control_channel, - &shared_buffers, - readiness.clone(), - PreparedProcessBootstrap { - format, - version, - payload: &bootstrap, - }, - |channel| send_shared_memory(channel, shared_buffers.memory(), control_ring.memory()), - ) - .map_err(map_host_error)? - { - Ok(association) => association, - Err(ConnectionTermination::PeerClosed) => { - return Err(IoError::new( - ErrorKind::UnexpectedEof, - "prepared runner closed before completing broker setup", - )); - } - Err(ConnectionTermination::ProtocolViolation) => { - return Err(IoError::new( - ErrorKind::InvalidData, - "prepared runner violated the broker protocol during setup", - )); - } - Err(_) => { - return Err(IoError::new( - ErrorKind::InvalidData, - "prepared runner ended broker setup unexpectedly", - )); - } - }; - let (request_source, response_sink, notification_channel, shutdown) = - match activate(control_channel, control_ring) { - Ok(active) => active, - Err(error) => { - association.finish(); - return Err(error); - } - }; - dispatch_requests_with_children( - association, - readiness, - request_source, - response_sink, - notification_channel, - shutdown, - Some(children), - ) -} - /// Maps a host setup or request failure to a precise [`std::io::Error`]. /// /// A channel failure is already an [`std::io::Error`] and is returned as-is. @@ -850,6 +785,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + startup: None, }) .unwrap(); local_setup.recv_handshake_response().unwrap().unwrap(); @@ -892,27 +828,28 @@ mod tests { UnixControlRingLocalNotificationChannel, UnixControlRingLocalShutdown, ) { - let (local, (notifications, shutdown)) = litebox_broker_local::BrokerLocal::negotiate( - UnixStreamLocalSetupChannel::from_connected(stream), - |mut setup| { - let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; - let control_memory = setup.receive_control_ring(None)?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - IoError::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; - let (call_channel, notifications, shutdown) = - setup.into_active(control_ring, || {})?; - Ok(( - call_channel, - Arc::new(shared_memory), - (notifications, shutdown), - )) - }, - ) - .unwrap(); + let (local, _startup, (notifications, shutdown)) = + litebox_broker_local::BrokerLocal::negotiate( + UnixStreamLocalSetupChannel::from_connected(stream), + |mut setup| { + let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; + let control_memory = setup.receive_control_ring(None)?; + let control_ring = ControlRing::new(control_memory).map_err(|error| { + IoError::new( + ErrorKind::InvalidData, + format!("invalid test control ring: {error:?}"), + ) + })?; + let (call_channel, notifications, shutdown) = + setup.into_active(control_ring, || {})?; + Ok(( + call_channel, + Arc::new(shared_memory), + (notifications, shutdown), + )) + }, + ) + .unwrap(); (local, notifications, shutdown) } @@ -954,6 +891,8 @@ mod tests { ); let association = setup_connection( &broker, + None, + None, &mut control, &shared_buffers, readiness.clone(), diff --git a/litebox_broker_userland/tests/notification_runtime.rs b/litebox_broker_userland/tests/notification_runtime.rs index 34122571d3..5ef7eed500 100644 --- a/litebox_broker_userland/tests/notification_runtime.rs +++ b/litebox_broker_userland/tests/notification_runtime.rs @@ -59,6 +59,8 @@ fn spawn_host( let mut control = UnixStreamHostSetupChannel::from_accepted(stream); let association = setup_connection( &broker, + None, + None, &mut control, &shared_buffers, Arc::new(ReadinessPublisherRuntime::new()), @@ -91,7 +93,7 @@ fn negotiate_local( BrokerLocal, BrokerNotifications, ) { - let (local, notifications) = BrokerLocal::negotiate( + let (local, _startup, notifications) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(stream), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; @@ -150,6 +152,8 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { let mut control = UnixStreamHostSetupChannel::from_accepted(host_control); let association = setup_connection( &broker, + None, + None, &mut control, &host_shared_buffers, Arc::new(ReadinessPublisherRuntime::new()), @@ -176,7 +180,7 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { } }); - let (local, notification_channel) = BrokerLocal::negotiate( + let (local, _startup, notification_channel) = BrokerLocal::negotiate( UnixStreamLocalSetupChannel::from_connected(local_control), |mut setup| { let shared_memory = setup.receive_memfd(SHARED_BUFFER_POOL_SIZE, None)?; diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 6b85193906..f3b1b4e8b3 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -26,7 +26,7 @@ const RUNNER_ARGUMENT: &str = "broker-userland-test-runner"; const NETWORK_RUNNER_ARGUMENT: &str = "broker-userland-network-test-runner"; const CHILD_START_RUNNER_ARGUMENT: &str = "broker-userland-child-start-runner"; const CHILD_CANCEL_RUNNER_ARGUMENT: &str = "broker-userland-child-cancel-runner"; -const PREPARED_CHILD_ARGUMENT: &str = "--prepared-child"; +const CHILD_ARGUMENT: &str = "--child"; const TEST_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x7465_7374); const FAILING_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x6661_696c); const BROKER_PROCESS_TIMEOUT: Duration = Duration::from_secs(30); @@ -145,12 +145,12 @@ fn run_fake_runner(args: &[OsString]) { ); let control_socket_path = args.get(2).unwrap(); - if args.get(3).and_then(|argument| argument.to_str()) == Some(PREPARED_CHILD_ARGUMENT) { - run_fake_prepared_child(Path::new(control_socket_path)); + if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_ARGUMENT) { + run_fake_child(Path::new(control_socket_path)); return; } let setup_channel = connect_control_with_retry(Path::new(control_socket_path)).unwrap(); - let (local, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { + let (local, startup, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd( SHARED_BUFFER_POOL_SIZE, Some(Instant::now() + Duration::from_secs(5)), @@ -167,6 +167,7 @@ fn run_fake_runner(args: &[OsString]) { Ok((call_channel, Arc::new(shared_memory), ())) }) .unwrap(); + assert!(startup.is_none()); let local = Arc::new(local); if args.get(3).and_then(|argument| argument.to_str()) == Some(NETWORK_RUNNER_ARGUMENT) { @@ -372,9 +373,9 @@ fn run_fake_runner(args: &[OsString]) { drop(local); } -fn run_fake_prepared_child(control_socket_path: &Path) { +fn run_fake_child(control_socket_path: &Path) { let setup_channel = connect_control_with_retry(control_socket_path).unwrap(); - let (local, bootstrap, ()) = BrokerLocal::negotiate_prepared(setup_channel, |mut setup| { + let (local, bootstrap, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd( SHARED_BUFFER_POOL_SIZE, Some(Instant::now() + Duration::from_secs(5)), @@ -391,6 +392,7 @@ fn run_fake_prepared_child(control_socket_path: &Path) { Ok((call_channel, Arc::new(shared_memory), ())) }) .unwrap(); + let bootstrap = bootstrap.expect("child negotiation must include startup data"); if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { return; } @@ -407,7 +409,7 @@ fn run_fake_prepared_child(control_socket_path: &Path) { match local.process_ready(None) { Ok(()) => {} Err(BrokerLocalError::Broker(ErrorCode::PeerClosed)) => return, - Err(error) => panic!("prepared child readiness failed: {error}"), + Err(error) => panic!("child readiness failed: {error}"), } std::fs::OpenOptions::new() .append(true) diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 836845ed86..ab4b39bb8f 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -31,7 +31,7 @@ pub struct CliArgs { /// /// The program path must be absolute and refer to a file in the broker-owned file system. #[arg( - required_unless_present = "prepared_child", + required_unless_present = "child", trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments )] @@ -45,14 +45,14 @@ pub struct CliArgs { /// Allow using unstable options #[arg(short = 'Z', long = "unstable")] pub unstable: bool, - /// Connect as a broker-reserved prepared child. + /// Start as a dynamically launched child. #[arg( - long = "prepared-child", + long = "child", hide = true, requires_all = ["unstable", "broker_control_channel"], help_heading = "Unstable Options" )] - pub prepared_child: bool, + pub child: bool, /// Broker-supplied Unix socket path for the local control channel. #[arg( long = "broker-control-channel", @@ -98,19 +98,17 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - if cli_args.prepared_child { + if cli_args.child { if !cli_args.program_and_arguments.is_empty() { - return Err(anyhow!( - "--prepared-child does not accept a root program argument" - )); + return Err(anyhow!("--child does not accept a root program argument")); } let control_socket_path = cli_args .broker_control_channel .as_deref() - .context("--prepared-child requires --broker-control-channel")?; - let (_connection, bootstrap) = broker::connect_prepared(control_socket_path)?; + .context("--child requires --broker-control-channel")?; + let (_connection, bootstrap) = broker::connect_child(control_socket_path)?; return Err(anyhow!( - "unsupported prepared Linux process bootstrap format {:?} version {:?}", + "unsupported child Linux process bootstrap format {:?} version {:?}", bootstrap.format, bootstrap.version )); @@ -256,17 +254,17 @@ mod tests { } #[test] - fn prepared_child_does_not_require_a_root_program() { + fn child_does_not_require_a_root_program() { let args = CliArgs::try_parse_from([ "runner", "--unstable", "--broker-control-channel", "/tmp/broker.sock", - "--prepared-child", + "--child", ]) .unwrap(); - assert!(args.prepared_child); + assert!(args.child); assert!(args.program_and_arguments.is_empty()); } diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index 928b77b275..5305174f3e 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -521,6 +521,8 @@ fn run_test_broker_connection( std::sync::Arc::new(litebox_broker_userland::readiness::ReadinessPublisherRuntime::new()); let association = litebox_broker_host::setup_connection( broker, + None, + None, &mut channel, &shared_buffers, readiness.clone(), diff --git a/litebox_runner_macos_userland/src/test_broker.rs b/litebox_runner_macos_userland/src/test_broker.rs index 4e700c9f36..76712d0fee 100644 --- a/litebox_runner_macos_userland/src/test_broker.rs +++ b/litebox_runner_macos_userland/src/test_broker.rs @@ -69,7 +69,7 @@ pub(crate) fn setup( .map_err(|error| anyhow!("test broker: {error:?}"))?; let setup = InProcessBrokerSetup::new(core); let readiness = setup.readiness_sink(); - let (local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index b665ca3099..94c88bd0e0 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -21,7 +21,7 @@ pub struct CliArgs { /// /// The program path refers to a path inside the broker-owned file system. #[arg( - required_unless_present = "prepared_child", + required_unless_present = "child", trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments )] @@ -35,14 +35,14 @@ pub struct CliArgs { /// Allow using unstable options. #[arg(short = 'Z', long = "unstable")] pub unstable: bool, - /// Connect as a broker-reserved prepared child. + /// Start as a dynamically launched child. #[arg( - long = "prepared-child", + long = "child", hide = true, requires_all = ["unstable", "broker_control_channel"], help_heading = "Unstable Options" )] - pub prepared_child: bool, + pub child: bool, /// Broker-supplied Windows named-pipe path for the local control channel. #[arg( long = "broker-control-channel", @@ -66,17 +66,17 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - if cli_args.prepared_child { + if cli_args.child { if !cli_args.program_and_arguments.is_empty() { - anyhow::bail!("--prepared-child does not accept a root program argument"); + anyhow::bail!("--child does not accept a root program argument"); } let control_pipe = cli_args .broker_control_channel .as_deref() - .context("--prepared-child requires --broker-control-channel")?; - let (_connection, bootstrap) = broker::connect_prepared(control_pipe)?; + .context("--child requires --broker-control-channel")?; + let (_connection, bootstrap) = broker::connect_child(control_pipe)?; anyhow::bail!( - "unsupported prepared Windows process bootstrap format {:?} version {:?}", + "unsupported child Windows process bootstrap format {:?} version {:?}", bootstrap.format, bootstrap.version ); diff --git a/litebox_shim_linux/src/syscalls/test_broker.rs b/litebox_shim_linux/src/syscalls/test_broker.rs index ec2204f462..804335b504 100644 --- a/litebox_shim_linux/src/syscalls/test_broker.rs +++ b/litebox_shim_linux/src/syscalls/test_broker.rs @@ -45,7 +45,7 @@ pub(crate) fn litebox_with_limits( ) -> (litebox::LiteBox, i32) { let setup = InProcessBrokerSetup::new(test_broker(limits).clone()); let readiness = setup.readiness_sink(); - let (broker_local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (broker_local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_shim_macos/src/syscalls/file.rs b/litebox_shim_macos/src/syscalls/file.rs index 203503fa89..4c639893a3 100644 --- a/litebox_shim_macos/src/syscalls/file.rs +++ b/litebox_shim_macos/src/syscalls/file.rs @@ -330,7 +330,7 @@ mod tests { .unwrap(); let setup = InProcessBrokerSetup::new(broker); let readiness = setup.readiness_sink(); - let (local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) diff --git a/litebox_shim_windows/src/test_broker.rs b/litebox_shim_windows/src/test_broker.rs index e912f3a32e..ce798b004a 100644 --- a/litebox_shim_windows/src/test_broker.rs +++ b/litebox_shim_windows/src/test_broker.rs @@ -77,7 +77,7 @@ fn connect( ) -> (litebox::LiteBox, usize) { let setup = InProcessBrokerSetup::new(broker); let readiness = setup.readiness_sink(); - let (broker_local, ()) = BrokerLocal::negotiate(setup, |setup| { + let (broker_local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| { let memory = setup.shared_memory(); Ok((setup.activate(), memory, ())) }) From 25be3a99995f03f71f5add60c9ac1b5c0e38456f Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Thu, 17 Sep 2026 21:56:01 -0700 Subject: [PATCH 04/21] Complete child process startup lifecycle Converge child startup publication, acknowledgement, failure, shutdown, and finalization across Linux and Windows while preserving process identity safety. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- Cargo.lock | 3 + dev_tests/src/ratchet.rs | 1 + litebox_broker_core/src/process.rs | 110 +- litebox_broker_host/src/error.rs | 2 + litebox_broker_host/src/lib.rs | 97 +- litebox_broker_local/src/process.rs | 23 +- litebox_broker_protocol/src/lib.rs | 2 +- litebox_broker_protocol/src/message.rs | 7 +- litebox_broker_protocol/src/wire.rs | 30 +- litebox_broker_transport/src/pending_calls.rs | 64 +- .../src/unix_socket/local.rs | 72 +- .../src/local.rs | 18 +- .../src/named_pipe.rs | 49 +- litebox_broker_userland/Cargo.toml | 2 + litebox_broker_userland/src/runner.rs | 2625 +++++++++++++++-- litebox_broker_userland/src/runner/linux.rs | 134 +- litebox_broker_userland/src/runner/windows.rs | 137 +- litebox_broker_userland/src/runtime.rs | 239 +- .../tests/userland_broker.rs | 5 +- litebox_runner_linux_userland/Cargo.toml | 2 +- litebox_runner_linux_userland/src/lib.rs | 6 +- litebox_runner_windows_userland/Cargo.toml | 1 + litebox_runner_windows_userland/src/lib.rs | 6 +- 23 files changed, 3314 insertions(+), 321 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0219e06d1d..5860b780e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,6 +1571,7 @@ version = "0.1.0" dependencies = [ "clap", "getrandom 0.3.4", + "libc", "litebox_broker_core", "litebox_broker_host", "litebox_broker_local", @@ -1585,6 +1586,7 @@ dependencies = [ "litebox_runner_linux_userland", "litebox_runner_windows_userland", "tempfile", + "windows-sys 0.60.2", ] [[package]] @@ -1938,6 +1940,7 @@ dependencies = [ "clap", "litebox", "litebox_broker_local_userland", + "litebox_broker_protocol", "litebox_common_linux", "litebox_platform_windows_userland", "litebox_shim_windows", diff --git a/dev_tests/src/ratchet.rs b/dev_tests/src/ratchet.rs index d8bcd75789..96756205bb 100644 --- a/dev_tests/src/ratchet.rs +++ b/dev_tests/src/ratchet.rs @@ -78,6 +78,7 @@ fn ratchet_maybe_uninit() -> Result<()> { ("dev_tests/", 1), ("litebox/", 1), ("litebox_broker_transport_linux_userland/", 3), + ("litebox_broker_userland/", 1), ("litebox_platform_linux_userland/", 2), ("litebox_platform_macos_userland/", 2), ], diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index c8efd0d289..5aa13c765b 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -77,6 +77,11 @@ struct ProcessReferences { pending_handles: usize, } +struct ProcessThreads { + entries: HashMap, + startup_pin: Option, +} + /// Broker-owned state for one guest thread. /// /// Execution remains platform-local. This object owns the authoritative @@ -115,7 +120,7 @@ pub struct BrokerProcess { /// Handles of the live object references owned by this process. references: Mutex, /// Authoritative broker threads owned by this process. - threads: Mutex>, + threads: Mutex, /// Pipe capacity charged to this process by live pipe objects. pub(crate) reserved_pipe_capacity: Arc, /// Socket quota held by pending, live, and closing in-flight resources. @@ -150,7 +155,10 @@ impl BrokerProcess { handles: Vec::new(), pending_handles: 0, }), - threads: Mutex::new(HashMap::new()), + threads: Mutex::new(ProcessThreads { + entries: HashMap::new(), + startup_pin: None, + }), reserved_pipe_capacity: Arc::new(AtomicUsize::new(0)), reserved_sockets: Arc::new(AtomicUsize::new(0)), cancellation: AssociationCancellation::default(), @@ -237,10 +245,11 @@ impl BrokerProcess { /// invariants. pub fn create_thread(&self) -> Result { let mut threads = self.threads.lock(); - if threads.len() >= self.core.limits.max_threads_per_process { + if threads.entries.len() >= self.core.limits.max_threads_per_process { return Err(BrokerError::ResourceExhausted); } threads + .entries .try_reserve(1) .map_err(|_| BrokerError::OutOfMemory)?; self.core @@ -261,19 +270,52 @@ impl BrokerProcess { let thread = BrokerThread::new(ThreadId(raw_id)); let thread_id = thread.id(); assert!( - threads.insert(thread_id, thread).is_none(), + threads.entries.insert(thread_id, thread).is_none(), "the ID allocator returned an occupied thread ID" ); Ok(thread_id) } + /// Returns whether this process owns the live broker thread. + #[must_use] + pub fn owns_thread(&self, thread_id: ThreadId) -> bool { + self.threads.lock().entries.contains_key(&thread_id) + } + + /// Pins a live thread while its identity is part of process-start publication. + pub fn pin_startup_thread(&self, thread_id: ThreadId) -> Result<()> { + let mut threads = self.threads.lock(); + if !threads.entries.contains_key(&thread_id) { + return Err(BrokerError::UnknownObject); + } + if threads.startup_pin.is_some() { + return Err(BrokerError::Internal); + } + threads.startup_pin = Some(thread_id); + Ok(()) + } + + /// Releases a thread identity after process-start publication reaches a terminal state. + pub fn release_startup_thread(&self, thread_id: ThreadId) -> Result<()> { + let mut threads = self.threads.lock(); + if threads.startup_pin != Some(thread_id) { + return Err(BrokerError::Internal); + } + threads.startup_pin = None; + Ok(()) + } + /// Records broker thread exit after its local task teardown completes. pub fn exit_thread(&self, thread_id: ThreadId) -> Result<()> { - let thread = self - .threads - .lock() + let mut threads = self.threads.lock(); + if threads.startup_pin == Some(thread_id) { + return Err(BrokerError::WouldBlock); + } + let thread = threads + .entries .remove(&thread_id) .ok_or(BrokerError::UnknownObject)?; + drop(threads); self.core .active_thread_count .fetch_sub(1, Ordering::Relaxed); @@ -302,6 +344,15 @@ impl BrokerProcess { self.cleanup(true); } + /// Completes abnormal process teardown without releasing numeric IDs. + /// + /// Calling this method more than once is harmless. It is used after an + /// unwind or invariant failure where reusing process or thread IDs would + /// make later observations ambiguous. + pub fn finish_abnormal(self: Arc) { + self.cleanup(false); + } + pub(crate) fn create_object_reference(&self, object: ObjectEntry) -> Result { let rights = self .core @@ -760,7 +811,11 @@ impl BrokerProcess { invariant_fault = true; } - let threads = core::mem::take(&mut *self.threads.lock()); + let threads = { + let mut threads = self.threads.lock(); + threads.startup_pin = None; + core::mem::take(&mut threads.entries) + }; if release_ids && !invariant_fault { self.core .active_thread_count @@ -908,6 +963,45 @@ mod tests { assert_eq!(second.parent_id(), None); } + #[test] + fn process_recognizes_only_its_live_threads() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let first = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let second = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let thread = first.create_thread().unwrap(); + + assert!(first.owns_thread(thread)); + assert!(!second.owns_thread(thread)); + first.exit_thread(thread).unwrap(); + assert!(!first.owns_thread(thread)); + } + + #[test] + fn startup_thread_pin_defers_thread_exit_until_publication_completes() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let process = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let thread = process.create_thread().unwrap(); + + process.pin_startup_thread(thread).unwrap(); + assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); + process.release_startup_thread(thread).unwrap(); + assert_eq!(process.exit_thread(thread), Ok(())); + } + #[test] fn child_is_parented_and_requires_commit() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( diff --git a/litebox_broker_host/src/error.rs b/litebox_broker_host/src/error.rs index c52813d084..af88432e6f 100644 --- a/litebox_broker_host/src/error.rs +++ b/litebox_broker_host/src/error.rs @@ -13,6 +13,8 @@ pub enum BrokerHostError { Channel(#[source] E), #[error("broker setup failed: {0}")] Broker(#[source] ErrorCode), + #[error("broker association had already failed")] + AssociationFailed, #[error("broker association shared-buffer layout does not match the protocol layout")] SharedBufferLayoutMismatch, } diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 438f988eb2..6b3ef9540f 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -147,7 +147,7 @@ impl BrokerHostAssociation<'_, Memory> { request, |_process, _operation, _shared_buffers| None, send_response, - |_result| {}, + |_operation, _result| {}, ) } @@ -165,18 +165,19 @@ impl BrokerHostAssociation<'_, Memory> { ) -> Option>, send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, - response_sent: impl FnOnce(&BrokerResult), + response_sent: impl FnOnce(&BrokerOperation, &BrokerResult), ) -> Result<(), ChannelError> { let BrokerRequest { request_id, operation, } = request; + let response_operation = operation.clone(); let buffer_sequence = operation.shared_buffer(); { let mut state = self.state.lock(); if state.failed { - return Err(BrokerHostError::Broker(ErrorCode::Internal)); + return Err(BrokerHostError::AssociationFailed); } if let Some(sequence) = buffer_sequence && let Err(error) = state.shared_buffer_usage.begin( @@ -220,7 +221,7 @@ impl BrokerHostAssociation<'_, Memory> { self.state.lock().failed = true; return Err(BrokerHostError::Channel(error)); } - response_sent(&response.result); + response_sent(&response_operation, &response.result); Ok(()) } } @@ -250,6 +251,35 @@ pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( readiness_sink: Arc, send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, ) -> Result, ChannelError> +where + SetupChannel: HostSetupChannel, + Memory: SharedMemory, +{ + setup_connection_with_process( + core, + process, + startup, + setup_channel, + shared_buffers, + readiness_sink, + |_| false, + send_shared_memory, + ) +} + +/// Authenticates and negotiates one broker control connection while allowing +/// a deployment owner to retain the negotiated process before setup completes. +#[allow(clippy::too_many_arguments)] +pub fn setup_connection_with_process<'a, SetupChannel, Memory, ChannelError>( + core: &BrokerCore, + process: Option>, + startup: Option, + setup_channel: &mut SetupChannel, + shared_buffers: &'a SharedBufferPool, + readiness_sink: Arc, + retain_process: impl FnOnce(&Arc) -> bool, + send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, +) -> Result, ChannelError> where SetupChannel: HostSetupChannel, Memory: SharedMemory, @@ -362,14 +392,15 @@ where process_id: process.id(), startup, }; + let process_retained = retain_process(&process); if let Err(error) = setup_channel.send_handshake_response(&response) { - if finish_on_setup_error { + if finish_on_setup_error && !process_retained { BrokerProcess::finish(process); } return Err(BrokerHostError::Channel(error)); } if let Err(error) = send_shared_memory(setup_channel) { - if finish_on_setup_error { + if finish_on_setup_error && !process_retained { BrokerProcess::finish(process); } return Err(BrokerHostError::Channel(error)); @@ -547,7 +578,8 @@ fn handle_request( } BrokerOperation::StartProcess(_) | BrokerOperation::AcknowledgeProcessStart(_) - | BrokerOperation::ProcessReady(_) => { + | BrokerOperation::ProcessReady(_) + | BrokerOperation::ReportProcessStartFailure(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) } } @@ -1557,6 +1589,7 @@ mod tests { test_channel_continues_after_recoverable_request_failure(&broker); test_channel_aborts_on_stale_shared_buffer_request(&broker); test_channel_aborts_without_response_on_shared_memory_failure(&broker); + setup_failure_transfers_process_to_the_deployment_owner(&broker); test_channel_rejects_incompatible_shared_buffer_layout(&broker); active_request_allocates_and_releases_thread_id(&broker); active_request_closes_object_reference(&broker); @@ -1569,6 +1602,42 @@ mod tests { association_executes_distinct_slots_concurrently(&broker); association_allows_slot_reuse_during_response_emission(&broker); association_allows_out_of_order_responses(&broker); + association_preserves_the_initiating_failure(&broker); + } + + fn setup_failure_transfers_process_to_the_deployment_owner(broker: &BrokerCore) { + let mut channel = FakeHostControlChannel::new( + std::vec::Vec::from([Ok(HostReceive::Message(BrokerHandshakeRequest { + protocol_version: BROKER_PROTOCOL_VERSION, + }))]), + std::vec::Vec::new(), + ); + let shared_buffers = test_shared_buffers(); + let retained = Mutex::new(None); + + assert!(matches!( + setup_connection_with_process( + broker, + None, + None, + &mut channel, + &shared_buffers, + test_readiness_sink(), + |process| { + *retained.lock().unwrap() = Some(Arc::clone(process)); + true + }, + |_| Err(()), + ), + Err(BrokerHostError::Channel(())) + )); + + let process = retained + .into_inner() + .unwrap() + .expect("deployment owner must retain the negotiated process"); + assert!(process.is_running()); + process.finish(); } fn association_shared_buffer_sequences_stage_file_data(broker: &BrokerCore) { @@ -2679,6 +2748,20 @@ mod tests { }); } + fn association_preserves_the_initiating_failure(broker: &BrokerCore) { + let shared_buffers = test_shared_buffers(); + let association = test_association(broker, &shared_buffers); + + assert!(matches!( + association.execute_request(event_create_request(1), |_| Err::<(), _>(())), + Err(BrokerHostError::Channel(())) + )); + assert!(matches!( + association.execute_request(event_create_request(2), |_| Ok::<_, ()>(())), + Err(BrokerHostError::AssociationFailed) + )); + } + fn test_association<'a, Memory: SharedMemory>( broker: &BrokerCore, shared_buffers: &'a SharedBufferPool, diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs index 6641a76ef7..926a51676a 100644 --- a/litebox_broker_local/src/process.rs +++ b/litebox_broker_local/src/process.rs @@ -62,7 +62,9 @@ impl BrokerLocal { ) -> Result<(), Channel::Error> { match self.request(BrokerOperation::AcknowledgeProcessStart(token))? { BrokerResult::ProcessStartAcknowledged => Ok(()), - BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + BrokerResult::ProcessStartFailed(error) | BrokerResult::Error(error) => { + Err(BrokerLocalError::Broker(error)) + } response => { panic!("broker returned unexpected process-start acknowledgement: {response:?}") } @@ -83,4 +85,23 @@ impl BrokerLocal { response => panic!("broker returned unexpected process-ready response: {response:?}"), } } + + /// Reports that this child rejected its startup data before becoming ready. + /// + /// # Panics + /// + /// Panics if the broker returns a response for another operation or echoes + /// a different failure. + pub fn report_process_start_failure(&self, error: ErrorCode) -> Result<(), Channel::Error> { + match self.request(BrokerOperation::ReportProcessStartFailure(error))? { + BrokerResult::ProcessStartFailed(reported) if reported == error => Ok(()), + BrokerResult::ProcessStartFailed(reported) => { + panic!("broker reported a different process-start failure: {reported}") + } + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => { + panic!("broker returned unexpected process-start failure response: {response:?}") + } + } + } } diff --git a/litebox_broker_protocol/src/lib.rs b/litebox_broker_protocol/src/lib.rs index 8806d1c297..ecb6bfdf2f 100644 --- a/litebox_broker_protocol/src/lib.rs +++ b/litebox_broker_protocol/src/lib.rs @@ -61,4 +61,4 @@ pub struct RequestId(pub u64); pub struct ProtocolVersion(pub u16); /// Current broker protocol version. -pub const BROKER_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion(1); +pub const BROKER_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion(2); diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index ed23c41d45..01b0f85039 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -74,6 +74,8 @@ pub enum BrokerOperation { AcknowledgeProcessStart(ProcessStartToken), /// Report that this child process is ready to begin guest execution. ProcessReady(ProcessReadyRequest), + /// Report that this child rejected its startup data before becoming ready. + ReportProcessStartFailure(ErrorCode), } impl BrokerOperation { @@ -134,7 +136,8 @@ impl BrokerOperation { FileRequest::Seek(_) | FileRequest::Truncate(_) | FileRequest::HandleStatus(_), ) | Self::AcknowledgeProcessStart(_) - | Self::ProcessReady(_) => None, + | Self::ProcessReady(_) + | Self::ReportProcessStartFailure(_) => None, } } } @@ -256,6 +259,8 @@ pub enum BrokerResult { ProcessStarted(StartedProcess), /// Parent acknowledgement committed the child. ProcessStartAcknowledged, + /// A child-start failure was reported or observed before commit. + ProcessStartFailed(ErrorCode), /// Parent acknowledgement released the child. ProcessReady, /// Operation failed with an ABI-neutral broker error. diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index b45448baa2..1cbfbc7bab 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -52,6 +52,7 @@ const REQUEST_TAG_EXIT_THREAD: u8 = 10; const REQUEST_TAG_START_PROCESS: u8 = 11; const REQUEST_TAG_ACKNOWLEDGE_PROCESS_START: u8 = 12; const REQUEST_TAG_PROCESS_READY: u8 = 13; +const REQUEST_TAG_REPORT_PROCESS_START_FAILURE: u8 = 14; // Paired request and successful-response tags intentionally share values. const RESPONSE_TAG_NEGOTIATED: u8 = 0; @@ -68,6 +69,7 @@ const RESPONSE_TAG_THREAD_EXITED: u8 = 10; const RESPONSE_TAG_PROCESS_STARTED: u8 = 11; const RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED: u8 = 12; const RESPONSE_TAG_PROCESS_READY: u8 = 13; +const RESPONSE_TAG_PROCESS_START_FAILED: u8 = 14; // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; @@ -129,7 +131,8 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_PROCESS_READY + | REQUEST_TAG_REPORT_PROCESS_START_FAILURE => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -224,6 +227,11 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); encode_optional_thread_id(&mut encoder, initial_thread_id); } + BrokerOperation::ReportProcessStartFailure(error) => { + encoder.u8(REQUEST_TAG_REPORT_PROCESS_START_FAILURE); + encoder.request_id(request_id); + encode_error_code(&mut encoder, error); + } } encoder.finish() } @@ -246,7 +254,8 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_EXIT_THREAD | REQUEST_TAG_START_PROCESS | REQUEST_TAG_ACKNOWLEDGE_PROCESS_START - | REQUEST_TAG_PROCESS_READY => {} + | REQUEST_TAG_PROCESS_READY + | REQUEST_TAG_REPORT_PROCESS_START_FAILURE => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -275,6 +284,9 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_PROCESS_READY => BrokerOperation::ProcessReady(ProcessReadyRequest { initial_thread_id: decode_optional_thread_id(&mut decoder)?, }), + REQUEST_TAG_REPORT_PROCESS_START_FAILURE => { + BrokerOperation::ReportProcessStartFailure(decode_error_code(&mut decoder)?) + } _ => unreachable!("active request tag was validated"), }; decoder.finish()?; @@ -366,6 +378,7 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { return Err(WireError::WrongMessagePhase); } @@ -451,6 +464,11 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.u8(RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED); encoder.request_id(request_id); } + BrokerResult::ProcessStartFailed(error) => { + encoder.u8(RESPONSE_TAG_PROCESS_START_FAILED); + encoder.request_id(request_id); + encode_error_code(&mut encoder, error); + } BrokerResult::ProcessReady => { encoder.u8(RESPONSE_TAG_PROCESS_READY); encoder.request_id(request_id); @@ -485,6 +503,7 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_THREAD_EXITED | RESPONSE_TAG_PROCESS_STARTED | RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED + | RESPONSE_TAG_PROCESS_START_FAILED | RESPONSE_TAG_PROCESS_READY => {} _ => return Err(WireError::InvalidTag), } @@ -507,6 +526,9 @@ pub fn decode_response(frame: &[u8]) -> Result { initial_thread_id: decode_optional_thread_id(&mut decoder)?, }), RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED => BrokerResult::ProcessStartAcknowledged, + RESPONSE_TAG_PROCESS_START_FAILED => { + BrokerResult::ProcessStartFailed(decode_error_code(&mut decoder)?) + } RESPONSE_TAG_PROCESS_READY => BrokerResult::ProcessReady, _ => unreachable!("active response tag was validated"), }; @@ -707,6 +729,7 @@ mod tests { RESPONSE_TAG_PROCESS_STARTED, RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED, RESPONSE_TAG_PROCESS_READY, + RESPONSE_TAG_PROCESS_START_FAILED, ], [ REQUEST_TAG_NEGOTIATE, @@ -723,6 +746,7 @@ mod tests { REQUEST_TAG_START_PROCESS, REQUEST_TAG_ACKNOWLEDGE_PROCESS_START, REQUEST_TAG_PROCESS_READY, + REQUEST_TAG_REPORT_PROCESS_START_FAILURE, ] ); assert_eq!( @@ -1012,6 +1036,7 @@ mod tests { BrokerOperation::ProcessReady(ProcessReadyRequest { initial_thread_id: Some(thread_id(19)), }), + BrokerOperation::ReportProcessStartFailure(ErrorCode::UnsupportedOperation), ]; let mut maximum_encoded_size = 0; @@ -1368,6 +1393,7 @@ mod tests { initial_thread_id: Some(thread_id(11)), }), BrokerResult::ProcessStartAcknowledged, + BrokerResult::ProcessStartFailed(ErrorCode::PeerClosed), BrokerResult::ProcessReady, BrokerResult::Error(ErrorCode::PolicyDenied), BrokerResult::Error(ErrorCode::WouldBlock), diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs index fad5613cdb..23e7476968 100644 --- a/litebox_broker_transport/src/pending_calls.rs +++ b/litebox_broker_transport/src/pending_calls.rs @@ -13,6 +13,10 @@ use litebox_broker_protocol::message::BrokerResponse; /// Maximum number of active calls waiting for broker responses. pub const MAX_PENDING_CALLS: usize = 64; +/// Pending-call capacity reserved for lifecycle-control operations. +pub const RESERVED_LIFECYCLE_PENDING_CALLS: usize = 8; +/// Maximum active ordinary calls after preserving lifecycle-control capacity. +pub const MAX_ORDINARY_PENDING_CALLS: usize = MAX_PENDING_CALLS - RESERVED_LIFECYCLE_PENDING_CALLS; /// A mutex usable by [`PendingCalls`]. pub trait PendingCallsMutex { @@ -82,10 +86,16 @@ pub struct PendingCalls { } struct PendingCallsState { - calls: BTreeMap>>, + calls: BTreeMap>, + ordinary_calls: usize, failure: Option>, } +struct RegisteredPendingCall { + call: Arc>, + ordinary: bool, +} + /// Completion state for one request awaiting a broker response. pub struct PendingCall { result: Sync::Mutex>>>, @@ -125,6 +135,7 @@ impl PendingCalls { Self { state: Sync::mutex(PendingCallsState { calls: BTreeMap::new(), + ordinary_calls: 0, failure: None, }), capacity_available: Sync::condvar(), @@ -135,10 +146,29 @@ impl PendingCalls { pub fn register( &self, request_id: RequestId, + ) -> Result>, PendingCallsError> { + self.register_with_class(request_id, true) + } + + /// Registers lifecycle-control work using capacity ordinary calls cannot consume. + pub fn register_lifecycle( + &self, + request_id: RequestId, + ) -> Result>, PendingCallsError> { + self.register_with_class(request_id, false) + } + + fn register_with_class( + &self, + request_id: RequestId, + ordinary: bool, ) -> Result>, PendingCallsError> { let pending_call = Arc::new(PendingCall::new()); let mut state = self.state.lock(); - while state.calls.len() == MAX_PENDING_CALLS && state.failure.is_none() { + while (state.calls.len() >= MAX_PENDING_CALLS + || (ordinary && state.ordinary_calls >= MAX_ORDINARY_PENDING_CALLS)) + && state.failure.is_none() + { state = self.capacity_available.wait(state); } if let Some(error) = state.failure.as_ref() { @@ -146,7 +176,13 @@ impl PendingCalls { } match state.calls.entry(request_id) { Entry::Vacant(entry) => { - entry.insert(Arc::clone(&pending_call)); + entry.insert(RegisteredPendingCall { + call: Arc::clone(&pending_call), + ordinary, + }); + if ordinary { + state.ordinary_calls += 1; + } } Entry::Occupied(_) => return Err(PendingCallsError::DuplicateRequestId), } @@ -154,17 +190,28 @@ impl PendingCalls { } /// Completes the pending call identified by `response`. + /// + /// # Panics + /// + /// Panics if the internal ordinary-call count is inconsistent with the + /// registered requests. pub fn complete(&self, response: BrokerResponse) -> Result<(), PendingCallsError> { let pending_call = { let mut state = self.state.lock(); if let Some(error) = state.failure.as_ref() { return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); } - let Some(pending_call) = state.calls.remove(&response.request_id) else { + let Some(registered) = state.calls.remove(&response.request_id) else { return Err(PendingCallsError::UnknownResponseId); }; - self.capacity_available.notify_one(); - pending_call + if registered.ordinary { + state.ordinary_calls = state + .ordinary_calls + .checked_sub(1) + .expect("ordinary pending-call count must remain balanced"); + } + self.capacity_available.notify_all(); + registered.call }; pending_call.resolve(Ok(response)); Ok(()) @@ -181,11 +228,12 @@ impl PendingCalls { } state.failure = Some(Arc::clone(&error)); let pending_calls = core::mem::take(&mut state.calls); + state.ordinary_calls = 0; self.capacity_available.notify_all(); pending_calls }; - for pending_call in pending_calls.into_values() { - pending_call.resolve(Err(Arc::clone(&error))); + for registered in pending_calls.into_values() { + registered.call.resolve(Err(Arc::clone(&error))); } true } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 03b5df7bd1..e7b4f90c04 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -22,8 +22,8 @@ use rustix::net::{ }; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, - BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, + BrokerRequest, BrokerResponse, }; use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, @@ -321,10 +321,16 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; + let pending_call = if matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + | BrokerOperation::ReportProcessStartFailure(_) + ) { + association.pending_calls.register_lifecycle(request_id) + } else { + association.pending_calls.register(request_id) + } + .map_err(pending_calls_error)?; let request_frame = encode_request(request); let write_result = { @@ -767,11 +773,15 @@ mod control_ring_tests { } #[test] - fn pending_capacity_blocks_before_sixty_fifth_publication() { + fn pending_capacity_reserves_lifecycle_calls() { + use litebox_broker_transport::pending_calls::{ + MAX_ORDINARY_PENDING_CALLS, RESERVED_LIFECYCLE_PENDING_CALLS, + }; + let (channel, shutdown, mut responses, mut requests, _peer) = activate_local(|| {}); let channel = Arc::new(channel); let start = Arc::new(Barrier::new(MAX_PENDING_CALLS + 2)); - let callers = (0..=MAX_PENDING_CALLS) + let mut callers = (0..=MAX_ORDINARY_PENDING_CALLS) .map(|id| { let channel = Arc::clone(&channel); let start = Arc::clone(&start); @@ -781,15 +791,53 @@ mod control_ring_tests { }) }) .collect::>(); + callers.extend((0..RESERVED_LIFECYCLE_PENDING_CALLS).map(|index| { + let lifecycle_channel = Arc::clone(&channel); + let lifecycle_start = Arc::clone(&start); + thread::spawn(move || { + lifecycle_start.wait(); + lifecycle_channel.call(BrokerRequest { + request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), + operation: BrokerOperation::AcknowledgeProcessStart( + litebox_broker_protocol::process::ProcessStartToken(index as u64), + ), + }) + }) + })); start.wait(); let mut published = Vec::new(); for _ in 0..MAX_PENDING_CALLS { - published.push(read_request(&mut requests).request_id); + published.push(read_request(&mut requests)); } - write_payload(&mut responses, &encode_response(response(published[0]))); - let released = read_request(&mut requests).request_id; - assert!(!published.contains(&released)); + assert!(published.iter().any(|request| matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + ))); + assert_eq!( + published + .iter() + .filter(|request| matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + )) + .count(), + RESERVED_LIFECYCLE_PENDING_CALLS + ); + let released_request = published + .iter() + .find(|request| matches!(&request.operation, BrokerOperation::CloseObject(_))) + .unwrap(); + write_payload( + &mut responses, + &encode_response(response(released_request.request_id)), + ); + let released = read_request(&mut requests); + assert!( + !published + .iter() + .any(|request| request.request_id == released.request_id) + ); shutdown.shutdown().unwrap(); let completed = callers diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 96d263d6ae..6c33ebd523 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -12,8 +12,8 @@ use std::thread; use std::time::{Duration, Instant}; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, - BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, + BrokerRequest, BrokerResponse, }; use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, @@ -230,10 +230,16 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; let request_id = request.request_id; - let pending_call = association - .pending_calls - .register(request_id) - .map_err(pending_calls_error)?; + let pending_call = if matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + | BrokerOperation::ReportProcessStartFailure(_) + ) { + association.pending_calls.register_lifecycle(request_id) + } else { + association.pending_calls.register(request_id) + } + .map_err(pending_calls_error)?; let frame = encode_request(request); let write_result = { let mut producer = association diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index a098d6437a..e863847184 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -474,7 +474,11 @@ mod tests { } #[test] - fn pending_capacity_blocks_before_sixty_fifth_publication() { + fn pending_capacity_reserves_lifecycle_calls() { + use litebox_broker_transport::pending_calls::{ + MAX_ORDINARY_PENDING_CALLS, RESERVED_LIFECYCLE_PENDING_CALLS, + }; + let control_name = pipe_name("pending-capacity-control"); let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); let (local_ring, host_ring) = control_rings(); @@ -499,18 +503,40 @@ mod tests { let HostReceive::Message(request) = requests.recv_request().unwrap() else { panic!("expected pending request"); }; - published.push(request.request_id); + published.push(request); } + assert!(published.iter().any(|request| matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + ))); + assert_eq!( + published + .iter() + .filter(|request| matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + )) + .count(), + RESERVED_LIFECYCLE_PENDING_CALLS + ); + let released_request = published + .iter() + .find(|request| matches!(&request.operation, BrokerOperation::CloseObject(_))) + .unwrap(); responses .send_response(&BrokerResponse { - request_id: published[0], + request_id: released_request.request_id, result: BrokerResult::ObjectClosed, }) .unwrap(); let HostReceive::Message(released) = requests.recv_request().unwrap() else { panic!("expected released request"); }; - assert!(!published.contains(&released.request_id)); + assert!( + !published + .iter() + .any(|request| request.request_id == released.request_id) + ); shutdown.shutdown().unwrap(); }); @@ -529,7 +555,7 @@ mod tests { let start = Arc::new(Barrier::new( litebox_broker_transport::pending_calls::MAX_PENDING_CALLS + 2, )); - let callers = (0..=litebox_broker_transport::pending_calls::MAX_PENDING_CALLS) + let mut callers = (0..=MAX_ORDINARY_PENDING_CALLS) .map(|id| { let calls = Arc::clone(&calls); let start = Arc::clone(&start); @@ -542,6 +568,19 @@ mod tests { }) }) .collect::>(); + callers.extend((0..RESERVED_LIFECYCLE_PENDING_CALLS).map(|index| { + let lifecycle_calls = Arc::clone(&calls); + let lifecycle_start = Arc::clone(&start); + std::thread::spawn(move || { + lifecycle_start.wait(); + lifecycle_calls.call(BrokerRequest { + request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), + operation: BrokerOperation::AcknowledgeProcessStart( + litebox_broker_protocol::process::ProcessStartToken(index as u64), + ), + }) + }) + })); start.wait(); host.join().unwrap(); diff --git a/litebox_broker_userland/Cargo.toml b/litebox_broker_userland/Cargo.toml index 6855ddec83..24c765fe12 100644 --- a/litebox_broker_userland/Cargo.toml +++ b/litebox_broker_userland/Cargo.toml @@ -13,6 +13,7 @@ litebox_broker_transport = { path = "../litebox_broker_transport", version = "0. litebox_platform = { path = "../litebox_platform", version = "0.1.0" } [target.'cfg(target_os = "linux")'.dependencies] +libc = { version = "0.2.177", default-features = false } litebox_broker_platform_linux_userland = { path = "../litebox_broker_platform_linux_userland", version = "0.1.0" } litebox_broker_transport_linux_userland = { path = "../litebox_broker_transport_linux_userland", version = "0.1.0" } litebox_platform_linux_userland = { path = "../litebox_platform_linux_userland", version = "0.1.0" } @@ -23,6 +24,7 @@ tempfile = { version = "3", default-features = false } litebox_broker_platform_windows_userland = { path = "../litebox_broker_platform_windows_userland", version = "0.1.0" } litebox_broker_transport_windows_userland = { path = "../litebox_broker_transport_windows_userland", version = "0.1.0" } litebox_runner_windows_userland = { path = "../litebox_runner_windows_userland", version = "0.1.0" } +windows-sys = { version = "0.60.2", features = ["Win32_Foundation", "Win32_System_Threading"] } [features] lock_tracing = [ diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 4880ec301f..dbda821aae 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -7,10 +7,13 @@ use std::ffi::{OsStr, OsString}; use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::path::PathBuf; use std::process::{Child, Command, ExitStatus}; -use std::sync::{Arc, Condvar, Mutex}; +use std::sync::{ + Arc, Condvar, Mutex, MutexGuard, + atomic::{AtomicBool, Ordering}, +}; use std::time::{Duration, Instant}; -use litebox_broker_core::{BrokerCore, BrokerProcess}; +use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; use litebox_broker_host::{BrokerHostExtensionError, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; @@ -21,21 +24,30 @@ use litebox_broker_protocol::process::{ use litebox_broker_protocol::{ObjectHandle, ProcessId, ThreadId}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; +use crate::runtime::AssociationFailureCause; + #[cfg(target_os = "linux")] mod linux; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(target_os = "linux")] -use linux::PlatformRunnerEndpoint; +use linux::{PlatformRunnerEndpoint, PlatformRunnerShutdown}; #[cfg(all(windows, target_arch = "x86_64"))] -use windows::PlatformRunnerEndpoint; +use windows::{PlatformRunnerEndpoint, PlatformRunnerShutdown}; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_RECEIPT_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_EXIT_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); const CHILD_ARGUMENT: &str = "--child"; const MAX_PENDING_CHILD_STARTS: usize = crate::WORKER_COUNT - 1; const _: () = assert!(MAX_PENDING_CHILD_STARTS > 0); +const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_WORKER_COUNT > MAX_PENDING_CHILD_STARTS); +const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_QUEUE_CAPACITY >= MAX_PENDING_CHILD_STARTS); /// Configuration for starting one out-of-process runner. /// @@ -95,11 +107,127 @@ impl RunnerConfig { /// Dropping an instance before [`Self::run_to_completion`] completes /// terminates and reaps the runner. pub struct RunnerInstance { - runner: Child, + runner: Arc>, + shutdown: Arc, endpoint: PlatformRunnerEndpoint, child_config: RunnerConfig, } +struct ChildRunResult { + result: IoResult, + runner_success: Option, + runner_signal: Option, + runner_exit_code: Option, + termination_provenance: ChildTerminationProvenance, + association_panicked: bool, + shutdown_observation_failed: bool, +} + +struct RunnerShutdown { + platform: PlatformRunnerShutdown, + state: Mutex, + changed: Condvar, + termination_dispatched: AtomicBool, +} + +#[derive(Clone, Copy, Default)] +struct ChildTerminationProvenance(u8); + +impl ChildTerminationProvenance { + const BROKER_TERMINATION: u8 = 1; + const REPORTED_START_FAILURE: u8 = 2; + + const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { + let mut value = 0; + if broker_termination { + value |= Self::BROKER_TERMINATION; + } + if reported_start_failure { + value |= Self::REPORTED_START_FAILURE; + } + Self(value) + } + + const fn broker_termination(self) -> bool { + self.0 & Self::BROKER_TERMINATION != 0 + } + + const fn reported_start_failure(self) -> bool { + self.0 & Self::REPORTED_START_FAILURE != 0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RunnerShutdownState { + Active, + Firing, + Fired, + Retired, +} + +impl RunnerShutdown { + fn shutdown(&self) { + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + loop { + match *state { + RunnerShutdownState::Active => { + *state = RunnerShutdownState::Firing; + break; + } + RunnerShutdownState::Firing => { + state = self + .changed + .wait(state) + .expect("runner shutdown mutex poisoned"); + } + RunnerShutdownState::Fired | RunnerShutdownState::Retired => return, + } + } + drop(state); + if !matches!(self.platform.has_exited(), Ok(true)) && self.platform.shutdown() { + self.termination_dispatched.store(true, Ordering::Release); + } + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + debug_assert_eq!(*state, RunnerShutdownState::Firing); + *state = RunnerShutdownState::Fired; + self.changed.notify_all(); + } + + fn retire(&self) { + let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); + while *state == RunnerShutdownState::Firing { + state = self + .changed + .wait(state) + .expect("runner shutdown mutex poisoned"); + } + *state = RunnerShutdownState::Retired; + self.changed.notify_all(); + } + + fn has_exited(&self) -> IoResult { + self.platform.has_exited() + } + + fn termination_was_dispatched(&self) -> bool { + self.termination_dispatched.load(Ordering::Acquire) + } + + fn wait_for_exit(&self, timeout: Duration) -> IoResult { + let deadline = Instant::now() + timeout; + loop { + if self.has_exited()? { + return Ok(true); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(false); + } + std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); + } + } +} + impl RunnerInstance { /// Creates the runner's dedicated control endpoint and starts the runner. pub fn start(config: RunnerConfig) -> IoResult { @@ -107,9 +235,16 @@ impl RunnerInstance { let runner = Command::new(&config.executable) .args(config.arguments(endpoint.control_channel())) .spawn()?; + let shutdown = Arc::new(RunnerShutdown { + platform: PlatformRunnerShutdown::new(&runner), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }); let child_config = config.child(); Ok(Self { - runner, + runner: Arc::new(Mutex::new(runner)), + shutdown, endpoint, child_config, }) @@ -120,17 +255,46 @@ impl RunnerInstance { /// Association failure terminates the runner before it is reaped. A /// non-successful runner exit is returned as ordinary instance data for the /// caller to interpret. + /// + /// # Panics + /// + /// Panics if another runner owner poisoned the process mutex. pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { let children = RunnerChildren::new(self.child_config.clone(), broker.clone()); - let association_result = self.endpoint.serve(&mut self.runner, Arc::clone(&children)); + let mut association_result = self.endpoint.serve(&self.runner, Arc::clone(&children)); self.endpoint.close(); - if association_result.is_err() { - let _ = self.runner.kill(); + let runner_exited = if association_result.result.is_ok() { + self.shutdown + .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) + } else { + self.shutdown.has_exited() + }; + if !matches!(runner_exited, Ok(true)) { + self.shutdown.shutdown(); + } + self.shutdown.retire(); + let runner_status = self + .runner + .lock() + .expect("runner process mutex poisoned") + .wait(); + let root_abnormal = association_result.abnormal + || association_result.panicked + || runner_exited.is_err() + || runner_status.is_err() + || runner_status.as_ref().is_ok_and(|status| { + runner_signal_is_abnormal( + runner_exit_signal(*status), + self.shutdown.termination_was_dispatched(), + ) || runner_exit_code_is_crash(status.code()) + }); + if let Some(process) = association_result.process.take() { + finish_child_process(&process, root_abnormal); } - let runner_status = self.runner.wait(); children.wait_for_drain(); let runner_status = runner_status?; - association_result?; + runner_exited?; + association_result.result?; Ok(runner_status) } @@ -138,24 +302,95 @@ impl RunnerInstance { mut self, child: ChildRunner, children: Arc, - ) -> IoResult { - let association_result = self.endpoint.serve_child(&mut self.runner, child, children); + ) -> ChildRunResult { + let launch = Arc::clone(&child.launch); + launch.install_shutdown(Arc::clone(&self.shutdown)); + let association_result = self.endpoint.serve_child(&self.runner, child, children); self.endpoint.close(); - if association_result.is_err() { - let _ = self.runner.kill(); + let shutdown_request = launch.shutdown_request(); + let shutdown_was_expected = shutdown_request.was_expected(); + if association_result.abnormal { + launch.mark_abnormal(); + } + let runner_exited = if !shutdown_was_expected + && matches!( + association_result.failure_cause, + AssociationFailureCause::None | AssociationFailureCause::PeerClosed + ) + && !association_result.panicked + { + self.shutdown + .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) + } else { + self.shutdown.has_exited() + }; + let shutdown_observation_failed = match runner_exited { + Ok(true) => { + if association_result.failure_cause == AssociationFailureCause::Other { + launch.mark_abnormal(); + } + false + } + Ok(false) => { + if !shutdown_was_expected + || association_result.failure_cause == AssociationFailureCause::Other + { + launch.mark_abnormal(); + } + launch.mark_shutdown_expected(); + self.shutdown.shutdown(); + false + } + Err(_) => { + launch.mark_abnormal(); + self.shutdown.shutdown(); + true + } + }; + self.shutdown.retire(); + let runner_status = self + .runner + .lock() + .expect("runner process mutex poisoned") + .wait(); + if runner_status.is_err() { + launch.mark_abnormal(); + } + let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); + let runner_signal = runner_status + .as_ref() + .ok() + .copied() + .and_then(runner_exit_signal); + let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); + let termination_provenance = ChildTerminationProvenance::new( + self.shutdown.termination_was_dispatched(), + shutdown_request.expected_start_failure_was_reported(), + ); + let result = runner_status.and_then(|status| { + association_result.result?; + Ok(status) + }); + ChildRunResult { + result, + runner_success, + runner_signal, + runner_exit_code, + termination_provenance, + association_panicked: association_result.panicked, + shutdown_observation_failed, } - let runner_status = self.runner.wait()?; - association_result?; - Ok(runner_status) } } impl Drop for RunnerInstance { fn drop(&mut self) { self.endpoint.close(); - if !matches!(self.runner.try_wait(), Ok(Some(_status))) { - let _ = self.runner.kill(); - let _ = self.runner.wait(); + self.shutdown.shutdown(); + self.shutdown.retire(); + let mut runner = self.runner.lock().expect("runner process mutex poisoned"); + if !matches!(runner.try_wait(), Ok(Some(_status))) { + let _ = runner.wait(); } } } @@ -169,6 +404,7 @@ pub(crate) struct RunnerChildren { pub(crate) struct ChildRunner { pub(crate) process: Arc, + pub(crate) launch: Arc, pub(crate) inherited_objects: Vec, pub(crate) format: ProcessBootstrapFormat, pub(crate) version: ProcessBootstrapVersion, @@ -177,27 +413,112 @@ pub(crate) struct ChildRunner { struct RunnerChildrenState { launches: Vec<(ProcessStartToken, Arc)>, + associations: Vec<(ProcessId, AssociationFailure)>, active_instances: usize, + active_watchdogs: usize, } -struct ChildLaunch { +pub(crate) type AssociationFailure = Arc; + +pub(crate) struct ChildLaunch { parent_id: ProcessId, process: Arc, - state: Mutex, + state: Mutex, changed: Condvar, } #[derive(Clone, Copy)] -enum ChildLaunchState { +enum ChildLaunchPhase { Starting, - Ready { - initial_thread_id: Option, - start_result_delivered: bool, - }, + Ready { initial_thread_id: Option }, + Committing, Committed, Aborted(ErrorCode), } +#[derive(Clone, Copy, PartialEq, Eq)] +enum StartResultPublication { + NotStarted, + Publishing, + Delivered, +} + +struct ChildLaunchData { + phase: ChildLaunchPhase, + publication: StartResultPublication, + shutdown: Option>, + association_failure: Option, + shutdown_request: ShutdownRequest, + abnormal: bool, + receipt: ReceiptState, + resolution_watchdog: DeadlineState, + acknowledgement_publication_watchdog: DeadlineState, + start_failure_publication_watchdog: DeadlineState, + active_control_callbacks: usize, + pinned_initial_thread_id: Option, + runner_finished: bool, + finalization_taken: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ShutdownRequest { + None, + Expected, + ExpectedStartFailurePending, + ExpectedStartFailure, + Unexpected, +} + +impl ShutdownRequest { + const fn was_expected(self) -> bool { + matches!( + self, + Self::Expected | Self::ExpectedStartFailurePending | Self::ExpectedStartFailure + ) + } + + const fn expected_start_failure_was_reported(self) -> bool { + matches!( + self, + Self::ExpectedStartFailurePending | Self::ExpectedStartFailure + ) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReceiptState { + AwaitingAcknowledgement, + AcknowledgementAdmitted, + TimeoutPending, + Draining, + Resolved, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum DeadlineState { + Unarmed, + Armed(Instant), + Disarmed, + Fired, +} + +enum ProcessStartAcknowledgement { + Acknowledged, + Failed(ErrorCode), +} + +struct ReceiptExpiration { + shutdown: Option>, + association_failure: Option, + fail_parent: bool, + commit_supervision_deadline: Option, +} + +enum ReceiptResolution { + Resolved(Option), + DeferredToDrain, +} + impl RunnerChildren { fn new(config: RunnerConfig, broker: BrokerCore) -> Arc { Arc::new(Self { @@ -205,7 +526,9 @@ impl RunnerChildren { config, state: Mutex::new(RunnerChildrenState { launches: Vec::new(), + associations: Vec::new(), active_instances: 0, + active_watchdogs: 0, }), drained: Condvar::new(), }) @@ -236,29 +559,72 @@ impl RunnerChildren { }) .map(BrokerResult::ProcessStarted), ), - BrokerOperation::AcknowledgeProcessStart(token) => Some( - self.acknowledge_process_start(process.id(), *token) - .map(|()| BrokerResult::ProcessStartAcknowledged) - .map_err(process_extension_error), - ), + BrokerOperation::AcknowledgeProcessStart(token) => { + Some(match self.resolve_acknowledgement(process.id(), *token) { + Ok(ProcessStartAcknowledgement::Acknowledged) => { + Ok(BrokerResult::ProcessStartAcknowledged) + } + Ok(ProcessStartAcknowledgement::Failed(error)) => { + Ok(BrokerResult::ProcessStartFailed(error)) + } + Err(error) => Err(BrokerHostExtensionError::Abort(error)), + }) + } BrokerOperation::ProcessReady(request) => Some( self.process_ready(process.id(), request.initial_thread_id) .map(|()| BrokerResult::ProcessReady) .map_err(process_extension_error), ), + BrokerOperation::ReportProcessStartFailure(error) => Some( + self.report_process_start_failure(process.id(), *error) + .map(|()| BrokerResult::ProcessStartFailed(*error)) + .map_err(process_extension_error), + ), _ => None, } } - pub(crate) fn response_sent(&self, parent_id: ProcessId, result: &BrokerResult) { - let BrokerResult::ProcessStarted(started) = result else { - return; - }; - let Some(launch) = self.find_launch(started.token) else { - return; - }; - if launch.parent_id == parent_id { - launch.mark_start_result_delivered(); + pub(crate) fn response_sent( + &self, + process_id: ProcessId, + operation: &BrokerOperation, + result: &BrokerResult, + ) { + match (operation, result) { + (_, BrokerResult::ProcessStarted(started)) => { + let Some(launch) = self.find_launch(started.token) else { + return; + }; + if launch.parent_id == process_id { + launch.mark_start_result_delivered(); + } + } + ( + BrokerOperation::AcknowledgeProcessStart(token), + BrokerResult::ProcessStartAcknowledged | BrokerResult::ProcessStartFailed(_), + ) => { + let Some(launch) = self.find_launch(*token) else { + return; + }; + if launch.parent_id != process_id { + return; + } + if let ReceiptResolution::Resolved(finalization) = launch.resolve_receipt() { + self.remove_launch(*token); + if let Some(abnormal) = finalization { + self.finish_child_launch(&launch, abnormal); + } + } + } + ( + BrokerOperation::ReportProcessStartFailure(error), + BrokerResult::ProcessStartFailed(reported), + ) if error == reported => { + if let Some(launch) = self.find_child_launch(process_id) { + launch.start_failure_response_sent(*error); + } + } + _ => {} } } @@ -280,7 +646,22 @@ impl RunnerChildren { let launch = Arc::new(ChildLaunch { parent_id: parent.id(), process: Arc::clone(&process), - state: Mutex::new(ChildLaunchState::Starting), + state: Mutex::new(ChildLaunchData { + phase: ChildLaunchPhase::Starting, + publication: StartResultPublication::NotStarted, + shutdown: None, + association_failure: None, + shutdown_request: ShutdownRequest::None, + abnormal: false, + receipt: ReceiptState::AwaitingAcknowledgement, + resolution_watchdog: DeadlineState::Unarmed, + acknowledgement_publication_watchdog: DeadlineState::Unarmed, + start_failure_publication_watchdog: DeadlineState::Unarmed, + active_control_callbacks: 0, + pinned_initial_thread_id: None, + runner_finished: false, + finalization_taken: false, + }), changed: Condvar::new(), }); let token = match (|| { @@ -332,10 +713,11 @@ impl RunnerChildren { .name(format!("litebox-runner-{}", child_id.0)) .spawn(move || { let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - RunnerInstance::start(config).and_then(|instance| { + RunnerInstance::start(config).map(|instance| { instance.run_child_to_completion( ChildRunner { process, + launch: Arc::clone(&thread_launch), inherited_objects, format, version, @@ -346,11 +728,33 @@ impl RunnerChildren { }) })); match outcome { - Ok(result) => children.child_finished(token, &thread_launch, result, false), + Ok(Ok(result)) => children.child_finished(token, &thread_launch, result, false), + Ok(Err(error)) => children.child_finished( + token, + &thread_launch, + ChildRunResult { + result: Err(error), + runner_success: None, + runner_signal: None, + runner_exit_code: None, + termination_provenance: ChildTerminationProvenance::default(), + association_panicked: false, + shutdown_observation_failed: false, + }, + false, + ), Err(_) => children.child_finished( token, &thread_launch, - Err(IoError::other("child runner thread panicked")), + ChildRunResult { + result: Err(IoError::other("child runner thread panicked")), + runner_success: None, + runner_signal: None, + runner_exit_code: None, + termination_provenance: ChildTerminationProvenance::default(), + association_panicked: false, + shutdown_observation_failed: false, + }, true, ), } @@ -364,6 +768,21 @@ impl RunnerChildren { drop(thread); let initial_thread_id = launch.wait_until_ready()?; + let receipt_deadline = launch.begin_start_result_publication()?; + if self + .arm_initial_receipt_deadline(token, &launch, receipt_deadline) + .is_err() + { + if let Some(expiration) = launch.expire_initial_receipt_deadline() { + self.apply_receipt_expiration( + token, + &launch, + expiration, + self.find_association_failure(launch.parent_id), + ); + } + return Err(ErrorCode::OutOfMemory); + } Ok(StartedProcess { token, process_id: child_id, @@ -371,8 +790,8 @@ impl RunnerChildren { }) } - fn acknowledge_process_start( - &self, + pub(crate) fn admit_acknowledgement( + self: &Arc, parent_id: ProcessId, token: ProcessStartToken, ) -> Result<(), ErrorCode> { @@ -380,11 +799,50 @@ impl RunnerChildren { if launch.parent_id != parent_id { return Err(ErrorCode::UnknownObject); } - launch.commit()?; - self.remove_launch(token); + launch.admit_acknowledgement()?; + if self + .arm_internal_resolution_watchdog(token, &launch) + .is_err() + { + if let Some(expiration) = launch.fail_internal_resolution_watchdog() { + self.apply_receipt_expiration( + token, + &launch, + expiration, + self.find_association_failure(launch.parent_id), + ); + } + return Err(ErrorCode::Internal); + } + if self + .arm_acknowledgement_publication_watchdog(token, &launch) + .is_err() + { + if let Some(expiration) = launch.fail_internal_resolution_watchdog() { + self.apply_receipt_expiration( + token, + &launch, + expiration, + self.find_association_failure(launch.parent_id), + ); + } + return Err(ErrorCode::Internal); + } Ok(()) } + fn resolve_acknowledgement( + &self, + parent_id: ProcessId, + token: ProcessStartToken, + ) -> Result { + let launch = self.find_launch(token).ok_or(ErrorCode::PeerClosed)?; + if launch.parent_id != parent_id { + return Err(ErrorCode::UnknownObject); + } + launch.resolve_acknowledgement() + } + fn process_ready( &self, child_id: ProcessId, @@ -392,211 +850,1307 @@ impl RunnerChildren { ) -> Result<(), ErrorCode> { let launch = self .find_child_launch(child_id) - .ok_or(ErrorCode::ProtocolState)?; + .ok_or(ErrorCode::PeerClosed)?; launch.ready_and_wait(initial_thread_id) } - pub(crate) fn association_ended(&self, process_id: ProcessId) { - loop { - let launch = { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state - .launches - .iter() - .position(|(_, launch)| { - launch.parent_id == process_id || launch.process.id() == process_id - }) - .map(|index| state.launches.swap_remove(index).1) - }; - let Some(launch) = launch else { - break; - }; - launch.abort(ErrorCode::PeerClosed); + fn report_process_start_failure( + self: &Arc, + child_id: ProcessId, + error: ErrorCode, + ) -> Result<(), ErrorCode> { + let (token, launch) = self + .find_child_launch_entry(child_id) + .ok_or(ErrorCode::PeerClosed)?; + launch.report_start_failure(error)?; + if self + .arm_start_failure_publication_watchdog(token, &launch) + .is_err() + { + if let Some(expiration) = launch.fail_start_failure_publication_watchdog() { + self.apply_receipt_expiration(token, &launch, expiration, None); + } + return Err(ErrorCode::Internal); } + Ok(()) } - fn find_launch(&self, token: ProcessStartToken) -> Option> { - self.state + pub(crate) fn association_ending(&self, process_id: ProcessId) -> Vec> { + let draining = self + .state .lock() .expect("runner child state mutex poisoned") .launches .iter() - .find_map(|(candidate, launch)| (*candidate == token).then(|| Arc::clone(launch))) + .filter(|(_, launch)| launch.parent_id == process_id) + .map(|(_, launch)| Arc::clone(launch)) + .collect::>(); + for launch in &draining { + launch.abort(ErrorCode::PeerClosed, false, true); + launch.begin_receipt_drain(); + } + + let child_launch = { + self.state + .lock() + .expect("runner child state mutex poisoned") + .launches + .iter() + .find_map(|(_, launch)| { + (launch.process.id() == process_id).then(|| Arc::clone(launch)) + }) + }; + if let Some(launch) = child_launch { + launch.association_closed(); + } + draining } - fn find_child_launch(&self, child_id: ProcessId) -> Option> { - self.state - .lock() - .expect("runner child state mutex poisoned") - .launches - .iter() - .find_map(|(_, launch)| (launch.process.id() == child_id).then(|| Arc::clone(launch))) + pub(crate) fn association_ended(&self, process_id: ProcessId, draining: Vec>) { + self.unregister_association(process_id); + for launch in draining { + self.remove_launch_by_process_id(launch.process.id()); + if let Some(abnormal) = launch.finish_receipt_drain() { + self.finish_child_launch(&launch, abnormal); + } + } } - fn remove_launch(&self, token: ProcessStartToken) { + pub(crate) fn register_association( + &self, + process_id: ProcessId, + failure: AssociationFailure, + ) -> IoResult<()> { let mut state = self .state .lock() .expect("runner child state mutex poisoned"); - if let Some(index) = state - .launches + state + .associations + .try_reserve(1) + .map_err(|_| IoError::other("failed to reserve broker association registration"))?; + if state + .associations .iter() - .position(|(candidate, _)| *candidate == token) + .any(|(candidate, _)| *candidate == process_id) { - state.launches.swap_remove(index); + return Err(IoError::other( + "a broker process already has a live association", + )); } + state.associations.push((process_id, failure)); + Ok(()) } - fn child_finished( + pub(crate) fn install_child_association_failure( &self, - token: ProcessStartToken, - launch: &ChildLaunch, - result: IoResult, - panicked: bool, + process_id: ProcessId, + failure: AssociationFailure, ) { - if result.is_err() || result.is_ok_and(|status| !status.success()) { - launch.abort(ErrorCode::PeerClosed); - } - self.remove_launch(token); - if !panicked { - BrokerProcess::finish(Arc::clone(&launch.process)); + if let Some(launch) = self.find_child_launch(process_id) { + launch.install_association_failure(failure); } - self.finish_instance(); } - fn finish_instance(&self) { + fn unregister_association(&self, process_id: ProcessId) { let mut state = self .state .lock() .expect("runner child state mutex poisoned"); - state.active_instances = state - .active_instances - .checked_sub(1) - .expect("runner child count must remain balanced"); - if state.active_instances == 0 { - self.drained.notify_all(); + if let Some(index) = state + .associations + .iter() + .position(|(candidate, _)| *candidate == process_id) + { + state.associations.swap_remove(index); } } - fn wait_for_drain(&self) { - let mut state = self - .state + fn find_association_failure(&self, process_id: ProcessId) -> Option { + self.state .lock() - .expect("runner child state mutex poisoned"); - while state.active_instances != 0 { - state = self - .drained - .wait(state) - .expect("runner child state mutex poisoned"); - } + .expect("runner child state mutex poisoned") + .associations + .iter() + .find_map(|(candidate, failure)| { + (*candidate == process_id).then(|| Arc::clone(failure)) + }) } -} -const fn process_extension_error(error: ErrorCode) -> BrokerHostExtensionError { - match error { - ErrorCode::PolicyDenied - | ErrorCode::UnknownObject - | ErrorCode::InvalidRights - | ErrorCode::ResourceExhausted - | ErrorCode::WouldBlock - | ErrorCode::PeerClosed - | ErrorCode::OutOfMemory - | ErrorCode::UnsupportedOperation => BrokerHostExtensionError::Respond(error), - _ => BrokerHostExtensionError::Abort(error), + fn arm_initial_receipt_deadline( + self: &Arc, + token: ProcessStartToken, + launch: &Arc, + deadline: Instant, + ) -> Result<(), ()> { + let launch = Arc::clone(launch); + let parent_failure = self.find_association_failure(launch.parent_id); + self.spawn_watchdog( + format!("litebox-start-receipt-{}", launch.process.id().0), + move |children| { + let Some(expiration) = launch.wait_for_initial_receipt_deadline(deadline) else { + return; + }; + children.apply_receipt_expiration(token, &launch, expiration, parent_failure); + }, + ) } -} -impl ChildLaunch { - fn wait_until_ready(&self) -> Result, ErrorCode> { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match *state { - ChildLaunchState::Starting => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - ChildLaunchState::Ready { - initial_thread_id, .. - } => return Ok(initial_thread_id), - ChildLaunchState::Committed => return Err(ErrorCode::ProtocolState), - ChildLaunchState::Aborted(error) => return Err(error), - } - } + fn arm_internal_resolution_watchdog( + self: &Arc, + token: ProcessStartToken, + launch: &Arc, + ) -> Result<(), ()> { + let launch = Arc::clone(launch); + let parent_failure = self.find_association_failure(launch.parent_id); + self.spawn_watchdog( + format!("litebox-start-resolution-{}", launch.process.id().0), + move |children| { + let Some(expiration) = launch.wait_for_internal_resolution_timeout() else { + return; + }; + children.apply_receipt_expiration(token, &launch, expiration, parent_failure); + }, + ) } - fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if !matches!(*state, ChildLaunchState::Starting) { - return Err(ErrorCode::ProtocolState); - } - *state = ChildLaunchState::Ready { - initial_thread_id, - start_result_delivered: false, - }; - self.changed.notify_all(); - loop { - match *state { - ChildLaunchState::Ready { .. } => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - ChildLaunchState::Committed => return Ok(()), - ChildLaunchState::Aborted(error) => return Err(error), - ChildLaunchState::Starting => unreachable!("ready state cannot regress"), - } - } + fn arm_acknowledgement_publication_watchdog( + self: &Arc, + token: ProcessStartToken, + launch: &Arc, + ) -> Result<(), ()> { + let launch = Arc::clone(launch); + let parent_failure = self.find_association_failure(launch.parent_id); + self.spawn_watchdog( + format!("litebox-start-ack-publication-{}", launch.process.id().0), + move |children| { + let Some(expiration) = launch.wait_for_acknowledgement_publication_timeout() else { + return; + }; + children.apply_receipt_expiration(token, &launch, expiration, parent_failure); + }, + ) } - fn commit(&self) -> Result<(), ErrorCode> { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if !matches!( - *state, - ChildLaunchState::Ready { - start_result_delivered: true, - .. + fn arm_start_failure_publication_watchdog( + self: &Arc, + token: ProcessStartToken, + launch: &Arc, + ) -> Result<(), ()> { + let launch = Arc::clone(launch); + self.spawn_watchdog( + format!( + "litebox-start-failure-publication-{}", + launch.process.id().0 + ), + move |children| { + let Some(expiration) = launch.wait_for_start_failure_publication_timeout() else { + return; + }; + children.apply_receipt_expiration(token, &launch, expiration, None); + }, + ) + } + + fn spawn_watchdog( + self: &Arc, + name: String, + watchdog: impl FnOnce(&Arc) + Send + 'static, + ) -> Result<(), ()> { + { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state.active_watchdogs = state.active_watchdogs.checked_add(1).ok_or(())?; + } + let children = Arc::clone(self); + if let Ok(thread) = std::thread::Builder::new().name(name).spawn(move || { + let _completion = WatchdogCompletion { + children: Arc::clone(&children), + }; + watchdog(&children); + }) { + drop(thread); + Ok(()) + } else { + self.finish_watchdog(); + Err(()) + } + } + + fn apply_receipt_expiration( + &self, + token: ProcessStartToken, + launch: &ChildLaunch, + expiration: ReceiptExpiration, + parent_failure: Option, + ) { + let fail_parent = expiration.fail_parent; + let commit_supervision_deadline = expiration.commit_supervision_deadline; + if let Some(association_failure) = expiration.association_failure { + association_failure(); + } + if let Some(shutdown) = expiration.shutdown { + shutdown.shutdown(); + } + let parent_failed = if fail_parent { + if let Some(parent_failure) = parent_failure { + parent_failure(); + true + } else { + false + } + } else { + true + }; + if let Some(abnormal) = launch.complete_timeout_callback() { + self.finish_child_launch(launch, abnormal); + } + if fail_parent && !parent_failed { + self.remove_launch(token); + if let Some(abnormal) = launch.finish_receipt_drain() { + self.finish_child_launch(launch, abnormal); + } + } + if let Some(deadline) = commit_supervision_deadline + && !launch.wait_for_commit_resolution(deadline) + { + std::process::abort(); + } + } + + fn find_launch(&self, token: ProcessStartToken) -> Option> { + self.state + .lock() + .expect("runner child state mutex poisoned") + .launches + .iter() + .find_map(|(candidate, launch)| (*candidate == token).then(|| Arc::clone(launch))) + } + + fn find_child_launch(&self, child_id: ProcessId) -> Option> { + self.find_child_launch_entry(child_id) + .map(|(_, launch)| launch) + } + + fn find_child_launch_entry( + &self, + child_id: ProcessId, + ) -> Option<(ProcessStartToken, Arc)> { + self.state + .lock() + .expect("runner child state mutex poisoned") + .launches + .iter() + .find_map(|(token, launch)| { + (launch.process.id() == child_id).then(|| (*token, Arc::clone(launch))) + }) + } + + fn remove_launch(&self, token: ProcessStartToken) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + if let Some(index) = state + .launches + .iter() + .position(|(candidate, _)| *candidate == token) + { + state.launches.swap_remove(index); + } + } + + fn remove_launch_by_process_id(&self, process_id: ProcessId) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + if let Some(index) = state + .launches + .iter() + .position(|(_, launch)| launch.process.id() == process_id) + { + state.launches.swap_remove(index); + } + } + + fn child_finished( + &self, + token: ProcessStartToken, + launch: &ChildLaunch, + result: ChildRunResult, + thread_panicked: bool, + ) { + let unexpected_runner_failure = result.runner_success == Some(false) + && result.runner_signal.is_none() + && !launch.commit_was_claimed() + && !result.termination_provenance.reported_start_failure() + && !runner_exit_code_is_expected_shutdown( + result.runner_exit_code, + result.termination_provenance.broker_termination(), + ); + let unexpected_crash = runner_signal_is_abnormal( + result.runner_signal, + result.termination_provenance.broker_termination(), + ); + let abnormal = thread_panicked + || result.association_panicked + || result.shutdown_observation_failed + || unexpected_crash + || runner_exit_code_is_crash(result.runner_exit_code) + || unexpected_runner_failure; + if result.result.is_err() || result.runner_success != Some(true) { + launch.abort(ErrorCode::PeerClosed, abnormal, false); + } else { + // Any clean host exit before commit still aborts creation. + launch.abort(ErrorCode::PeerClosed, false, false); + } + if !launch.retains_published_receipt() { + self.remove_launch(token); + if let ReceiptResolution::Resolved(finalization) = launch.resolve_receipt() { + debug_assert!(finalization.is_none()); + } + } + if let Some(abnormal) = launch.runner_finished(abnormal) { + self.finish_child_launch(launch, abnormal); + } + } + + fn finish_child_launch(&self, launch: &ChildLaunch, abnormal: bool) { + finish_child_process(&launch.process, abnormal); + self.finish_instance(); + } + + fn finish_instance(&self) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state.active_instances = state + .active_instances + .checked_sub(1) + .expect("runner child count must remain balanced"); + if state.active_instances == 0 && state.active_watchdogs == 0 { + self.drained.notify_all(); + } + } + + fn finish_watchdog(&self) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + state.active_watchdogs = state + .active_watchdogs + .checked_sub(1) + .expect("runner watchdog count must remain balanced"); + if state.active_instances == 0 && state.active_watchdogs == 0 { + self.drained.notify_all(); + } + } + + fn wait_for_drain(&self) { + let mut state = self + .state + .lock() + .expect("runner child state mutex poisoned"); + while state.active_instances != 0 || state.active_watchdogs != 0 { + state = self + .drained + .wait(state) + .expect("runner child state mutex poisoned"); + } + } +} + +struct WatchdogCompletion { + children: Arc, +} + +impl Drop for WatchdogCompletion { + fn drop(&mut self) { + self.children.finish_watchdog(); + } +} + +const fn process_extension_error(error: ErrorCode) -> BrokerHostExtensionError { + match error { + ErrorCode::PolicyDenied + | ErrorCode::UnknownObject + | ErrorCode::InvalidRights + | ErrorCode::ResourceExhausted + | ErrorCode::WouldBlock + | ErrorCode::PeerClosed + | ErrorCode::OutOfMemory + | ErrorCode::UnsupportedOperation => BrokerHostExtensionError::Respond(error), + _ => BrokerHostExtensionError::Abort(error), + } +} + +const fn process_start_failure_is_expected(error: ErrorCode) -> bool { + matches!( + error, + ErrorCode::UnsupportedOperation + | ErrorCode::PolicyDenied + | ErrorCode::InvalidRights + | ErrorCode::ResourceExhausted + | ErrorCode::WouldBlock + | ErrorCode::OutOfMemory + ) +} + +impl ChildLaunch { + fn wait_until_ready(&self) -> Result, ErrorCode> { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match state.phase { + ChildLaunchPhase::Starting => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + ChildLaunchPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), + ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => { + return Err(ErrorCode::ProtocolState); + } + ChildLaunchPhase::Aborted(error) => return Err(error), + } + } + } + + fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { + if let Some(thread_id) = initial_thread_id { + self.process + .pin_startup_thread(thread_id) + .map_err(|error| match error { + BrokerError::UnknownObject => ErrorCode::ProtocolState, + error => ErrorCode::from(error), + })?; + } + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if !matches!(state.phase, ChildLaunchPhase::Starting) { + let error = match state.phase { + ChildLaunchPhase::Aborted(error) => error, + _ => ErrorCode::ProtocolState, + }; + drop(state); + if let Some(thread_id) = initial_thread_id { + let _ = self.process.release_startup_thread(thread_id); + } + return Err(error); + } + state.pinned_initial_thread_id = initial_thread_id; + state.phase = ChildLaunchPhase::Ready { initial_thread_id }; + self.changed.notify_all(); + loop { + match state.phase { + ChildLaunchPhase::Committed + if state.pinned_initial_thread_id.is_none() + && state.active_control_callbacks == 0 => + { + return Ok(()); + } + ChildLaunchPhase::Ready { .. } + | ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + ChildLaunchPhase::Aborted(error) => return Err(error), + ChildLaunchPhase::Starting => unreachable!("ready state cannot regress"), + } + } + } + + fn begin_start_result_publication(&self) -> Result { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + match state.phase { + ChildLaunchPhase::Ready { .. } + if state.publication == StartResultPublication::NotStarted => + { + state.publication = StartResultPublication::Publishing; + Ok(Instant::now() + PROCESS_START_RECEIPT_TIMEOUT) + } + ChildLaunchPhase::Aborted(error) => Err(error), + _ => Err(ErrorCode::ProtocolState), + } + } + + fn report_start_failure(&self, error: ErrorCode) -> Result<(), ErrorCode> { + if !process_start_failure_is_expected(error) { + return Err(ErrorCode::ProtocolState); + } + let mut state = self.state.lock().expect("child launch mutex poisoned"); + match state.phase { + ChildLaunchPhase::Starting => {} + ChildLaunchPhase::Aborted(error) => return Err(error), + _ => return Err(ErrorCode::ProtocolState), + } + state.phase = ChildLaunchPhase::Aborted(error); + state.shutdown_request = ShutdownRequest::ExpectedStartFailurePending; + arm_deadline( + &mut state.start_failure_publication_watchdog, + PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, + ); + self.changed.notify_all(); + Ok(()) + } + + fn start_failure_response_sent(&self, error: ErrorCode) { + let (association_failure, shutdown) = { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + complete_deadline(&mut state.start_failure_publication_watchdog); + let actions = if matches!(state.phase, ChildLaunchPhase::Aborted(cause) if cause == error) + && state.shutdown_request == ShutdownRequest::ExpectedStartFailurePending + { + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } else { + (None, None) + }; + self.changed.notify_all(); + actions + }; + if let Some(association_failure) = association_failure { + association_failure(); + } + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn admit_acknowledgement(&self) -> Result<(), ErrorCode> { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if state.publication == StartResultPublication::NotStarted { + return Err(ErrorCode::ProtocolState); + } + match state.receipt { + ReceiptState::AwaitingAcknowledgement => {} + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { + return Err(ErrorCode::PeerClosed); + } + ReceiptState::AcknowledgementAdmitted => return Err(ErrorCode::UnknownObject), + } + state.receipt = ReceiptState::AcknowledgementAdmitted; + if state.publication == StartResultPublication::Delivered + && matches!( + state.phase, + ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) + ) + { + arm_deadline( + &mut state.resolution_watchdog, + PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, + ); + self.changed.notify_all(); + } + Ok(()) + } + + fn resolve_acknowledgement(&self) -> Result { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match state.receipt { + ReceiptState::AcknowledgementAdmitted => {} + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { + return Err(ErrorCode::PeerClosed); + } + ReceiptState::AwaitingAcknowledgement => return Err(ErrorCode::ProtocolState), + } + if state.publication == StartResultPublication::NotStarted { + return Err(ErrorCode::ProtocolState); + } + match (state.phase, state.publication) { + ( + ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_), + StartResultPublication::Publishing, + ) => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + (ChildLaunchPhase::Ready { .. }, StartResultPublication::Delivered) => { + debug_assert!(matches!(state.resolution_watchdog, DeadlineState::Armed(_))); + state.phase = ChildLaunchPhase::Committing; + drop(state); + let commit_result = self.process.commit_start().map_err(ErrorCode::from); + state = self.state.lock().expect("child launch mutex poisoned"); + match commit_result { + Ok(()) => { + state.phase = ChildLaunchPhase::Committed; + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Acknowledged); + } + Err(error) => { + state.phase = ChildLaunchPhase::Aborted(error); + state.abnormal |= error == ErrorCode::Internal; + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Failed(error)); + } + } + } + (ChildLaunchPhase::Aborted(error), StartResultPublication::Delivered) => { + debug_assert!(matches!( + state.resolution_watchdog, + DeadlineState::Armed(_) | DeadlineState::Fired + )); + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Failed(error)); + } + (ChildLaunchPhase::Starting, _) => return Err(ErrorCode::ProtocolState), + (ChildLaunchPhase::Committing | ChildLaunchPhase::Committed, _) => { + return Err(ErrorCode::ProtocolState); + } + (_, StartResultPublication::NotStarted) => { + unreachable!("publication state was checked before phase dispatch") + } + } + } + } + + fn mark_start_result_delivered(&self) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if state.publication == StartResultPublication::Publishing { + state.publication = StartResultPublication::Delivered; + if state.receipt == ReceiptState::AcknowledgementAdmitted + && matches!( + state.phase, + ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) + ) + { + arm_deadline( + &mut state.resolution_watchdog, + PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, + ); + } + self.changed.notify_all(); + } + } + + fn install_shutdown(&self, shutdown: Arc) { + let shutdown = { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.shutdown = Some(Arc::clone(&shutdown)); + (state.shutdown_request != ShutdownRequest::None).then_some(shutdown) + }; + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn install_association_failure(&self, failure: AssociationFailure) { + let failure = { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.association_failure = Some(Arc::clone(&failure)); + (state.shutdown_request != ShutdownRequest::None).then_some(failure) + }; + if let Some(failure) = failure { + failure(); + } + } + + fn abort(&self, error: ErrorCode, abnormal: bool, expected_shutdown: bool) { + let (association_failure, shutdown) = { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.abnormal |= abnormal; + match state.phase { + ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } => { + state.phase = ChildLaunchPhase::Aborted(error); + let newly_requested = state.shutdown_request == ShutdownRequest::None; + if newly_requested { + state.shutdown_request = if expected_shutdown { + ShutdownRequest::Expected + } else { + ShutdownRequest::Unexpected + }; + } + self.changed.notify_all(); + ( + newly_requested + .then(|| state.association_failure.as_ref().map(Arc::clone)) + .flatten(), + state.shutdown.clone(), + ) + } + ChildLaunchPhase::Aborted(_) => match state.shutdown_request { + ShutdownRequest::None => { + state.shutdown_request = if expected_shutdown { + ShutdownRequest::Expected + } else { + ShutdownRequest::Unexpected + }; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ShutdownRequest::ExpectedStartFailurePending => { + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ShutdownRequest::Expected + | ShutdownRequest::ExpectedStartFailure + | ShutdownRequest::Unexpected => (None, None), + }, + ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => (None, None), + } + }; + if let Some(association_failure) = association_failure { + association_failure(); + } + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn association_closed(&self) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.association_failure = None; + if matches!( + state.phase, + ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } + ) { + state.phase = ChildLaunchPhase::Aborted(ErrorCode::PeerClosed); + self.changed.notify_all(); + } + } + + fn mark_shutdown_expected(&self) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + } + + fn mark_abnormal(&self) { + self.state + .lock() + .expect("child launch mutex poisoned") + .abnormal = true; + } + + fn shutdown_request(&self) -> ShutdownRequest { + self.state + .lock() + .expect("child launch mutex poisoned") + .shutdown_request + } + + fn commit_was_claimed(&self) -> bool { + matches!( + self.state + .lock() + .expect("child launch mutex poisoned") + .phase, + ChildLaunchPhase::Committing | ChildLaunchPhase::Committed + ) + } + + fn retains_published_receipt(&self) -> bool { + let state = self.state.lock().expect("child launch mutex poisoned"); + state.publication != StartResultPublication::NotStarted + && state.receipt != ReceiptState::Resolved + } + + fn resolve_receipt(&self) -> ReceiptResolution { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if matches!( + state.receipt, + ReceiptState::TimeoutPending | ReceiptState::Draining + ) { + return ReceiptResolution::DeferredToDrain; + } + state.receipt = ReceiptState::Resolved; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + ReceiptResolution::Resolved(self.release_startup_pin_and_take_finalization(state)) + } + + fn begin_receipt_drain(&self) { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if state.receipt != ReceiptState::Resolved { + state.receipt = ReceiptState::Draining; + if !matches!(state.phase, ChildLaunchPhase::Committing) { + complete_deadline(&mut state.resolution_watchdog); + } + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + } + } + + fn finish_receipt_drain(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.receipt = ReceiptState::Resolved; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + self.release_startup_pin_and_take_finalization(state) + } + + fn expire_initial_receipt_deadline(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + let (error, abnormal) = match (state.receipt, state.publication) { + (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), + (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { + (ErrorCode::Internal, true) + } + _ => return None, + }; + expire_launch(&mut state, error, abnormal, true, &self.changed) + } + + fn wait_for_initial_receipt_deadline(&self, deadline: Instant) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + if !matches!( + (state.receipt, state.publication), + (ReceiptState::AwaitingAcknowledgement, _) + | ( + ReceiptState::AcknowledgementAdmitted, + StartResultPublication::Publishing + ) + ) { + return None; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("child launch mutex poisoned"); + state = next; + } + let (error, abnormal) = match (state.receipt, state.publication) { + (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), + (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { + (ErrorCode::Internal, true) + } + _ => return None, + }; + expire_launch(&mut state, error, abnormal, true, &self.changed) + } + + fn wait_for_internal_resolution_timeout(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match state.resolution_watchdog { + DeadlineState::Unarmed => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.resolution_watchdog = DeadlineState::Fired; + return expire_internal_resolution(&mut state, &self.changed); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("child launch mutex poisoned"); + state = next; + } + DeadlineState::Disarmed | DeadlineState::Fired => return None, + } + } + } + + fn wait_for_acknowledgement_publication_timeout(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match state.acknowledgement_publication_watchdog { + DeadlineState::Unarmed => { + state = self + .changed + .wait(state) + .expect("child launch mutex poisoned"); + } + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.acknowledgement_publication_watchdog = DeadlineState::Fired; + return expire_launch( + &mut state, + ErrorCode::PeerClosed, + false, + true, + &self.changed, + ); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("child launch mutex poisoned"); + state = next; + } + DeadlineState::Disarmed | DeadlineState::Fired => return None, + } + } + } + + fn wait_for_start_failure_publication_timeout(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + loop { + match state.start_failure_publication_watchdog { + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.start_failure_publication_watchdog = DeadlineState::Fired; + return expire_start_failure_publication(&mut state, &self.changed); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("child launch mutex poisoned"); + state = next; + } + DeadlineState::Unarmed | DeadlineState::Disarmed | DeadlineState::Fired => { + return None; + } } + } + } + + fn fail_start_failure_publication_watchdog(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + if !matches!( + state.start_failure_publication_watchdog, + DeadlineState::Armed(_) ) { - return Err(match *state { - ChildLaunchState::Aborted(error) => error, - _ => ErrorCode::ProtocolState, - }); + return None; } - self.process.commit_start().map_err(ErrorCode::from)?; - *state = ChildLaunchState::Committed; + state.start_failure_publication_watchdog = DeadlineState::Fired; + expire_start_failure_publication(&mut state, &self.changed) + } + + fn fail_internal_resolution_watchdog(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + expire_launch(&mut state, ErrorCode::Internal, true, true, &self.changed) + } + + fn complete_timeout_callback(&self) -> Option { + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.active_control_callbacks = state + .active_control_callbacks + .checked_sub(1) + .expect("process-start timeout callback count must remain balanced"); self.changed.notify_all(); - Ok(()) + take_finalization(&mut state) } - fn mark_start_result_delivered(&self) { + fn wait_for_commit_resolution(&self, deadline: Instant) -> bool { let mut state = self.state.lock().expect("child launch mutex poisoned"); - if let ChildLaunchState::Ready { - start_result_delivered, - .. - } = &mut *state - { - *start_result_delivered = true; + while matches!(state.phase, ChildLaunchPhase::Committing) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return false; + } + let (next, wait_result) = self + .changed + .wait_timeout(state, remaining) + .expect("child launch mutex poisoned"); + state = next; + if wait_result.timed_out() && matches!(state.phase, ChildLaunchPhase::Committing) { + return false; + } + } + true + } + + fn release_startup_pin_and_take_finalization( + &self, + mut state: MutexGuard<'_, ChildLaunchData>, + ) -> Option { + let pinned_thread_id = state.pinned_initial_thread_id.take(); + if pinned_thread_id.is_none() { + return take_finalization(&mut state); } + state.active_control_callbacks = state + .active_control_callbacks + .checked_add(1) + .expect("process-start control callback count must remain bounded"); + drop(state); + + let release_failed = self + .process + .release_startup_thread(pinned_thread_id.expect("thread pin was checked")) + .is_err(); + let mut state = self.state.lock().expect("child launch mutex poisoned"); + state.active_control_callbacks = state + .active_control_callbacks + .checked_sub(1) + .expect("process-start control callback count must remain balanced"); + state.abnormal |= release_failed; + self.changed.notify_all(); + take_finalization(&mut state) } - fn abort(&self, error: ErrorCode) { + fn runner_finished(&self, abnormal: bool) -> Option { let mut state = self.state.lock().expect("child launch mutex poisoned"); - if matches!( - *state, - ChildLaunchState::Starting | ChildLaunchState::Ready { .. } - ) { - *state = ChildLaunchState::Aborted(error); - self.changed.notify_all(); + state.abnormal |= abnormal; + state.runner_finished = true; + take_finalization(&mut state) + } +} + +fn expire_launch( + state: &mut ChildLaunchData, + error: ErrorCode, + abnormal: bool, + fail_parent: bool, + changed: &Condvar, +) -> Option { + if matches!( + state.receipt, + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved + ) { + return None; + } + state.active_control_callbacks += 1; + state.receipt = ReceiptState::TimeoutPending; + state.abnormal |= abnormal; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + let (association_failure, shutdown) = match state.phase { + ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } => { + state.phase = ChildLaunchPhase::Aborted(error); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ChildLaunchPhase::Aborted(_) => { + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } else { + (None, None) + } } + ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => (None, None), + }; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown, + association_failure, + fail_parent, + commit_supervision_deadline: None, + }) +} + +fn expire_start_failure_publication( + state: &mut ChildLaunchData, + changed: &Condvar, +) -> Option { + if !matches!(state.phase, ChildLaunchPhase::Aborted(_)) + || state.shutdown_request != ShutdownRequest::ExpectedStartFailurePending + { + return None; + } + state.active_control_callbacks += 1; + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown: state.shutdown.clone(), + association_failure: state.association_failure.as_ref().map(Arc::clone), + fail_parent: false, + commit_supervision_deadline: None, + }) +} + +fn expire_internal_resolution( + state: &mut ChildLaunchData, + changed: &Condvar, +) -> Option { + let committing_drain = state.receipt == ReceiptState::Draining + && matches!(state.phase, ChildLaunchPhase::Committing); + if state.receipt != ReceiptState::AcknowledgementAdmitted && !committing_drain { + return None; + } + state.active_control_callbacks += 1; + state.abnormal = true; + let (association_failure, shutdown, fail_parent, commit_supervision_deadline) = + match state.phase { + ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) => { + state.phase = ChildLaunchPhase::Aborted(ErrorCode::Internal); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + false, + None, + ) + } + ChildLaunchPhase::Committing => { + state.receipt = ReceiptState::TimeoutPending; + complete_deadline(&mut state.acknowledgement_publication_watchdog); + ( + None, + None, + true, + Some(Instant::now() + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT), + ) + } + ChildLaunchPhase::Starting | ChildLaunchPhase::Committed => { + state.active_control_callbacks -= 1; + return None; + } + }; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown, + association_failure, + fail_parent, + commit_supervision_deadline, + }) +} + +fn arm_deadline(state: &mut DeadlineState, timeout: Duration) { + if *state == DeadlineState::Unarmed { + *state = DeadlineState::Armed(Instant::now() + timeout); + } +} + +fn complete_deadline(state: &mut DeadlineState) { + if matches!(state, DeadlineState::Unarmed | DeadlineState::Armed(_)) { + *state = DeadlineState::Disarmed; + } +} + +fn complete_acknowledgement_resolution(state: &mut ChildLaunchData) { + complete_deadline(&mut state.resolution_watchdog); + if state.receipt == ReceiptState::AcknowledgementAdmitted { + arm_deadline( + &mut state.acknowledgement_publication_watchdog, + PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, + ); + } +} + +fn take_finalization(state: &mut ChildLaunchData) -> Option { + if state.finalization_taken + || state.active_control_callbacks != 0 + || state.receipt != ReceiptState::Resolved + { + return None; + } + if !state.runner_finished { + return None; + } + state.runner_finished = false; + state.finalization_taken = true; + Some(state.abnormal) +} + +fn finish_child_process(process: &Arc, abnormal: bool) { + if abnormal { + Arc::clone(process).finish_abnormal(); + } else { + Arc::clone(process).finish(); } } +#[cfg(target_os = "linux")] +fn runner_exit_signal(status: ExitStatus) -> Option { + use std::os::unix::process::ExitStatusExt; + + status.signal() +} + +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_signal(_status: ExitStatus) -> Option { + None +} + +#[cfg(not(any(target_os = "linux", all(windows, target_arch = "x86_64"))))] +const fn runner_exit_signal(_status: ExitStatus) -> Option { + None +} + +#[cfg(target_os = "linux")] +const fn runner_signal_is_abnormal(signal: Option, broker_termination: bool) -> bool { + matches!(signal, Some(signal) if signal != libc::SIGKILL || !broker_termination) +} + +#[cfg(not(target_os = "linux"))] +const fn runner_signal_is_abnormal(_signal: Option, _broker_termination: bool) -> bool { + false +} + +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_code_is_crash(exit_code: Option) -> bool { + matches!(exit_code, Some(code) if code as u32 >= 0x8000_0000) +} + +#[cfg(not(all(windows, target_arch = "x86_64")))] +const fn runner_exit_code_is_crash(_exit_code: Option) -> bool { + false +} + +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_code_is_expected_shutdown( + exit_code: Option, + broker_termination: bool, +) -> bool { + broker_termination && matches!(exit_code, Some(1)) +} + +#[cfg(all(windows, target_arch = "x86_64"))] +const _: () = { + let access_violation = 0xc000_0005_u32 as i32; + let breakpoint = 0x8000_0003_u32 as i32; + assert!(runner_exit_code_is_crash(Some(access_violation))); + assert!(runner_exit_code_is_crash(Some(breakpoint))); + assert!(!runner_exit_code_is_expected_shutdown( + Some(access_violation), + true + )); + assert!(runner_exit_code_is_expected_shutdown(Some(1), true)); +}; + +#[cfg(not(all(windows, target_arch = "x86_64")))] +const fn runner_exit_code_is_expected_shutdown( + _exit_code: Option, + _broker_termination: bool, +) -> bool { + false +} + fn accept_runner_channel( deadline: Instant, channel_name: &'static str, @@ -604,6 +2158,12 @@ fn accept_runner_channel( mut try_accept: impl FnMut() -> IoResult, ) -> IoResult { loop { + if let Some(status) = runner_status()? { + return Err(IoError::new( + ErrorKind::BrokenPipe, + format!("runner {status} before connecting its {channel_name} channel"), + )); + } let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { return Err(IoError::new( @@ -611,12 +2171,6 @@ fn accept_runner_channel( format!("timed out waiting for runner {channel_name} channel"), )); } - if let Some(status) = runner_status()? { - return Err(IoError::new( - ErrorKind::BrokenPipe, - format!("runner {status} before connecting its {channel_name} channel"), - )); - } match try_accept() { Ok(channel) => return Ok(channel), Err(error) if error.kind() == ErrorKind::WouldBlock => {} @@ -625,3 +2179,670 @@ fn accept_runner_channel( std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); } } + +#[cfg(test)] +mod tests { + use super::{ + ChildLaunch, ChildLaunchData, ChildLaunchPhase, DeadlineState, + PROCESS_START_RECEIPT_TIMEOUT, PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT, + ProcessStartAcknowledgement, ReceiptResolution, ReceiptState, RunnerChildren, + RunnerChildrenState, RunnerConfig, ShutdownRequest, StartResultPublication, + }; + use litebox_broker_core::test_support::TestBrokerCoreBuilder; + use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; + use litebox_broker_protocol::ProcessId; + use litebox_broker_protocol::error::ErrorCode; + use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; + use litebox_broker_protocol::process::ProcessStartToken; + use std::path::PathBuf; + use std::sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, Ordering}, + mpsc, + }; + use std::time::{Duration, Instant}; + + fn launch_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let (process, _) = parent.create_child(&[]).unwrap(); + parent.finish(); + ( + broker, + Arc::new(ChildLaunch { + parent_id: ProcessId(1), + process, + state: Mutex::new(ChildLaunchData { + phase: ChildLaunchPhase::Ready { + initial_thread_id: None, + }, + publication, + shutdown: None, + association_failure: None, + shutdown_request: ShutdownRequest::None, + abnormal: false, + receipt: ReceiptState::AwaitingAcknowledgement, + resolution_watchdog: DeadlineState::Unarmed, + acknowledgement_publication_watchdog: DeadlineState::Unarmed, + start_failure_publication_watchdog: DeadlineState::Unarmed, + active_control_callbacks: 0, + pinned_initial_thread_id: None, + runner_finished: false, + finalization_taken: false, + }), + changed: Condvar::new(), + }), + ) + } + + fn launch(publication: StartResultPublication) -> Arc { + launch_with_broker(publication).1 + } + + #[test] + fn publication_captures_an_absolute_receipt_deadline() { + let launch = launch(StartResultPublication::NotStarted); + let before = Instant::now(); + + let deadline = launch.begin_start_result_publication().unwrap(); + let after = Instant::now(); + + assert!(deadline >= before + PROCESS_START_RECEIPT_TIMEOUT); + assert!(deadline <= after + PROCESS_START_RECEIPT_TIMEOUT); + } + + #[test] + fn acknowledgement_ingress_claims_receipt_before_worker_resolution() { + let (broker, launch) = launch_with_broker(StartResultPublication::Delivered); + let token = ProcessStartToken(7); + let parent_id = launch.parent_id; + let children = Arc::new(RunnerChildren { + broker, + config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerChildrenState { + launches: vec![(token, Arc::clone(&launch))], + associations: Vec::new(), + active_instances: 1, + active_watchdogs: 0, + }), + drained: Condvar::new(), + }); + + children.admit_acknowledgement(parent_id, token).unwrap(); + + assert_eq!(children.state.lock().unwrap().active_watchdogs, 2); + assert!(matches!( + launch.state.lock().unwrap().receipt, + ReceiptState::AcknowledgementAdmitted + )); + assert!(launch.expire_initial_receipt_deadline().is_none()); + assert!(matches!( + children.resolve_acknowledgement(parent_id, token), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + children.response_sent( + parent_id, + &BrokerOperation::AcknowledgeProcessStart(token), + &BrokerResult::ProcessStartAcknowledged, + ); + Arc::clone(&launch.process).finish(); + children.finish_instance(); + children.wait_for_drain(); + assert_eq!(children.state.lock().unwrap().active_watchdogs, 0); + } + + #[test] + fn reported_bootstrap_rejection_selects_normal_rollback() { + let launch = launch(StartResultPublication::NotStarted); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + + launch + .report_start_failure(ErrorCode::UnsupportedOperation) + .unwrap(); + + assert!(matches!( + launch.wait_until_ready(), + Err(ErrorCode::UnsupportedOperation) + )); + assert!( + launch + .shutdown_request() + .expected_start_failure_was_reported() + ); + assert!(!launch.state.lock().unwrap().abnormal); + launch.start_failure_response_sent(ErrorCode::UnsupportedOperation); + assert!(launch.shutdown_request().was_expected()); + assert_eq!(launch.runner_finished(false), None); + assert!(matches!( + launch.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn acknowledgement_waits_for_publication_bookkeeping() { + let launch = launch(StartResultPublication::Publishing); + launch.admit_acknowledgement().unwrap(); + let waiting = Arc::clone(&launch); + let (sender, receiver) = mpsc::sync_channel(1); + let worker = + std::thread::spawn(move || sender.send(waiting.resolve_acknowledgement()).unwrap()); + + assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); + assert!(matches!( + launch.state.lock().unwrap().resolution_watchdog, + DeadlineState::Unarmed + )); + launch.mark_start_result_delivered(); + assert!(matches!( + receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + worker.join().unwrap(); + assert!(launch.process.is_running()); + assert!(matches!( + launch.state.lock().unwrap().resolution_watchdog, + DeadlineState::Disarmed + )); + assert!(matches!( + launch + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog, + DeadlineState::Armed(_) + )); + + launch.resolve_receipt(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn acknowledgement_interrupted_by_drain_returns_peer_closed() { + let launch = launch(StartResultPublication::Publishing); + launch.admit_acknowledgement().unwrap(); + let waiting = Arc::clone(&launch); + let worker = std::thread::spawn(move || waiting.resolve_acknowledgement()); + + std::thread::sleep(Duration::from_millis(20)); + launch.begin_receipt_drain(); + + assert!(matches!(worker.join().unwrap(), Err(ErrorCode::PeerClosed))); + launch.finish_receipt_drain(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn process_ready_interrupted_by_abort_returns_the_abort_cause() { + let ready = launch(StartResultPublication::NotStarted); + ready.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + ready.abort(ErrorCode::PeerClosed, false, true); + assert!(matches!( + ready.ready_and_wait(None), + Err(ErrorCode::PeerClosed) + )); + ready.resolve_receipt(); + Arc::clone(&ready.process).finish(); + } + + #[test] + fn failure_report_interrupted_by_abort_returns_the_abort_cause() { + let failed = launch(StartResultPublication::NotStarted); + failed.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + failed.abort(ErrorCode::PeerClosed, false, true); + assert!(matches!( + failed.report_start_failure(ErrorCode::UnsupportedOperation), + Err(ErrorCode::PeerClosed) + )); + failed.resolve_receipt(); + Arc::clone(&failed.process).finish(); + } + + #[test] + fn delivery_arms_the_resolution_watchdog_before_the_worker_resumes() { + let launch = launch(StartResultPublication::Publishing); + launch.admit_acknowledgement().unwrap(); + + assert!(matches!( + launch.state.lock().unwrap().resolution_watchdog, + DeadlineState::Unarmed + )); + launch.mark_start_result_delivered(); + assert!(matches!( + launch.state.lock().unwrap().resolution_watchdog, + DeadlineState::Armed(_) + )); + + launch.begin_receipt_drain(); + launch.finish_receipt_drain(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn acknowledgement_publication_timeout_retains_receipt_for_drain() { + let launch = launch(StartResultPublication::Delivered); + launch.admit_acknowledgement().unwrap(); + assert!(matches!( + launch.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + launch + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = launch + .wait_for_acknowledgement_publication_timeout() + .unwrap(); + assert!(expiration.fail_parent); + let state = launch.state.lock().unwrap(); + assert!(matches!(state.receipt, ReceiptState::TimeoutPending)); + assert!(matches!( + state.acknowledgement_publication_watchdog, + DeadlineState::Fired + )); + assert!(!state.abnormal); + drop(state); + + assert_eq!(launch.complete_timeout_callback(), None); + launch.begin_receipt_drain(); + launch.finish_receipt_drain(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn start_failure_publication_timeout_terminates_the_child() { + let launch = launch(StartResultPublication::NotStarted); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + launch + .report_start_failure(ErrorCode::UnsupportedOperation) + .unwrap(); + launch + .state + .lock() + .unwrap() + .start_failure_publication_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = launch.wait_for_start_failure_publication_timeout().unwrap(); + + assert!(!expiration.fail_parent); + assert!(launch.shutdown_request().was_expected()); + assert!(matches!( + launch + .state + .lock() + .unwrap() + .start_failure_publication_watchdog, + DeadlineState::Fired + )); + assert_eq!(launch.complete_timeout_callback(), None); + assert_eq!(launch.runner_finished(false), None); + assert!(matches!( + launch.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn precommit_resolution_timeout_returns_typed_internal_failure() { + let launch = launch(StartResultPublication::Delivered); + launch.admit_acknowledgement().unwrap(); + launch.state.lock().unwrap().resolution_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = launch.wait_for_internal_resolution_timeout().unwrap(); + assert!(!expiration.fail_parent); + let state = launch.state.lock().unwrap(); + assert!(matches!( + state.phase, + ChildLaunchPhase::Aborted(ErrorCode::Internal) + )); + assert!(matches!( + state.receipt, + ReceiptState::AcknowledgementAdmitted + )); + assert!(matches!( + state.acknowledgement_publication_watchdog, + DeadlineState::Unarmed + )); + assert!(state.abnormal); + drop(state); + assert_eq!(launch.complete_timeout_callback(), None); + + assert!(matches!( + launch.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Failed(ErrorCode::Internal)) + )); + assert!(matches!( + launch + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog, + DeadlineState::Armed(_) + )); + launch.resolve_receipt(); + Arc::clone(&launch.process).finish_abnormal(); + } + + #[test] + fn child_abort_fails_an_installed_active_association() { + let launch = launch(StartResultPublication::NotStarted); + let failed = Arc::new(AtomicBool::new(false)); + let recorded = Arc::clone(&failed); + launch.install_association_failure(Arc::new(move || { + recorded.store(true, Ordering::Release); + })); + + launch.abort(ErrorCode::PeerClosed, false, true); + + assert!(failed.load(Ordering::Acquire)); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn supervisor_wait_observes_commit_resolution() { + let launch = launch(StartResultPublication::Delivered); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Committing; + let waiting = Arc::clone(&launch); + let worker = std::thread::spawn(move || { + waiting.wait_for_commit_resolution(Instant::now() + Duration::from_secs(1)) + }); + + std::thread::sleep(Duration::from_millis(20)); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Committed; + launch.changed.notify_all(); + + assert!(worker.join().unwrap()); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn receipt_drain_keeps_commit_supervision_armed() { + let launch = launch(StartResultPublication::Delivered); + launch.admit_acknowledgement().unwrap(); + { + let mut state = launch.state.lock().unwrap(); + state.phase = ChildLaunchPhase::Committing; + state.resolution_watchdog = DeadlineState::Armed(Instant::now()); + } + launch.begin_receipt_drain(); + assert!(matches!( + launch.state.lock().unwrap().resolution_watchdog, + DeadlineState::Armed(_) + )); + + let before = Instant::now(); + let expiration = launch.wait_for_internal_resolution_timeout().unwrap(); + let after = Instant::now(); + let deadline = expiration.commit_supervision_deadline.unwrap(); + assert!(deadline >= before + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); + assert!(deadline <= after + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); + { + let mut state = launch.state.lock().unwrap(); + state.phase = ChildLaunchPhase::Aborted(ErrorCode::Internal); + } + launch.changed.notify_all(); + assert_eq!(launch.complete_timeout_callback(), None); + launch.finish_receipt_drain(); + Arc::clone(&launch.process).finish_abnormal(); + } + + #[test] + fn published_abort_returns_typed_start_failure() { + let launch = launch(StartResultPublication::Publishing); + launch.abort(ErrorCode::PeerClosed, false, false); + launch.mark_start_result_delivered(); + launch.admit_acknowledgement().unwrap(); + + assert!(matches!( + launch.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Failed(ErrorCode::PeerClosed)) + )); + launch.resolve_receipt(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn acknowledgement_before_publication_is_protocol_violation() { + let launch = launch(StartResultPublication::NotStarted); + + assert!(matches!( + launch.admit_acknowledgement(), + Err(ErrorCode::ProtocolState) + )); + launch.resolve_receipt(); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn published_receipt_defers_process_finalization() { + let launch = launch(StartResultPublication::Delivered); + launch.abort(ErrorCode::PeerClosed, false, false); + + assert_eq!(launch.runner_finished(false), None); + assert!(matches!( + launch.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn receipt_resolution_releases_the_initial_thread_pin() { + let launch = launch(StartResultPublication::Delivered); + let thread_id = launch.process.create_thread().unwrap(); + launch.process.pin_startup_thread(thread_id).unwrap(); + launch.state.lock().unwrap().pinned_initial_thread_id = Some(thread_id); + + launch.resolve_receipt(); + + assert_eq!(launch.process.exit_thread(thread_id), Ok(())); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn process_ready_waits_until_the_initial_thread_pin_is_released() { + let launch = launch(StartResultPublication::Delivered); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + let thread_id = launch.process.create_thread().unwrap(); + let waiting = Arc::clone(&launch); + let (sender, receiver) = mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + sender + .send(waiting.ready_and_wait(Some(thread_id))) + .unwrap(); + }); + + let mut state = launch.state.lock().unwrap(); + while !matches!(state.phase, ChildLaunchPhase::Ready { .. }) { + state = launch.changed.wait(state).unwrap(); + } + state.phase = ChildLaunchPhase::Committed; + launch.changed.notify_all(); + drop(state); + assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); + + launch.resolve_receipt(); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(()) + ); + worker.join().unwrap(); + assert_eq!(launch.process.exit_thread(thread_id), Ok(())); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn acknowledgement_admission_preserves_the_publication_deadline() { + let launch = launch(StartResultPublication::Publishing); + launch.admit_acknowledgement().unwrap(); + + let _expiration = launch.expire_initial_receipt_deadline().unwrap(); + let state = launch.state.lock().unwrap(); + assert!(matches!( + state.phase, + ChildLaunchPhase::Aborted(ErrorCode::Internal) + )); + assert!(state.abnormal); + drop(state); + assert_eq!(launch.complete_timeout_callback(), None); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn receipt_timeout_defers_finalization_until_association_drain() { + let launch = launch(StartResultPublication::Delivered); + + assert_eq!(launch.runner_finished(false), None); + let _expiration = launch.expire_initial_receipt_deadline().unwrap(); + assert!(matches!( + launch.resolve_receipt(), + ReceiptResolution::DeferredToDrain + )); + assert_eq!(launch.complete_timeout_callback(), None); + launch.begin_receipt_drain(); + assert_eq!(launch.finish_receipt_drain(), Some(false)); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn unpublished_launch_waits_for_receipt_drain_before_finalization() { + let launch = launch(StartResultPublication::NotStarted); + + launch.begin_receipt_drain(); + assert_eq!(launch.runner_finished(false), None); + assert_eq!(launch.finish_receipt_drain(), Some(false)); + Arc::clone(&launch.process).finish(); + } + + #[test] + fn deferred_finalization_uses_the_latest_abnormal_disposition() { + let launch = launch(StartResultPublication::Publishing); + launch.admit_acknowledgement().unwrap(); + + assert_eq!(launch.runner_finished(false), None); + let _expiration = launch.expire_initial_receipt_deadline().unwrap(); + assert_eq!(launch.complete_timeout_callback(), None); + launch.begin_receipt_drain(); + assert_eq!(launch.finish_receipt_drain(), Some(true)); + Arc::clone(&launch.process).finish_abnormal(); + } + + #[test] + fn acknowledgement_send_does_not_steal_a_timed_out_receipt_from_drain() { + let (broker, launch) = launch_with_broker(StartResultPublication::Delivered); + let token = ProcessStartToken(7); + let parent_id = launch.parent_id; + let children = RunnerChildren { + broker, + config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerChildrenState { + launches: vec![(token, Arc::clone(&launch))], + associations: Vec::new(), + active_instances: 1, + active_watchdogs: 0, + }), + drained: Condvar::new(), + }; + + assert_eq!(launch.runner_finished(false), None); + let _expiration = launch.expire_initial_receipt_deadline().unwrap(); + children.response_sent( + parent_id, + &BrokerOperation::AcknowledgeProcessStart(token), + &BrokerResult::ProcessStartAcknowledged, + ); + + assert!(children.find_launch(token).is_some()); + assert_eq!(launch.complete_timeout_callback(), None); + let draining = children.association_ending(parent_id); + assert_eq!(draining.len(), 1); + children.association_ended(parent_id, draining); + assert!(children.find_launch(token).is_none()); + } + + #[cfg(target_os = "linux")] + #[test] + fn shutdown_can_terminate_while_the_launch_owner_waits() { + use super::{PlatformRunnerShutdown, RunnerShutdown, RunnerShutdownState}; + use std::process::Command; + + let child = Command::new("sh") + .args(["-c", "exec sleep 30"]) + .spawn() + .unwrap(); + let shutdown = RunnerShutdown { + platform: PlatformRunnerShutdown::new(&child), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }; + let child = Arc::new(Mutex::new(child)); + let waiting = Arc::clone(&child); + let (finished, completion) = mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || { + let status = waiting.lock().unwrap().wait().unwrap(); + finished.send(status).unwrap(); + }); + + shutdown.shutdown(); + assert!(shutdown.termination_was_dispatched()); + assert!( + !completion + .recv_timeout(Duration::from_secs(1)) + .unwrap() + .success() + ); + waiter.join().unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn shutdown_after_observed_exit_records_no_termination() { + use super::{PlatformRunnerShutdown, RunnerShutdown, RunnerShutdownState}; + use std::process::Command; + + let mut child = Command::new("sh").args(["-c", "exit 1"]).spawn().unwrap(); + let shutdown = RunnerShutdown { + platform: PlatformRunnerShutdown::new(&child), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }; + assert!(shutdown.wait_for_exit(Duration::from_secs(1)).unwrap()); + + shutdown.shutdown(); + shutdown.retire(); + + assert!(!shutdown.termination_was_dispatched()); + assert!(!child.wait().unwrap().success()); + } + + #[cfg(target_os = "linux")] + #[test] + fn signal_termination_is_distinct_from_an_ordinary_nonzero_exit() { + use super::{runner_exit_signal, runner_signal_is_abnormal}; + use std::process::Command; + + let signaled = Command::new("sh") + .args(["-c", "kill -SEGV $$"]) + .status() + .unwrap(); + let nonzero = Command::new("sh").args(["-c", "exit 7"]).status().unwrap(); + + let signal = runner_exit_signal(signaled); + assert!(signal.is_some()); + assert!(runner_exit_signal(nonzero).is_none()); + assert!(runner_signal_is_abnormal(signal, true)); + assert!(!runner_signal_is_abnormal(Some(libc::SIGKILL), true)); + } +} diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index 7fcdced7f4..8c0a8d8938 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -6,7 +6,7 @@ use std::io::Result as IoResult; use std::os::unix::net::UnixListener; use std::path::PathBuf; use std::process::Child; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Instant; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; @@ -16,6 +16,33 @@ use litebox_broker_transport_linux_userland::unix_socket::{ }; use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use crate::runtime::{AssociationFailureCause, AssociationRunResult}; + +pub(super) struct PlatformRunnerShutdown { + process_id: u32, +} + +fn runner_has_exited(runner: &Arc>) -> IoResult { + non_reaping_runner_has_exited(runner.lock().expect("runner process mutex poisoned").id()) +} + +impl PlatformRunnerShutdown { + pub(super) fn new(runner: &Child) -> Self { + Self { + process_id: runner.id(), + } + } + + pub(super) fn shutdown(&self) -> bool { + // SAFETY: the unreaped `Child` keeps this PID allocated to the runner + // until the launch owner closes the endpoint and waits for it. + unsafe { libc::kill(self.process_id.cast_signed(), libc::SIGKILL) == 0 } + } + + pub(super) fn has_exited(&self) -> IoResult { + non_reaping_runner_has_exited(self.process_id) + } +} pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, @@ -44,9 +71,9 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, - runner: &mut Child, + runner: &Arc>, children: Arc, - ) -> IoResult<()> { + ) -> AssociationRunResult { serve_runner_process( self.listener .as_ref() @@ -58,10 +85,10 @@ impl PlatformRunnerEndpoint { pub(super) fn serve_child( &mut self, - runner: &mut Child, + runner: &Arc>, child: ChildRunner, children: Arc, - ) -> IoResult<()> { + ) -> AssociationRunResult { serve_child_runner_process( self.listener .as_ref() @@ -80,10 +107,26 @@ impl PlatformRunnerEndpoint { fn serve_runner_process( control_listener: &UnixListener, - runner: &mut Child, + runner: &Arc>, children: Arc, -) -> IoResult<()> { - let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; +) -> AssociationRunResult { + let (control_channel, setup_deadline) = match accept_control_channel(control_listener, runner) { + Ok(connection) => connection, + Err(error) => { + let failure_cause = if runner_has_exited(runner).unwrap_or(false) { + AssociationFailureCause::RunnerExit + } else { + AssociationFailureCause::Other + }; + return AssociationRunResult { + result: Err(error), + process: None, + panicked: false, + abnormal: failure_cause == AssociationFailureCause::Other, + failure_cause, + }; + } + }; crate::runtime::serve_runner_association( None, control_channel, @@ -101,12 +144,36 @@ fn serve_runner_process( fn serve_child_runner_process( control_listener: &UnixListener, - runner: &mut Child, + runner: &Arc>, child: ChildRunner, children: Arc, -) -> IoResult<()> { - let (control_channel, setup_deadline) = accept_control_channel(control_listener, runner)?; - crate::runtime::serve_runner_association( +) -> AssociationRunResult { + let (control_channel, setup_deadline) = match accept_control_channel(control_listener, runner) { + Ok(connection) => connection, + Err(error) => { + let failure_cause = if runner_has_exited(runner).unwrap_or(false) { + AssociationFailureCause::RunnerExit + } else if matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + ) { + AssociationFailureCause::PeerClosed + } else { + AssociationFailureCause::Other + }; + return AssociationRunResult { + result: Err(error), + process: None, + panicked: false, + abnormal: failure_cause == AssociationFailureCause::Other, + failure_cause, + }; + } + }; + let mut result = crate::runtime::serve_runner_association( Some(child), control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), @@ -118,27 +185,58 @@ fn serve_child_runner_process( }, UnixStreamHostSetupChannel::into_active, children, - ) + ); + if result.failure_cause == AssociationFailureCause::Other + && result + .result + .as_ref() + .is_err_and(|error| error.kind() == std::io::ErrorKind::BrokenPipe) + && runner_has_exited(runner).unwrap_or(false) + { + result.failure_cause = AssociationFailureCause::RunnerExit; + } + result } fn accept_control_channel( control_listener: &UnixListener, - runner: &mut Child, + runner: &Arc>, ) -> IoResult<(UnixStreamHostSetupChannel, Instant)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; + let runner_id = runner.lock().expect("runner process mutex poisoned").id(); let control_stream = accept_runner_channel( setup_deadline, "control", || { - runner - .try_wait() - .map(|status| status.map(|status| format!("exited with {status}"))) + non_reaping_runner_has_exited(runner_id) + .map(|exited| exited.then(|| "exited".to_owned())) }, || control_listener.accept().map(|(stream, _)| stream), )?; - validate_peer_process(&control_stream, runner.id())?; + validate_peer_process(&control_stream, runner_id)?; Ok(( UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline), setup_deadline, )) } + +fn non_reaping_runner_has_exited(runner_id: u32) -> IoResult { + let mut info = std::mem::MaybeUninit::::zeroed(); + // SAFETY: `info` points to writable `siginfo_t` storage, and `waitid` is + // restricted to observing this known child without consuming its status. + let result = unsafe { + libc::waitid( + libc::P_PID, + runner_id, + info.as_mut_ptr(), + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: successful `waitid` initialized `info`; `si_pid == 0` denotes + // that the nonblocking observation found no waitable child state. + let exited = unsafe { info.assume_init().si_pid() != 0 }; + Ok(exited) +} diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 14dee40d5b..eda067d076 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -5,7 +5,7 @@ use std::ffi::OsString; use std::io::Result as IoResult; use std::os::windows::io::AsRawHandle; use std::process::Child; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Instant; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; @@ -13,8 +13,42 @@ use litebox_broker_transport_windows_userland::named_pipe::{ WindowsNamedPipeHostSetupChannel, WindowsNamedPipeListener, validate_client_process, }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; +use windows_sys::Win32::Foundation::{HANDLE, WAIT_FAILED, WAIT_OBJECT_0, WAIT_TIMEOUT}; +use windows_sys::Win32::System::Threading::{TerminateProcess, WaitForSingleObject}; use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use crate::runtime::{AssociationFailureCause, AssociationRunResult}; + +pub(super) struct PlatformRunnerShutdown { + process_handle: isize, +} + +impl PlatformRunnerShutdown { + pub(super) fn new(runner: &Child) -> Self { + Self { + process_handle: runner.as_raw_handle() as isize, + } + } + + pub(super) fn shutdown(&self) -> bool { + // SAFETY: the launch owner retains the `Child` and its process handle + // until all shutdown callers finish and the runner has been waited. + unsafe { TerminateProcess(self.process_handle as HANDLE, 1) != 0 } + } + + pub(super) fn has_exited(&self) -> IoResult { + // SAFETY: the launch owner retains the `Child` process handle until + // shutdown is retired and the runner is waited. + match unsafe { WaitForSingleObject(self.process_handle as HANDLE, 0) } { + WAIT_OBJECT_0 => Ok(true), + WAIT_TIMEOUT => Ok(false), + WAIT_FAILED => Err(std::io::Error::last_os_error()), + _ => Err(std::io::Error::other( + "waiting for the runner process returned an unknown status", + )), + } + } +} pub(super) struct PlatformRunnerEndpoint { pipe_name: OsString, @@ -37,9 +71,9 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, - runner: &mut Child, + runner: &Arc>, children: Arc, - ) -> IoResult<()> { + ) -> AssociationRunResult { serve_runner_process( self.listener .as_mut() @@ -51,10 +85,10 @@ impl PlatformRunnerEndpoint { pub(super) fn serve_child( &mut self, - runner: &mut Child, + runner: &Arc>, child: ChildRunner, children: Arc, - ) -> IoResult<()> { + ) -> AssociationRunResult { serve_child_runner_process( self.listener .as_mut() @@ -72,11 +106,31 @@ impl PlatformRunnerEndpoint { fn serve_runner_process( control_listener: &mut WindowsNamedPipeListener, - runner: &mut Child, + runner: &Arc>, children: Arc, -) -> IoResult<()> { - let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; - let runner_process = runner.as_raw_handle(); +) -> AssociationRunResult { + let (control_channel, _setup_deadline) = match accept_control_channel(control_listener, runner) + { + Ok(connection) => connection, + Err(error) => { + let failure_cause = if non_reaping_runner_has_exited(runner).unwrap_or(false) { + AssociationFailureCause::RunnerExit + } else { + AssociationFailureCause::Other + }; + return AssociationRunResult { + result: Err(error), + process: None, + panicked: false, + abnormal: failure_cause == AssociationFailureCause::Other, + failure_cause, + }; + } + }; + let runner_process = runner + .lock() + .expect("runner process mutex poisoned") + .as_raw_handle(); crate::runtime::serve_runner_association( None, control_channel, @@ -93,13 +147,41 @@ fn serve_runner_process( fn serve_child_runner_process( control_listener: &mut WindowsNamedPipeListener, - runner: &mut Child, + runner: &Arc>, child: ChildRunner, children: Arc, -) -> IoResult<()> { - let (control_channel, _setup_deadline) = accept_control_channel(control_listener, runner)?; - let runner_process = runner.as_raw_handle(); - crate::runtime::serve_runner_association( +) -> AssociationRunResult { + let (control_channel, _setup_deadline) = match accept_control_channel(control_listener, runner) + { + Ok(connection) => connection, + Err(error) => { + let failure_cause = if non_reaping_runner_has_exited(runner).unwrap_or(false) { + AssociationFailureCause::RunnerExit + } else if matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + ) { + AssociationFailureCause::PeerClosed + } else { + AssociationFailureCause::Other + }; + return AssociationRunResult { + result: Err(error), + process: None, + panicked: false, + abnormal: failure_cause == AssociationFailureCause::Other, + failure_cause, + }; + } + }; + let runner_process = runner + .lock() + .expect("runner process mutex poisoned") + .as_raw_handle(); + let mut result = crate::runtime::serve_runner_association( Some(child), control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), @@ -110,31 +192,42 @@ fn serve_child_runner_process( }, WindowsNamedPipeHostSetupChannel::into_active, children, - ) + ); + if result.failure_cause == AssociationFailureCause::Other + && result + .result + .as_ref() + .is_err_and(|error| error.kind() == std::io::ErrorKind::BrokenPipe) + && non_reaping_runner_has_exited(runner).unwrap_or(false) + { + result.failure_cause = AssociationFailureCause::RunnerExit; + } + result } fn accept_control_channel( control_listener: &mut WindowsNamedPipeListener, - runner: &mut Child, + runner: &Arc>, ) -> IoResult<(WindowsNamedPipeHostSetupChannel, Instant)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; let control_stream = accept_runner_channel( setup_deadline, "control", - || { - runner - .try_wait() - .map(|status| status.map(|status| format!("exited with {status}"))) - }, + || non_reaping_runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), || control_listener.try_accept(), )?; - validate_client_process(&control_stream, runner.id())?; + let runner_id = runner.lock().expect("runner process mutex poisoned").id(); + validate_client_process(&control_stream, runner_id)?; Ok(( WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline), setup_deadline, )) } +fn non_reaping_runner_has_exited(runner: &Arc>) -> IoResult { + PlatformRunnerShutdown::new(&runner.lock().expect("runner process mutex poisoned")).has_exited() +} + fn unique_control_pipe_name() -> OsString { let process_id = std::process::id(); let nonce = std::time::SystemTime::now() diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index b6123d0de6..d549ea6eb0 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -25,13 +25,14 @@ use std::sync::{ }; use std::time::{Duration, Instant}; -use litebox_broker_core::BrokerCore; +use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{ BrokerHostAssociation, BrokerHostError, ConnectionTermination, ProcessStartupData, - setup_connection, + setup_connection_with_process, }; +use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::BrokerRequest; +use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest}; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, @@ -44,9 +45,27 @@ use crate::readiness::ReadinessPublisherRuntime; use crate::runner::{ChildRunner, RunnerChildren}; const REQUEST_QUEUE_CAPACITY: usize = 64; +pub(crate) const LIFECYCLE_CONTROL_WORKER_COUNT: usize = crate::WORKER_COUNT; +pub(crate) const LIFECYCLE_CONTROL_QUEUE_CAPACITY: usize = crate::WORKER_COUNT; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); const REQUEST_QUEUE_STALL_TIMEOUT: Duration = Duration::from_secs(5); +pub(crate) struct AssociationRunResult { + pub(crate) result: IoResult<()>, + pub(crate) process: Option>, + pub(crate) panicked: bool, + pub(crate) abnormal: bool, + pub(crate) failure_cause: AssociationFailureCause, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum AssociationFailureCause { + None, + PeerClosed, + RunnerExit, + Other, +} + /// Serves one broker association from setup through teardown. /// /// `control_channel` must already be configured with whatever deadline and @@ -84,7 +103,7 @@ where RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { serve_association_with_children( broker, @@ -95,6 +114,9 @@ where activate, None, None, + None, + None, + None, ) } @@ -116,17 +138,21 @@ pub(crate) fn serve_runner_association< ControlRing, ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, children: Arc, -) -> IoResult<()> +) -> AssociationRunResult where Memory: ControlRingMemory, SetupChannel: HostSetupChannel, RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { + let panicked = AtomicBool::new(false); + let abnormal = AtomicBool::new(false); + let process = Mutex::new(None); + let defer_process_finish = child.is_none(); let broker = children.broker.clone(); - serve_association_with_children( + let result = serve_association_with_children( &broker, control_channel, create_shared_memory, @@ -135,7 +161,33 @@ where activate, Some(children), child, - ) + Some(&panicked), + Some(&abnormal), + defer_process_finish.then_some(&process), + ); + let failure_cause = match result.as_ref() { + Ok(()) => AssociationFailureCause::None, + Err(error) + if matches!( + error.kind(), + ErrorKind::BrokenPipe + | ErrorKind::UnexpectedEof + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + ) => + { + AssociationFailureCause::PeerClosed + } + Err(_) => AssociationFailureCause::Other, + }; + AssociationRunResult { + result, + process: process.into_inner().expect("runner process mutex poisoned"), + panicked: panicked.load(Ordering::Acquire), + abnormal: abnormal.load(Ordering::Acquire) + || failure_cause == AssociationFailureCause::Other, + failure_cause, + } } #[allow(clippy::too_many_arguments)] @@ -158,6 +210,9 @@ fn serve_association_with_children< ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, children: Option>, child: Option, + panicked_out: Option<&AtomicBool>, + abnormal_out: Option<&AtomicBool>, + process_out: Option<&Mutex>>>, ) -> IoResult<()> where Memory: ControlRingMemory, @@ -165,12 +220,13 @@ where RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { let is_child = child.is_some(); let (process, startup) = match child { Some(ChildRunner { process, + launch: _, inherited_objects, format, version, @@ -193,13 +249,20 @@ where let control_ring = ControlRing::new(control_memory) .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; let readiness = Arc::new(ReadinessPublisherRuntime::new()); - let association = match setup_connection( + let association = match setup_connection_with_process( broker, process, startup, &mut control_channel, &shared_buffers, readiness.clone(), + |process| { + let Some(process_out) = process_out else { + return false; + }; + *process_out.lock().expect("runner process mutex poisoned") = Some(Arc::clone(process)); + true + }, |channel| send_shared_memory(channel, shared_buffers.memory(), control_ring.memory()), ) .map_err(map_host_error)? @@ -228,7 +291,7 @@ where match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { - if !is_child { + if !is_child && process_out.is_none() { association.finish(); } return Err(error); @@ -242,6 +305,9 @@ where notification_channel, shutdown, children, + !is_child && process_out.is_none(), + panicked_out, + abnormal_out, ) } @@ -272,6 +338,9 @@ fn map_host_error(error: BrokerHostError) -> IoError { BrokerHostError::SharedBufferLayoutMismatch => { IoError::new(ErrorKind::InvalidData, description) } + BrokerHostError::AssociationFailed => { + IoError::new(ErrorKind::ConnectionAborted, description) + } _ => IoError::other(description), } } @@ -285,6 +354,7 @@ struct HostAssociationFailureCoordinator { failed: AtomicBool, /// Whether an association worker panicked, requiring teardown without ID reuse. panicked: AtomicBool, + abnormal: AtomicBool, error: Mutex>, shutdown: Shutdown, } @@ -296,6 +366,7 @@ impl> Self { failed: AtomicBool::new(false), panicked: AtomicBool::new(false), + abnormal: AtomicBool::new(false), error: Mutex::new(None), shutdown, } @@ -306,6 +377,15 @@ impl> } fn report(&self, error: IoError) { + if !matches!( + error.kind(), + ErrorKind::BrokenPipe + | ErrorKind::UnexpectedEof + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + ) { + self.abnormal.store(true, Ordering::Release); + } if self.failed.swap(true, Ordering::AcqRel) { return; } @@ -325,6 +405,10 @@ impl> self.panicked.load(Ordering::Acquire) } + fn abnormal(&self) -> bool { + self.abnormal.load(Ordering::Acquire) + } + /// Ends the association transport without recording a failure. /// /// Teardown uses this to release blocked endpoints without turning a @@ -417,7 +501,7 @@ where RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { dispatch_requests_with_children( association, @@ -427,9 +511,13 @@ where notification_channel, shutdown, None, + true, + None, + None, ) } +#[allow(clippy::too_many_arguments)] fn dispatch_requests_with_children< Memory, RequestSource, @@ -444,20 +532,43 @@ fn dispatch_requests_with_children< mut notification_channel: NotificationChannel, shutdown: Shutdown, children: Option>, + finish_process: bool, + panicked_out: Option<&AtomicBool>, + abnormal_out: Option<&AtomicBool>, ) -> IoResult<()> where Memory: SharedMemory, RequestSource: HostRequestSource, ResponseSink: HostResponseSink + Clone + Send, NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync, + Shutdown: HostAssociationShutdown + Send + Sync + 'static, { let association = Arc::new(association); + let process_id = association.process_id(); let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(shutdown)); + if let Some(children) = &children { + let association_failure = Arc::clone(&failure_coordinator); + let association_failure: crate::runner::AssociationFailure = Arc::new(move || { + association_failure.report(IoError::new( + ErrorKind::ConnectionAborted, + "process association terminated by lifecycle control", + )); + }); + if let Err(error) = children + .register_association(association.process_id(), Arc::clone(&association_failure)) + { + failure_coordinator.report(error); + } else { + children + .install_child_association_failure(association.process_id(), association_failure); + } + } let (request_sender, request_receiver) = sync_channel(REQUEST_QUEUE_CAPACITY); let request_receiver = Arc::new(Mutex::new(request_receiver)); + let (control_sender, control_receiver) = sync_channel(LIFECYCLE_CONTROL_QUEUE_CAPACITY); + let control_receiver = Arc::new(Mutex::new(control_receiver)); - std::thread::scope(|scope| { + let draining_launches = std::thread::scope(|scope| { let publisher_readiness = Arc::clone(&readiness); let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() @@ -494,7 +605,31 @@ where association: &association, }; - let mut workers = Vec::with_capacity(crate::WORKER_COUNT); + let mut workers = Vec::with_capacity(crate::WORKER_COUNT + LIFECYCLE_CONTROL_WORKER_COUNT); + for worker_id in 0..LIFECYCLE_CONTROL_WORKER_COUNT { + let association = Arc::clone(&association); + let control_receiver = Arc::clone(&control_receiver); + let response_sink = response_sink.clone(); + let worker_failure_coordinator = Arc::clone(&failure_coordinator); + let worker_children = children.clone(); + match std::thread::Builder::new() + .name(format!("litebox-broker-lifecycle-worker-{worker_id}")) + .spawn_scoped(scope, move || { + run_worker( + &association, + &control_receiver, + &response_sink, + &worker_failure_coordinator, + worker_children.as_ref(), + ); + }) { + Ok(worker) => workers.push(worker), + Err(error) => { + failure_coordinator.report(error); + break; + } + } + } for worker_id in 0..crate::WORKER_COUNT { let association = Arc::clone(&association); let request_receiver = Arc::clone(&request_receiver); @@ -520,11 +655,18 @@ where } } - read_requests(&mut request_source, request_sender, &failure_coordinator); + read_requests( + &mut request_source, + request_sender, + control_sender, + &failure_coordinator, + children.as_ref(), + process_id, + ); drop(cancellation); - if let Some(children) = &children { - children.association_ended(association.process_id()); - } + let draining_launches = children + .as_ref() + .map_or_else(Vec::new, |children| children.association_ending(process_id)); for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -544,8 +686,13 @@ where { failure_coordinator.report_panic(IoError::other("broker readiness publisher panicked")); } + draining_launches }); + if let Some(children) = &children { + children.association_ended(process_id, draining_launches); + } + let result = match failure_coordinator.take_error() { Some(error) => Err(error), None => Ok(()), @@ -553,7 +700,14 @@ where let Ok(association) = Arc::try_unwrap(association) else { panic!("all broker association workers must be joined before teardown"); }; - if failure_coordinator.panicked() { + let panicked = failure_coordinator.panicked(); + if let Some(panicked_out) = panicked_out { + panicked_out.store(panicked, Ordering::Release); + } + if let Some(abnormal_out) = abnormal_out { + abnormal_out.store(failure_coordinator.abnormal(), Ordering::Release); + } + if panicked || !finish_process { drop(association); } else { association.finish(); @@ -564,7 +718,10 @@ where fn read_requests( request_source: &mut RequestSource, request_sender: SyncSender, + control_sender: SyncSender, failure_coordinator: &HostAssociationFailureCoordinator, + children: Option<&Arc>, + process_id: ProcessId, ) where RequestSource: HostRequestSource, Shutdown: HostAssociationShutdown, @@ -575,8 +732,24 @@ fn read_requests( } match request_source.recv_request() { Ok(HostReceive::Message(request)) => { + if let BrokerOperation::AcknowledgeProcessStart(token) = &request.operation + && let Some(children) = children + && let Err(error) = children.admit_acknowledgement(process_id, *token) + { + failure_coordinator.report(map_host_error(BrokerHostError::Broker(error))); + break; + } + let sender = if matches!( + &request.operation, + BrokerOperation::AcknowledgeProcessStart(_) + | BrokerOperation::ReportProcessStartFailure(_) + ) { + &control_sender + } else { + &request_sender + }; if !enqueue_request( - &request_sender, + sender, request, failure_coordinator, REQUEST_QUEUE_STALL_TIMEOUT, @@ -670,14 +843,14 @@ fn run_worker( }) }, |response| response_sink.send_response(response), - |result| { + |operation, result| { if let Some(children) = children { - children.response_sent(process_id, result); + children.response_sent(process_id, operation, result); } }, ) })) { - Ok(Ok(())) => {} + Ok(Ok(()) | Err(BrokerHostError::AssociationFailed)) => {} Ok(Err(error)) => failure_coordinator.report(map_host_error(error)), Err(_) => { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -1113,6 +1286,24 @@ mod tests { assert_eq!(error.to_string(), "first failure"); } + #[test] + fn later_protocol_failure_upgrades_abnormal_disposition() { + let association = live_association(); + let failure_coordinator = HostAssociationFailureCoordinator::new(association.shutdown); + + failure_coordinator.report(IoError::new( + ErrorKind::ConnectionAborted, + "lifecycle shutdown", + )); + failure_coordinator.report(IoError::new(ErrorKind::InvalidData, "protocol failure")); + + assert!(failure_coordinator.abnormal()); + assert_eq!( + failure_coordinator.take_error().unwrap().kind(), + ErrorKind::ConnectionAborted + ); + } + #[test] fn a_stalled_request_queue_fails_after_its_deadline() { let request = |request_id| BrokerRequest { diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index f3b1b4e8b3..34a93a05d7 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -262,7 +262,7 @@ fn run_fake_runner(args: &[OsString]) { bootstrap, inherited_objects, ), - Err(BrokerLocalError::Broker(ErrorCode::PeerClosed)) + Err(BrokerLocalError::Broker(ErrorCode::UnsupportedOperation)) )); let started = local .request_process_start( @@ -394,6 +394,9 @@ fn run_fake_child(control_socket_path: &Path) { .unwrap(); let bootstrap = bootstrap.expect("child negotiation must include startup data"); if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { + local + .report_process_start_failure(ErrorCode::UnsupportedOperation) + .unwrap(); return; } assert_eq!(bootstrap.format, TEST_BOOTSTRAP_FORMAT); diff --git a/litebox_runner_linux_userland/Cargo.toml b/litebox_runner_linux_userland/Cargo.toml index 9f3696370e..8187495238 100644 --- a/litebox_runner_linux_userland/Cargo.toml +++ b/litebox_runner_linux_userland/Cargo.toml @@ -9,6 +9,7 @@ clap = { version = "4.5.33", features = ["derive"] } libc = { version = "0.2.169", default-features = false } litebox = { version = "0.1.0", path = "../litebox" } litebox_broker_local_userland = { version = "0.1.0", path = "../litebox_broker_local_userland" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_transport = { version = "0.1.0", path = "../litebox_broker_transport" } litebox_broker_transport_linux_userland = { version = "0.1.0", path = "../litebox_broker_transport_linux_userland" } litebox_common_linux = { version = "0.1.0", path = "../litebox_common_linux" } @@ -24,7 +25,6 @@ glob = "0.3" litebox_broker_core = { version = "0.1.0", path = "../litebox_broker_core", features = ["test-support"] } litebox_broker_host = { version = "0.1.0", path = "../litebox_broker_host" } litebox_broker_platform_linux_userland = { version = "0.1.0", path = "../litebox_broker_platform_linux_userland" } -litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_broker_userland = { version = "0.1.0", path = "../litebox_broker_userland" } [features] diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index ab4b39bb8f..92c7a8d58e 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -3,6 +3,7 @@ use anyhow::{Context as _, Result, anyhow}; use clap::Parser; +use litebox_broker_protocol::error::ErrorCode; use litebox_platform_linux_userland::LinuxUserland as Platform; use std::path::PathBuf; @@ -106,7 +107,10 @@ pub fn run(cli_args: CliArgs) -> Result { .broker_control_channel .as_deref() .context("--child requires --broker-control-channel")?; - let (_connection, bootstrap) = broker::connect_child(control_socket_path)?; + let (connection, bootstrap) = broker::connect_child(control_socket_path)?; + connection + .local + .report_process_start_failure(ErrorCode::UnsupportedOperation)?; return Err(anyhow!( "unsupported child Linux process bootstrap format {:?} version {:?}", bootstrap.format, diff --git a/litebox_runner_windows_userland/Cargo.toml b/litebox_runner_windows_userland/Cargo.toml index 05a3e888ca..74119768ff 100644 --- a/litebox_runner_windows_userland/Cargo.toml +++ b/litebox_runner_windows_userland/Cargo.toml @@ -8,6 +8,7 @@ anyhow = "1.0.97" clap = { version = "4.5.33", features = ["derive"] } litebox = { version = "0.1.0", path = "../litebox" } litebox_broker_local_userland = { version = "0.1.0", path = "../litebox_broker_local_userland" } +litebox_broker_protocol = { version = "0.1.0", path = "../litebox_broker_protocol" } litebox_common_linux = { version = "0.1.0", path = "../litebox_common_linux" } litebox_platform_windows_userland = { version = "0.1.0", path = "../litebox_platform_windows_userland" } litebox_shim_windows = { version = "0.1.0", path = "../litebox_shim_windows" } diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 94c88bd0e0..327fa40c0a 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -10,6 +10,7 @@ extern crate alloc; use anyhow::{Context as _, Result}; use clap::Parser; use litebox_broker_local_userland as broker; +use litebox_broker_protocol::error::ErrorCode; use litebox_platform_windows_userland::{GuestTlsMode, WindowsUserland}; /// Runs a Windows PE program with LiteBox on unmodified Windows and returns its exit code. @@ -74,7 +75,10 @@ pub fn run(cli_args: CliArgs) -> Result { .broker_control_channel .as_deref() .context("--child requires --broker-control-channel")?; - let (_connection, bootstrap) = broker::connect_child(control_pipe)?; + let (connection, bootstrap) = broker::connect_child(control_pipe)?; + connection + .local + .report_process_start_failure(ErrorCode::UnsupportedOperation)?; anyhow::bail!( "unsupported child Windows process bootstrap format {:?} version {:?}", bootstrap.format, From 7e9f4ab4e5aa24b952b09c79cdc121445e47a85c Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 08:42:05 -0700 Subject: [PATCH 05/21] Unify runner process observation Use one mutex-serialized std::process::Child lifecycle for Linux and Windows exit observation, termination, and final waiting. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- dev_tests/src/ratchet.rs | 1 - litebox_broker_userland/src/runner.rs | 103 ++++++++++++------ litebox_broker_userland/src/runner/linux.rs | 54 +-------- litebox_broker_userland/src/runner/windows.rs | 47 +------- 4 files changed, 74 insertions(+), 131 deletions(-) diff --git a/dev_tests/src/ratchet.rs b/dev_tests/src/ratchet.rs index 96756205bb..d8bcd75789 100644 --- a/dev_tests/src/ratchet.rs +++ b/dev_tests/src/ratchet.rs @@ -78,7 +78,6 @@ fn ratchet_maybe_uninit() -> Result<()> { ("dev_tests/", 1), ("litebox/", 1), ("litebox_broker_transport_linux_userland/", 3), - ("litebox_broker_userland/", 1), ("litebox_platform_linux_userland/", 2), ("litebox_platform_macos_userland/", 2), ], diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index dbda821aae..fe7d204810 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -32,9 +32,9 @@ mod linux; mod windows; #[cfg(target_os = "linux")] -use linux::{PlatformRunnerEndpoint, PlatformRunnerShutdown}; +use linux::PlatformRunnerEndpoint; #[cfg(all(windows, target_arch = "x86_64"))] -use windows::{PlatformRunnerEndpoint, PlatformRunnerShutdown}; +use windows::PlatformRunnerEndpoint; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); const PROCESS_START_RECEIPT_TIMEOUT: Duration = Duration::from_secs(5); @@ -124,7 +124,7 @@ struct ChildRunResult { } struct RunnerShutdown { - platform: PlatformRunnerShutdown, + runner: Arc>, state: Mutex, changed: Condvar, termination_dispatched: AtomicBool, @@ -184,7 +184,21 @@ impl RunnerShutdown { } } drop(state); - if !matches!(self.platform.has_exited(), Ok(true)) && self.platform.shutdown() { + let termination_dispatched = { + // Serialize observation and termination so collecting an exit + // status can never expose a reusable PID between the check and + // the kill request. + let mut runner = self.runner.lock().expect("runner process mutex poisoned"); + match runner.try_wait() { + Ok(Some(_)) => false, + Ok(None) => runner.kill().is_ok(), + Err(_) => { + let _ = runner.kill(); + false + } + } + }; + if termination_dispatched { self.termination_dispatched.store(true, Ordering::Release); } let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); @@ -206,7 +220,7 @@ impl RunnerShutdown { } fn has_exited(&self) -> IoResult { - self.platform.has_exited() + runner_has_exited(&self.runner) } fn termination_was_dispatched(&self) -> bool { @@ -232,18 +246,20 @@ impl RunnerInstance { /// Creates the runner's dedicated control endpoint and starts the runner. pub fn start(config: RunnerConfig) -> IoResult { let endpoint = PlatformRunnerEndpoint::create()?; - let runner = Command::new(&config.executable) - .args(config.arguments(endpoint.control_channel())) - .spawn()?; + let runner = Arc::new(Mutex::new( + Command::new(&config.executable) + .args(config.arguments(endpoint.control_channel())) + .spawn()?, + )); let shutdown = Arc::new(RunnerShutdown { - platform: PlatformRunnerShutdown::new(&runner), + runner: Arc::clone(&runner), state: Mutex::new(RunnerShutdownState::Active), changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), }); let child_config = config.child(); Ok(Self { - runner: Arc::new(Mutex::new(runner)), + runner, shutdown, endpoint, child_config, @@ -273,11 +289,7 @@ impl RunnerInstance { self.shutdown.shutdown(); } self.shutdown.retire(); - let runner_status = self - .runner - .lock() - .expect("runner process mutex poisoned") - .wait(); + let runner_status = wait_for_runner_exit(&self.runner); let root_abnormal = association_result.abnormal || association_result.panicked || runner_exited.is_err() @@ -348,11 +360,7 @@ impl RunnerInstance { } }; self.shutdown.retire(); - let runner_status = self - .runner - .lock() - .expect("runner process mutex poisoned") - .wait(); + let runner_status = wait_for_runner_exit(&self.runner); if runner_status.is_err() { launch.mark_abnormal(); } @@ -388,10 +396,7 @@ impl Drop for RunnerInstance { self.endpoint.close(); self.shutdown.shutdown(); self.shutdown.retire(); - let mut runner = self.runner.lock().expect("runner process mutex poisoned"); - if !matches!(runner.try_wait(), Ok(Some(_status))) { - let _ = runner.wait(); - } + let _ = wait_for_runner_exit(&self.runner); } } @@ -2180,6 +2185,29 @@ fn accept_runner_channel( } } +fn runner_has_exited(runner: &Arc>) -> IoResult { + // A pre-authentication caller stops accepting before acting on `true`; + // post-authentication callers no longer rely on PID-based authentication. + runner + .lock() + .expect("runner process mutex poisoned") + .try_wait() + .map(|status| status.is_some()) +} + +fn wait_for_runner_exit(runner: &Arc>) -> IoResult { + loop { + if let Some(status) = runner + .lock() + .expect("runner process mutex poisoned") + .try_wait()? + { + return Ok(status); + } + std::thread::sleep(ACCEPT_RETRY_DELAY); + } +} + #[cfg(test)] mod tests { use super::{ @@ -2773,24 +2801,25 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn shutdown_can_terminate_while_the_launch_owner_waits() { - use super::{PlatformRunnerShutdown, RunnerShutdown, RunnerShutdownState}; + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; use std::process::Command; - let child = Command::new("sh") - .args(["-c", "exec sleep 30"]) - .spawn() - .unwrap(); + let child = Arc::new(Mutex::new( + Command::new("sh") + .args(["-c", "exec sleep 30"]) + .spawn() + .unwrap(), + )); let shutdown = RunnerShutdown { - platform: PlatformRunnerShutdown::new(&child), + runner: Arc::clone(&child), state: Mutex::new(RunnerShutdownState::Active), changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), }; - let child = Arc::new(Mutex::new(child)); let waiting = Arc::clone(&child); let (finished, completion) = mpsc::sync_channel(1); let waiter = std::thread::spawn(move || { - let status = waiting.lock().unwrap().wait().unwrap(); + let status = wait_for_runner_exit(&waiting).unwrap(); finished.send(status).unwrap(); }); @@ -2808,12 +2837,14 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn shutdown_after_observed_exit_records_no_termination() { - use super::{PlatformRunnerShutdown, RunnerShutdown, RunnerShutdownState}; + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; use std::process::Command; - let mut child = Command::new("sh").args(["-c", "exit 1"]).spawn().unwrap(); + let child = Arc::new(Mutex::new( + Command::new("sh").args(["-c", "exit 1"]).spawn().unwrap(), + )); let shutdown = RunnerShutdown { - platform: PlatformRunnerShutdown::new(&child), + runner: Arc::clone(&child), state: Mutex::new(RunnerShutdownState::Active), changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), @@ -2824,7 +2855,7 @@ mod tests { shutdown.retire(); assert!(!shutdown.termination_was_dispatched()); - assert!(!child.wait().unwrap().success()); + assert!(!wait_for_runner_exit(&child).unwrap().success()); } #[cfg(target_os = "linux")] diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index 8c0a8d8938..db2d34548f 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -15,35 +15,9 @@ use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; -use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited}; use crate::runtime::{AssociationFailureCause, AssociationRunResult}; -pub(super) struct PlatformRunnerShutdown { - process_id: u32, -} - -fn runner_has_exited(runner: &Arc>) -> IoResult { - non_reaping_runner_has_exited(runner.lock().expect("runner process mutex poisoned").id()) -} - -impl PlatformRunnerShutdown { - pub(super) fn new(runner: &Child) -> Self { - Self { - process_id: runner.id(), - } - } - - pub(super) fn shutdown(&self) -> bool { - // SAFETY: the unreaped `Child` keeps this PID allocated to the runner - // until the launch owner closes the endpoint and waits for it. - unsafe { libc::kill(self.process_id.cast_signed(), libc::SIGKILL) == 0 } - } - - pub(super) fn has_exited(&self) -> IoResult { - non_reaping_runner_has_exited(self.process_id) - } -} - pub(super) struct PlatformRunnerEndpoint { socket_path: PathBuf, listener: Option, @@ -207,10 +181,7 @@ fn accept_control_channel( let control_stream = accept_runner_channel( setup_deadline, "control", - || { - non_reaping_runner_has_exited(runner_id) - .map(|exited| exited.then(|| "exited".to_owned())) - }, + || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), || control_listener.accept().map(|(stream, _)| stream), )?; validate_peer_process(&control_stream, runner_id)?; @@ -219,24 +190,3 @@ fn accept_control_channel( setup_deadline, )) } - -fn non_reaping_runner_has_exited(runner_id: u32) -> IoResult { - let mut info = std::mem::MaybeUninit::::zeroed(); - // SAFETY: `info` points to writable `siginfo_t` storage, and `waitid` is - // restricted to observing this known child without consuming its status. - let result = unsafe { - libc::waitid( - libc::P_PID, - runner_id, - info.as_mut_ptr(), - libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, - ) - }; - if result != 0 { - return Err(std::io::Error::last_os_error()); - } - // SAFETY: successful `waitid` initialized `info`; `si_pid == 0` denotes - // that the nonblocking observation found no waitable child state. - let exited = unsafe { info.assume_init().si_pid() != 0 }; - Ok(exited) -} diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index eda067d076..2ae86250e0 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -13,43 +13,10 @@ use litebox_broker_transport_windows_userland::named_pipe::{ WindowsNamedPipeHostSetupChannel, WindowsNamedPipeListener, validate_client_process, }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; -use windows_sys::Win32::Foundation::{HANDLE, WAIT_FAILED, WAIT_OBJECT_0, WAIT_TIMEOUT}; -use windows_sys::Win32::System::Threading::{TerminateProcess, WaitForSingleObject}; -use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel}; +use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited}; use crate::runtime::{AssociationFailureCause, AssociationRunResult}; -pub(super) struct PlatformRunnerShutdown { - process_handle: isize, -} - -impl PlatformRunnerShutdown { - pub(super) fn new(runner: &Child) -> Self { - Self { - process_handle: runner.as_raw_handle() as isize, - } - } - - pub(super) fn shutdown(&self) -> bool { - // SAFETY: the launch owner retains the `Child` and its process handle - // until all shutdown callers finish and the runner has been waited. - unsafe { TerminateProcess(self.process_handle as HANDLE, 1) != 0 } - } - - pub(super) fn has_exited(&self) -> IoResult { - // SAFETY: the launch owner retains the `Child` process handle until - // shutdown is retired and the runner is waited. - match unsafe { WaitForSingleObject(self.process_handle as HANDLE, 0) } { - WAIT_OBJECT_0 => Ok(true), - WAIT_TIMEOUT => Ok(false), - WAIT_FAILED => Err(std::io::Error::last_os_error()), - _ => Err(std::io::Error::other( - "waiting for the runner process returned an unknown status", - )), - } - } -} - pub(super) struct PlatformRunnerEndpoint { pipe_name: OsString, listener: Option, @@ -113,7 +80,7 @@ fn serve_runner_process( { Ok(connection) => connection, Err(error) => { - let failure_cause = if non_reaping_runner_has_exited(runner).unwrap_or(false) { + let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit } else { AssociationFailureCause::Other @@ -155,7 +122,7 @@ fn serve_child_runner_process( { Ok(connection) => connection, Err(error) => { - let failure_cause = if non_reaping_runner_has_exited(runner).unwrap_or(false) { + let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit } else if matches!( error.kind(), @@ -198,7 +165,7 @@ fn serve_child_runner_process( .result .as_ref() .is_err_and(|error| error.kind() == std::io::ErrorKind::BrokenPipe) - && non_reaping_runner_has_exited(runner).unwrap_or(false) + && runner_has_exited(runner).unwrap_or(false) { result.failure_cause = AssociationFailureCause::RunnerExit; } @@ -213,7 +180,7 @@ fn accept_control_channel( let control_stream = accept_runner_channel( setup_deadline, "control", - || non_reaping_runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), + || runner_has_exited(runner).map(|exited| exited.then(|| "exited".to_owned())), || control_listener.try_accept(), )?; let runner_id = runner.lock().expect("runner process mutex poisoned").id(); @@ -224,10 +191,6 @@ fn accept_control_channel( )) } -fn non_reaping_runner_has_exited(runner: &Arc>) -> IoResult { - PlatformRunnerShutdown::new(&runner.lock().expect("runner process mutex poisoned")).has_exited() -} - fn unique_control_pipe_name() -> OsString { let process_id = std::process::id(); let nonce = std::time::SystemTime::now() From afe4b91880d4b4e11c2528c9b289c741bcd8c6e5 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 09:16:41 -0700 Subject: [PATCH 06/21] Model child startup in process state Replace the parallel startup-thread pin with explicit StartReady and StartCommitted process states completed after acknowledgement publication. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/process.rs | 111 +++++++++++++------- litebox_broker_userland/src/runner.rs | 146 +++++++++++++++----------- 2 files changed, 162 insertions(+), 95 deletions(-) diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index 5aa13c765b..7c54a4b994 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -79,7 +79,6 @@ struct ProcessReferences { struct ProcessThreads { entries: HashMap, - startup_pin: Option, } /// Broker-owned state for one guest thread. @@ -132,6 +131,8 @@ pub struct BrokerProcess { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ProcessState { Attaching, + StartReady { initial_thread_id: Option }, + StartCommitted { initial_thread_id: Option }, Running, Exiting, } @@ -157,7 +158,6 @@ impl BrokerProcess { }), threads: Mutex::new(ProcessThreads { entries: HashMap::new(), - startup_pin: None, }), reserved_pipe_capacity: Arc::new(AtomicUsize::new(0)), reserved_sockets: Arc::new(AtomicUsize::new(0)), @@ -186,18 +186,57 @@ impl BrokerProcess { /// Returns whether parent acknowledgement committed this process. #[must_use] pub fn is_running(&self) -> bool { - *self.state.lock() == ProcessState::Running + matches!( + *self.state.lock(), + ProcessState::StartCommitted { .. } | ProcessState::Running + ) + } + + /// Records the initial thread supplied by `ProcessReady`. + pub fn mark_start_ready(&self, initial_thread_id: Option) -> Result<()> { + let mut state = self.state.lock(); + match *state { + ProcessState::Attaching => {} + ProcessState::Exiting => return Err(BrokerError::PeerClosed), + ProcessState::StartReady { .. } + | ProcessState::StartCommitted { .. } + | ProcessState::Running => return Err(BrokerError::Internal), + } + if let Some(thread_id) = initial_thread_id + && !self.threads.lock().entries.contains_key(&thread_id) + { + return Err(BrokerError::UnknownObject); + } + *state = ProcessState::StartReady { initial_thread_id }; + Ok(()) } /// Commits this child after its start result reaches the parent. pub fn commit_start(&self) -> Result<()> { let mut state = self.state.lock(); match *state { - ProcessState::Attaching => { + ProcessState::StartReady { initial_thread_id } => { + *state = ProcessState::StartCommitted { initial_thread_id }; + Ok(()) + } + ProcessState::Attaching + | ProcessState::StartCommitted { .. } + | ProcessState::Running => Err(BrokerError::Internal), + ProcessState::Exiting => Err(BrokerError::PeerClosed), + } + } + + /// Completes startup after the acknowledgement response is published. + pub fn complete_start(&self) -> Result<()> { + let mut state = self.state.lock(); + match *state { + ProcessState::StartCommitted { .. } => { *state = ProcessState::Running; Ok(()) } - ProcessState::Running => Err(BrokerError::Internal), + ProcessState::Attaching | ProcessState::StartReady { .. } | ProcessState::Running => { + Err(BrokerError::Internal) + } ProcessState::Exiting => Err(BrokerError::PeerClosed), } } @@ -282,39 +321,25 @@ impl BrokerProcess { self.threads.lock().entries.contains_key(&thread_id) } - /// Pins a live thread while its identity is part of process-start publication. - pub fn pin_startup_thread(&self, thread_id: ThreadId) -> Result<()> { - let mut threads = self.threads.lock(); - if !threads.entries.contains_key(&thread_id) { - return Err(BrokerError::UnknownObject); - } - if threads.startup_pin.is_some() { - return Err(BrokerError::Internal); - } - threads.startup_pin = Some(thread_id); - Ok(()) - } - - /// Releases a thread identity after process-start publication reaches a terminal state. - pub fn release_startup_thread(&self, thread_id: ThreadId) -> Result<()> { - let mut threads = self.threads.lock(); - if threads.startup_pin != Some(thread_id) { - return Err(BrokerError::Internal); - } - threads.startup_pin = None; - Ok(()) - } - /// Records broker thread exit after its local task teardown completes. pub fn exit_thread(&self, thread_id: ThreadId) -> Result<()> { - let mut threads = self.threads.lock(); - if threads.startup_pin == Some(thread_id) { + let state = self.state.lock(); + if matches!( + *state, + ProcessState::StartReady { + initial_thread_id: Some(pinned), + } | ProcessState::StartCommitted { + initial_thread_id: Some(pinned), + } if pinned == thread_id + ) { return Err(BrokerError::WouldBlock); } + let mut threads = self.threads.lock(); let thread = threads .entries .remove(&thread_id) .ok_or(BrokerError::UnknownObject)?; + drop(state); drop(threads); self.core .active_thread_count @@ -813,7 +838,6 @@ impl BrokerProcess { let threads = { let mut threads = self.threads.lock(); - threads.startup_pin = None; core::mem::take(&mut threads.entries) }; if release_ids && !invariant_fault { @@ -919,7 +943,7 @@ impl Drop for BrokerProcess { mod tests { use core::sync::atomic::{AtomicUsize, Ordering}; - use super::{ProcessReferences, release_pending_reference}; + use super::{ProcessReferences, ProcessState, release_pending_reference}; use crate::test_platform::TestPlatform; use crate::test_support::{TestBrokerCoreBuilder, TestStdioProvider}; use crate::{ @@ -985,21 +1009,27 @@ mod tests { } #[test] - fn startup_thread_pin_defers_thread_exit_until_publication_completes() { + fn startup_states_defer_initial_thread_exit_until_publication_completes() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) .build() .unwrap(); - let process = broker + let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); + let (process, _) = parent.create_child(&[]).unwrap(); let thread = process.create_thread().unwrap(); - process.pin_startup_thread(thread).unwrap(); + process.mark_start_ready(Some(thread)).unwrap(); assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); - process.release_startup_thread(thread).unwrap(); + process.commit_start().unwrap(); + assert!(process.is_running()); + assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); + process.complete_start().unwrap(); assert_eq!(process.exit_thread(thread), Ok(())); + process.finish(); + parent.finish(); } #[test] @@ -1026,8 +1056,17 @@ mod tests { ); assert!(!child.is_running()); + child.mark_start_ready(None).unwrap(); child.commit_start().unwrap(); assert!(child.is_running()); + assert!(matches!( + *child.state.lock(), + ProcessState::StartCommitted { + initial_thread_id: None + } + )); + child.complete_start().unwrap(); + assert_eq!(*child.state.lock(), ProcessState::Running); } #[test] diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index fe7d204810..7f5e51dade 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -438,6 +438,8 @@ enum ChildLaunchPhase { Ready { initial_thread_id: Option }, Committing, Committed, + CompletingStart, + StartComplete, Aborted(ErrorCode), } @@ -460,7 +462,6 @@ struct ChildLaunchData { acknowledgement_publication_watchdog: DeadlineState, start_failure_publication_watchdog: DeadlineState, active_control_callbacks: usize, - pinned_initial_thread_id: Option, runner_finished: bool, finalization_taken: bool, } @@ -663,7 +664,6 @@ impl RunnerChildren { acknowledgement_publication_watchdog: DeadlineState::Unarmed, start_failure_publication_watchdog: DeadlineState::Unarmed, active_control_callbacks: 0, - pinned_initial_thread_id: None, runner_finished: false, finalization_taken: false, }), @@ -1318,7 +1318,10 @@ impl ChildLaunch { .expect("child launch mutex poisoned"); } ChildLaunchPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), - ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => { + ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete => { return Err(ErrorCode::ProtocolState); } ChildLaunchPhase::Aborted(error) => return Err(error), @@ -1327,40 +1330,31 @@ impl ChildLaunch { } fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { - if let Some(thread_id) = initial_thread_id { - self.process - .pin_startup_thread(thread_id) - .map_err(|error| match error { - BrokerError::UnknownObject => ErrorCode::ProtocolState, - error => ErrorCode::from(error), - })?; - } + self.process + .mark_start_ready(initial_thread_id) + .map_err(|error| match error { + BrokerError::UnknownObject => ErrorCode::ProtocolState, + error => ErrorCode::from(error), + })?; let mut state = self.state.lock().expect("child launch mutex poisoned"); if !matches!(state.phase, ChildLaunchPhase::Starting) { - let error = match state.phase { + return Err(match state.phase { ChildLaunchPhase::Aborted(error) => error, _ => ErrorCode::ProtocolState, - }; - drop(state); - if let Some(thread_id) = initial_thread_id { - let _ = self.process.release_startup_thread(thread_id); - } - return Err(error); + }); } - state.pinned_initial_thread_id = initial_thread_id; state.phase = ChildLaunchPhase::Ready { initial_thread_id }; self.changed.notify_all(); loop { match state.phase { - ChildLaunchPhase::Committed - if state.pinned_initial_thread_id.is_none() - && state.active_control_callbacks == 0 => - { + ChildLaunchPhase::StartComplete if state.active_control_callbacks == 0 => { return Ok(()); } ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed => { + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete => { state = self .changed .wait(state) @@ -1515,7 +1509,13 @@ impl ChildLaunch { return Ok(ProcessStartAcknowledgement::Failed(error)); } (ChildLaunchPhase::Starting, _) => return Err(ErrorCode::ProtocolState), - (ChildLaunchPhase::Committing | ChildLaunchPhase::Committed, _) => { + ( + ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete, + _, + ) => { return Err(ErrorCode::ProtocolState); } (_, StartResultPublication::NotStarted) => { @@ -1612,7 +1612,10 @@ impl ChildLaunch { | ShutdownRequest::ExpectedStartFailure | ShutdownRequest::Unexpected => (None, None), }, - ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => (None, None), + ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete => (None, None), } }; if let Some(association_failure) = association_failure { @@ -1662,7 +1665,10 @@ impl ChildLaunch { .lock() .expect("child launch mutex poisoned") .phase, - ChildLaunchPhase::Committing | ChildLaunchPhase::Committed + ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete ) } @@ -1685,7 +1691,7 @@ impl ChildLaunch { complete_deadline(&mut state.acknowledgement_publication_watchdog); complete_deadline(&mut state.start_failure_publication_watchdog); self.changed.notify_all(); - ReceiptResolution::Resolved(self.release_startup_pin_and_take_finalization(state)) + ReceiptResolution::Resolved(self.complete_process_start_and_take_finalization(state)) } fn begin_receipt_drain(&self) { @@ -1708,7 +1714,7 @@ impl ChildLaunch { complete_deadline(&mut state.acknowledgement_publication_watchdog); complete_deadline(&mut state.start_failure_publication_watchdog); self.changed.notify_all(); - self.release_startup_pin_and_take_finalization(state) + self.complete_process_start_and_take_finalization(state) } fn expire_initial_receipt_deadline(&self) -> Option { @@ -1888,30 +1894,28 @@ impl ChildLaunch { true } - fn release_startup_pin_and_take_finalization( + fn complete_process_start_and_take_finalization( &self, mut state: MutexGuard<'_, ChildLaunchData>, ) -> Option { - let pinned_thread_id = state.pinned_initial_thread_id.take(); - if pinned_thread_id.is_none() { + if !matches!(state.phase, ChildLaunchPhase::Committed) { return take_finalization(&mut state); } + state.phase = ChildLaunchPhase::CompletingStart; state.active_control_callbacks = state .active_control_callbacks .checked_add(1) .expect("process-start control callback count must remain bounded"); drop(state); - let release_failed = self - .process - .release_startup_thread(pinned_thread_id.expect("thread pin was checked")) - .is_err(); + let completion_failed = self.process.complete_start().is_err(); let mut state = self.state.lock().expect("child launch mutex poisoned"); state.active_control_callbacks = state .active_control_callbacks .checked_sub(1) .expect("process-start control callback count must remain balanced"); - state.abnormal |= release_failed; + state.abnormal |= completion_failed; + state.phase = ChildLaunchPhase::StartComplete; self.changed.notify_all(); take_finalization(&mut state) } @@ -1965,7 +1969,10 @@ fn expire_launch( (None, None) } } - ChildLaunchPhase::Committing | ChildLaunchPhase::Committed => (None, None), + ChildLaunchPhase::Committing + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete => (None, None), }; changed.notify_all(); Some(ReceiptExpiration { @@ -2031,7 +2038,10 @@ fn expire_internal_resolution( Some(Instant::now() + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT), ) } - ChildLaunchPhase::Starting | ChildLaunchPhase::Committed => { + ChildLaunchPhase::Starting + | ChildLaunchPhase::Committed + | ChildLaunchPhase::CompletingStart + | ChildLaunchPhase::StartComplete => { state.active_control_callbacks -= 1; return None; } @@ -2230,7 +2240,10 @@ mod tests { }; use std::time::{Duration, Instant}; - fn launch_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { + fn launch_with_broker_in_phase( + publication: StartResultPublication, + phase: ChildLaunchPhase, + ) -> (BrokerCore, Arc) { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -2241,15 +2254,16 @@ mod tests { .unwrap(); let (process, _) = parent.create_child(&[]).unwrap(); parent.finish(); + if let ChildLaunchPhase::Ready { initial_thread_id } = phase { + process.mark_start_ready(initial_thread_id).unwrap(); + } ( broker, Arc::new(ChildLaunch { parent_id: ProcessId(1), process, state: Mutex::new(ChildLaunchData { - phase: ChildLaunchPhase::Ready { - initial_thread_id: None, - }, + phase, publication, shutdown: None, association_failure: None, @@ -2260,7 +2274,6 @@ mod tests { acknowledgement_publication_watchdog: DeadlineState::Unarmed, start_failure_publication_watchdog: DeadlineState::Unarmed, active_control_callbacks: 0, - pinned_initial_thread_id: None, runner_finished: false, finalization_taken: false, }), @@ -2269,10 +2282,23 @@ mod tests { ) } + fn launch_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { + launch_with_broker_in_phase( + publication, + ChildLaunchPhase::Ready { + initial_thread_id: None, + }, + ) + } + fn launch(publication: StartResultPublication) -> Arc { launch_with_broker(publication).1 } + fn starting_launch(publication: StartResultPublication) -> Arc { + launch_with_broker_in_phase(publication, ChildLaunchPhase::Starting).1 + } + #[test] fn publication_captures_an_absolute_receipt_deadline() { let launch = launch(StartResultPublication::NotStarted); @@ -2327,8 +2353,7 @@ mod tests { #[test] fn reported_bootstrap_rejection_selects_normal_rollback() { - let launch = launch(StartResultPublication::NotStarted); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + let launch = starting_launch(StartResultPublication::NotStarted); launch .report_start_failure(ErrorCode::UnsupportedOperation) @@ -2409,8 +2434,7 @@ mod tests { #[test] fn process_ready_interrupted_by_abort_returns_the_abort_cause() { - let ready = launch(StartResultPublication::NotStarted); - ready.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + let ready = starting_launch(StartResultPublication::NotStarted); ready.abort(ErrorCode::PeerClosed, false, true); assert!(matches!( ready.ready_and_wait(None), @@ -2422,8 +2446,7 @@ mod tests { #[test] fn failure_report_interrupted_by_abort_returns_the_abort_cause() { - let failed = launch(StartResultPublication::NotStarted); - failed.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + let failed = starting_launch(StartResultPublication::NotStarted); failed.abort(ErrorCode::PeerClosed, false, true); assert!(matches!( failed.report_start_failure(ErrorCode::UnsupportedOperation), @@ -2488,8 +2511,7 @@ mod tests { #[test] fn start_failure_publication_timeout_terminates_the_child() { - let launch = launch(StartResultPublication::NotStarted); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + let launch = starting_launch(StartResultPublication::NotStarted); launch .report_start_failure(ErrorCode::UnsupportedOperation) .unwrap(); @@ -2665,22 +2687,27 @@ mod tests { } #[test] - fn receipt_resolution_releases_the_initial_thread_pin() { - let launch = launch(StartResultPublication::Delivered); + fn receipt_resolution_completes_process_start() { + let launch = starting_launch(StartResultPublication::Delivered); let thread_id = launch.process.create_thread().unwrap(); - launch.process.pin_startup_thread(thread_id).unwrap(); - launch.state.lock().unwrap().pinned_initial_thread_id = Some(thread_id); + launch.process.mark_start_ready(Some(thread_id)).unwrap(); + launch.process.commit_start().unwrap(); + launch.state.lock().unwrap().phase = ChildLaunchPhase::Committed; + launch.resolve_receipt(); launch.resolve_receipt(); + let state = launch.state.lock().unwrap(); + assert!(matches!(state.phase, ChildLaunchPhase::StartComplete)); + assert!(!state.abnormal); + drop(state); assert_eq!(launch.process.exit_thread(thread_id), Ok(())); Arc::clone(&launch.process).finish(); } #[test] - fn process_ready_waits_until_the_initial_thread_pin_is_released() { - let launch = launch(StartResultPublication::Delivered); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Starting; + fn process_ready_waits_until_process_start_is_complete() { + let launch = starting_launch(StartResultPublication::Delivered); let thread_id = launch.process.create_thread().unwrap(); let waiting = Arc::clone(&launch); let (sender, receiver) = mpsc::sync_channel(1); @@ -2694,6 +2721,7 @@ mod tests { while !matches!(state.phase, ChildLaunchPhase::Ready { .. }) { state = launch.changed.wait(state).unwrap(); } + launch.process.commit_start().unwrap(); state.phase = ChildLaunchPhase::Committed; launch.changed.notify_all(); drop(state); From f40e3df19cea65d07680f287c58bd95993c61f4e Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 10:12:14 -0700 Subject: [PATCH 07/21] Simplify process lifecycle cleanup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/process.rs | 96 +++++++-------------------- litebox_broker_host/src/lib.rs | 8 +-- litebox_broker_userland/src/runner.rs | 62 ++++++++--------- 3 files changed, 55 insertions(+), 111 deletions(-) diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index 7c54a4b994..7b8378b6c9 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -77,10 +77,6 @@ struct ProcessReferences { pending_handles: usize, } -struct ProcessThreads { - entries: HashMap, -} - /// Broker-owned state for one guest thread. /// /// Execution remains platform-local. This object owns the authoritative @@ -119,7 +115,7 @@ pub struct BrokerProcess { /// Handles of the live object references owned by this process. references: Mutex, /// Authoritative broker threads owned by this process. - threads: Mutex, + threads: Mutex>, /// Pipe capacity charged to this process by live pipe objects. pub(crate) reserved_pipe_capacity: Arc, /// Socket quota held by pending, live, and closing in-flight resources. @@ -156,9 +152,7 @@ impl BrokerProcess { handles: Vec::new(), pending_handles: 0, }), - threads: Mutex::new(ProcessThreads { - entries: HashMap::new(), - }), + threads: Mutex::new(HashMap::new()), reserved_pipe_capacity: Arc::new(AtomicUsize::new(0)), reserved_sockets: Arc::new(AtomicUsize::new(0)), cancellation: AssociationCancellation::default(), @@ -203,7 +197,7 @@ impl BrokerProcess { | ProcessState::Running => return Err(BrokerError::Internal), } if let Some(thread_id) = initial_thread_id - && !self.threads.lock().entries.contains_key(&thread_id) + && !self.threads.lock().contains_key(&thread_id) { return Err(BrokerError::UnknownObject); } @@ -270,7 +264,7 @@ impl BrokerProcess { match inherited_result { Ok(child_handles) => Ok((process, child_handles)), Err(error) => { - BrokerProcess::finish(process); + process.cleanup(true); Err(error) } } @@ -284,11 +278,10 @@ impl BrokerProcess { /// invariants. pub fn create_thread(&self) -> Result { let mut threads = self.threads.lock(); - if threads.entries.len() >= self.core.limits.max_threads_per_process { + if threads.len() >= self.core.limits.max_threads_per_process { return Err(BrokerError::ResourceExhausted); } threads - .entries .try_reserve(1) .map_err(|_| BrokerError::OutOfMemory)?; self.core @@ -309,7 +302,7 @@ impl BrokerProcess { let thread = BrokerThread::new(ThreadId(raw_id)); let thread_id = thread.id(); assert!( - threads.entries.insert(thread_id, thread).is_none(), + threads.insert(thread_id, thread).is_none(), "the ID allocator returned an occupied thread ID" ); Ok(thread_id) @@ -318,7 +311,7 @@ impl BrokerProcess { /// Returns whether this process owns the live broker thread. #[must_use] pub fn owns_thread(&self, thread_id: ThreadId) -> bool { - self.threads.lock().entries.contains_key(&thread_id) + self.threads.lock().contains_key(&thread_id) } /// Records broker thread exit after its local task teardown completes. @@ -336,7 +329,6 @@ impl BrokerProcess { } let mut threads = self.threads.lock(); let thread = threads - .entries .remove(&thread_id) .ok_or(BrokerError::UnknownObject)?; drop(state); @@ -360,24 +352,6 @@ impl BrokerProcess { self.cancellation.is_cancelled() } - /// Completes non-unwinding process teardown and releases its IDs. - /// - /// Dropping a process without calling this method performs authority - /// cleanup but leaves its numeric IDs occupied so they cannot be reused - /// after an unwind. Calling this method more than once is harmless. - pub fn finish(self: Arc) { - self.cleanup(true); - } - - /// Completes abnormal process teardown without releasing numeric IDs. - /// - /// Calling this method more than once is harmless. It is used after an - /// unwind or invariant failure where reusing process or thread IDs would - /// make later observations ambiguous. - pub fn finish_abnormal(self: Arc) { - self.cleanup(false); - } - pub(crate) fn create_object_reference(&self, object: ObjectEntry) -> Result { let rights = self .core @@ -771,10 +745,15 @@ impl BrokerProcess { Ok(reference) } - fn cleanup(&self, release_ids: bool) -> bool { + /// Cleans up this process, optionally releasing its numeric IDs for reuse. + /// + /// Set `release_ids` only after fully accounted teardown. Dropping a + /// process calls this with `false`, retaining IDs after an unwind or other + /// uncertain retirement. Calling this method more than once is harmless. + pub fn cleanup(&self, release_ids: bool) { let mut process_state = self.state.lock(); if *process_state == ProcessState::Exiting { - return false; + return; } *process_state = ProcessState::Exiting; drop(process_state); @@ -836,10 +815,7 @@ impl BrokerProcess { invariant_fault = true; } - let threads = { - let mut threads = self.threads.lock(); - core::mem::take(&mut threads.entries) - }; + let threads = core::mem::take(&mut *self.threads.lock()); if release_ids && !invariant_fault { self.core .active_thread_count @@ -850,7 +826,6 @@ impl BrokerProcess { } ids.release(self.id.0); } - invariant_fault } } @@ -935,7 +910,7 @@ fn release_pending_reference( impl Drop for BrokerProcess { fn drop(&mut self) { - let _ = self.cleanup(false); + self.cleanup(false); } } @@ -1028,8 +1003,8 @@ mod tests { assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); process.complete_start().unwrap(); assert_eq!(process.exit_thread(thread), Ok(())); - process.finish(); - parent.finish(); + process.cleanup(true); + parent.cleanup(true); } #[test] @@ -1044,10 +1019,8 @@ mod tests { .unwrap(); let source_handle = crate::event::create(&parent, 1).unwrap(); let (child, inherited_objects) = parent.create_child(&[source_handle]).unwrap(); - let child_id = child.id(); let inherited_handle = inherited_objects[0]; - assert_eq!(child.id(), child_id); assert_eq!(child.parent_id(), Some(parent.id())); assert_ne!(inherited_handle, source_handle); assert_eq!( @@ -1086,7 +1059,7 @@ mod tests { parent.create_child(&[]).err(), Some(BrokerError::ResourceExhausted) ); - child.finish(); + child.cleanup(true); assert!(parent.create_child(&[]).is_ok()); } @@ -1103,33 +1076,12 @@ mod tests { .unwrap(); let (child, _) = parent.create_child(&[]).unwrap(); - Arc::clone(&child).finish(); - Arc::clone(&child).finish(); + child.cleanup(true); + child.cleanup(true); assert_eq!(child.commit_start(), Err(BrokerError::PeerClosed)); let (replacement, _) = parent.create_child(&[]).unwrap(); - replacement.finish(); - } - - #[test] - fn child_releases_process_capacity_after_teardown() { - let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( - ObjectRights::all(), - )) - .with_limits(BrokerCoreLimits::DEFAULT.with_process_limit(2)) - .build() - .unwrap(); - let parent = broker - .create_process(CallerCredential::Unauthenticated) - .unwrap(); - let (child, _) = parent.create_child(&[]).unwrap(); - - assert_eq!( - parent.create_child(&[]).err(), - Some(BrokerError::ResourceExhausted) - ); - child.finish(); - assert!(parent.create_child(&[]).is_ok()); + replacement.cleanup(true); } #[test] @@ -1215,7 +1167,7 @@ mod tests { Err(BrokerError::ResourceExhausted) )); - first.finish(); + first.cleanup(true); assert!( broker .create_process(CallerCredential::Unauthenticated) @@ -1247,7 +1199,7 @@ mod tests { assert!(Arc::ptr_eq(&process, ®istered)); drop(registered); - process.finish(); + process.cleanup(true); assert!(!broker.processes.read().contains_key(&process_id)); assert_eq!(broker.active_thread_count.load(Ordering::Relaxed), 0); diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 6b3ef9540f..6cb7c57726 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -130,7 +130,7 @@ impl BrokerHostAssociation<'_, Memory> { /// Completes non-unwinding association teardown and releases its process ID. pub fn finish(self) { - BrokerProcess::finish(self.process); + self.process.cleanup(true); } /// Executes one active request and emits its response. @@ -395,13 +395,13 @@ where let process_retained = retain_process(&process); if let Err(error) = setup_channel.send_handshake_response(&response) { if finish_on_setup_error && !process_retained { - BrokerProcess::finish(process); + process.cleanup(true); } return Err(BrokerHostError::Channel(error)); } if let Err(error) = send_shared_memory(setup_channel) { if finish_on_setup_error && !process_retained { - BrokerProcess::finish(process); + process.cleanup(true); } return Err(BrokerHostError::Channel(error)); } @@ -1637,7 +1637,7 @@ mod tests { .unwrap() .expect("deployment owner must retain the negotiated process"); assert!(process.is_running()); - process.finish(); + process.cleanup(true); } fn association_shared_buffer_sequences_stage_file_data(broker: &BrokerCore) { diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 7f5e51dade..fb5a327bab 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -301,7 +301,7 @@ impl RunnerInstance { ) || runner_exit_code_is_crash(status.code()) }); if let Some(process) = association_result.process.take() { - finish_child_process(&process, root_abnormal); + process.cleanup(!root_abnormal); } children.wait_for_drain(); let runner_status = runner_status?; @@ -706,7 +706,7 @@ impl RunnerChildren { })() { Ok(token) => token, Err(error) => { - BrokerProcess::finish(process); + process.cleanup(true); return Err(error); } }; @@ -766,7 +766,7 @@ impl RunnerChildren { }); if thread.is_err() { self.remove_launch(token); - BrokerProcess::finish(Arc::clone(&launch.process)); + launch.process.cleanup(true); self.finish_instance(); return Err(ErrorCode::OutOfMemory); } @@ -1224,7 +1224,7 @@ impl RunnerChildren { } fn finish_child_launch(&self, launch: &ChildLaunch, abnormal: bool) { - finish_child_process(&launch.process, abnormal); + launch.process.cleanup(!abnormal); self.finish_instance(); } @@ -2092,14 +2092,6 @@ fn take_finalization(state: &mut ChildLaunchData) -> Option { Some(state.abnormal) } -fn finish_child_process(process: &Arc, abnormal: bool) { - if abnormal { - Arc::clone(process).finish_abnormal(); - } else { - Arc::clone(process).finish(); - } -} - #[cfg(target_os = "linux")] fn runner_exit_signal(status: ExitStatus) -> Option { use std::os::unix::process::ExitStatusExt; @@ -2253,7 +2245,7 @@ mod tests { .create_process(CallerCredential::Unauthenticated) .unwrap(); let (process, _) = parent.create_child(&[]).unwrap(); - parent.finish(); + parent.cleanup(true); if let ChildLaunchPhase::Ready { initial_thread_id } = phase { process.mark_start_ready(initial_thread_id).unwrap(); } @@ -2345,7 +2337,7 @@ mod tests { &BrokerOperation::AcknowledgeProcessStart(token), &BrokerResult::ProcessStartAcknowledged, ); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); children.finish_instance(); children.wait_for_drain(); assert_eq!(children.state.lock().unwrap().active_watchdogs, 0); @@ -2376,7 +2368,7 @@ mod tests { launch.resolve_receipt(), ReceiptResolution::Resolved(Some(false)) )); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2414,7 +2406,7 @@ mod tests { )); launch.resolve_receipt(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2429,7 +2421,7 @@ mod tests { assert!(matches!(worker.join().unwrap(), Err(ErrorCode::PeerClosed))); launch.finish_receipt_drain(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2441,7 +2433,7 @@ mod tests { Err(ErrorCode::PeerClosed) )); ready.resolve_receipt(); - Arc::clone(&ready.process).finish(); + ready.process.cleanup(true); } #[test] @@ -2453,7 +2445,7 @@ mod tests { Err(ErrorCode::PeerClosed) )); failed.resolve_receipt(); - Arc::clone(&failed.process).finish(); + failed.process.cleanup(true); } #[test] @@ -2473,7 +2465,7 @@ mod tests { launch.begin_receipt_drain(); launch.finish_receipt_drain(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2506,7 +2498,7 @@ mod tests { assert_eq!(launch.complete_timeout_callback(), None); launch.begin_receipt_drain(); launch.finish_receipt_drain(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2539,7 +2531,7 @@ mod tests { launch.resolve_receipt(), ReceiptResolution::Resolved(Some(false)) )); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2580,7 +2572,7 @@ mod tests { DeadlineState::Armed(_) )); launch.resolve_receipt(); - Arc::clone(&launch.process).finish_abnormal(); + launch.process.cleanup(false); } #[test] @@ -2595,7 +2587,7 @@ mod tests { launch.abort(ErrorCode::PeerClosed, false, true); assert!(failed.load(Ordering::Acquire)); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2612,7 +2604,7 @@ mod tests { launch.changed.notify_all(); assert!(worker.join().unwrap()); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2643,7 +2635,7 @@ mod tests { launch.changed.notify_all(); assert_eq!(launch.complete_timeout_callback(), None); launch.finish_receipt_drain(); - Arc::clone(&launch.process).finish_abnormal(); + launch.process.cleanup(false); } #[test] @@ -2658,7 +2650,7 @@ mod tests { Ok(ProcessStartAcknowledgement::Failed(ErrorCode::PeerClosed)) )); launch.resolve_receipt(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2670,7 +2662,7 @@ mod tests { Err(ErrorCode::ProtocolState) )); launch.resolve_receipt(); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2683,7 +2675,7 @@ mod tests { launch.resolve_receipt(), ReceiptResolution::Resolved(Some(false)) )); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2702,7 +2694,7 @@ mod tests { assert!(!state.abnormal); drop(state); assert_eq!(launch.process.exit_thread(thread_id), Ok(())); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2734,7 +2726,7 @@ mod tests { ); worker.join().unwrap(); assert_eq!(launch.process.exit_thread(thread_id), Ok(())); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2751,7 +2743,7 @@ mod tests { assert!(state.abnormal); drop(state); assert_eq!(launch.complete_timeout_callback(), None); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2767,7 +2759,7 @@ mod tests { assert_eq!(launch.complete_timeout_callback(), None); launch.begin_receipt_drain(); assert_eq!(launch.finish_receipt_drain(), Some(false)); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2777,7 +2769,7 @@ mod tests { launch.begin_receipt_drain(); assert_eq!(launch.runner_finished(false), None); assert_eq!(launch.finish_receipt_drain(), Some(false)); - Arc::clone(&launch.process).finish(); + launch.process.cleanup(true); } #[test] @@ -2790,7 +2782,7 @@ mod tests { assert_eq!(launch.complete_timeout_callback(), None); launch.begin_receipt_drain(); assert_eq!(launch.finish_receipt_drain(), Some(true)); - Arc::clone(&launch.process).finish_abnormal(); + launch.process.cleanup(false); } #[test] From 4e3bc453d9caf604d14ff87eea19682e4220e6fa Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 12:23:05 -0700 Subject: [PATCH 08/21] Simplify broker host request plumbing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_host/src/lib.rs | 137 ++++++------------ litebox_broker_host/src/test_support.rs | 1 + litebox_broker_userland/src/runner.rs | 12 +- litebox_broker_userland/src/runtime.rs | 5 +- .../tests/notification_runtime.rs | 2 + litebox_runner_linux_userland/tests/run.rs | 1 + 6 files changed, 59 insertions(+), 99 deletions(-) diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 6cb7c57726..786186a616 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -95,9 +95,9 @@ struct AssociationState { shared_buffer_usage: SharedBufferUsage, } -/// Failure classification for deployment-specific broker operations. +/// Failure classification for broker request handling. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BrokerHostExtensionError { +pub enum RequestFailure { /// Return an error result and keep serving the association. Respond(ErrorCode), /// Fail the association without publishing a response. @@ -116,7 +116,23 @@ pub struct ProcessStartupData { pub inherited_objects: Vec, } -impl BrokerHostAssociation<'_, Memory> { +impl<'a, Memory: SharedMemory> BrokerHostAssociation<'a, Memory> { + fn new( + process: Arc, + shared_buffers: &'a SharedBufferPool, + readiness_sink: Arc, + ) -> Self { + Self { + process, + shared_buffers, + readiness_sink, + state: SpinMutex::new(AssociationState { + failed: false, + shared_buffer_usage: SharedBufferUsage::new(), + }), + } + } + /// Returns the broker-assigned process ID for this association. #[must_use] pub fn process_id(&self) -> litebox_broker_protocol::ProcessId { @@ -162,8 +178,7 @@ impl BrokerHostAssociation<'_, Memory> { &BrokerProcess, &BrokerOperation, &SharedBufferPool, - ) - -> Option>, + ) -> Option>, send_response: impl FnOnce(&BrokerResponse) -> core::result::Result<(), ChannelError>, response_sent: impl FnOnce(&BrokerOperation, &BrokerResult), ) -> Result<(), ChannelError> { @@ -192,10 +207,7 @@ impl BrokerHostAssociation<'_, Memory> { } let request_result = match extension(&self.process, &operation, self.shared_buffers) { - Some(result) => result.map_err(|error| match error { - BrokerHostExtensionError::Respond(error) => RequestFailure::Respond(error), - BrokerHostExtensionError::Abort(error) => RequestFailure::Abort(error), - }), + Some(result) => result, None => handle_request( &self.process, operation, @@ -226,51 +238,10 @@ impl BrokerHostAssociation<'_, Memory> { } } -/// Copies a validated operation-scoped shared-buffer sequence. -pub fn copy_shared_buffer( - shared_buffers: &SharedBufferPool, - buffer: SharedBufferSequence, - maximum_length: u32, -) -> core::result::Result, BrokerHostExtensionError> { - read_shared_buffer(shared_buffers, buffer, maximum_length).map_err(|error| match error { - RequestFailure::Respond(error) => BrokerHostExtensionError::Respond(error), - RequestFailure::Abort(error) => BrokerHostExtensionError::Abort(error), - }) -} - -/// Authenticates and negotiates one broker control connection. -/// -/// `send_shared_memory` runs after version negotiation and before the active -/// association is returned. -pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( - core: &BrokerCore, - process: Option>, - startup: Option, - setup_channel: &mut SetupChannel, - shared_buffers: &'a SharedBufferPool, - readiness_sink: Arc, - send_shared_memory: impl FnOnce(&mut SetupChannel) -> core::result::Result<(), ChannelError>, -) -> Result, ChannelError> -where - SetupChannel: HostSetupChannel, - Memory: SharedMemory, -{ - setup_connection_with_process( - core, - process, - startup, - setup_channel, - shared_buffers, - readiness_sink, - |_| false, - send_shared_memory, - ) -} - /// Authenticates and negotiates one broker control connection while allowing /// a deployment owner to retain the negotiated process before setup completes. #[allow(clippy::too_many_arguments)] -pub fn setup_connection_with_process<'a, SetupChannel, Memory, ChannelError>( +pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( core: &BrokerCore, process: Option>, startup: Option, @@ -305,7 +276,7 @@ where let buffer = SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], bootstrap_length) .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; write_shared_buffer(shared_buffers, buffer, &payload, MAX_PROCESS_BOOTSTRAP_SIZE) - .map_err(|error| BrokerHostError::Broker(request_failure_error(error)))?; + .map_err(|error| BrokerHostError::Broker(error.into()))?; Some(ProcessStartup { bootstrap: ProcessBootstrap { format, @@ -405,36 +376,16 @@ where } return Err(BrokerHostError::Channel(error)); } - return Ok(Ok(new_association(process, shared_buffers, readiness_sink))); - } -} - -fn new_association( - process: Arc, - shared_buffers: &SharedBufferPool, - readiness_sink: Arc, -) -> BrokerHostAssociation<'_, Memory> { - BrokerHostAssociation { - process, - shared_buffers, - readiness_sink, - state: SpinMutex::new(AssociationState { - failed: false, - shared_buffer_usage: SharedBufferUsage::new(), - }), + return Ok(Ok(BrokerHostAssociation::new( + process, + shared_buffers, + readiness_sink, + ))); } } type RequestResult = core::result::Result; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RequestFailure { - /// Send an error response and continue serving the association. - Respond(ErrorCode), - /// Terminate the association without sending a response. - Abort(ErrorCode), -} - impl From for RequestFailure { fn from(error: litebox_broker_core::BrokerError) -> Self { match error { @@ -447,6 +398,14 @@ impl From for RequestFailure { } } +impl From for ErrorCode { + fn from(error: RequestFailure) -> Self { + match error { + RequestFailure::Respond(error) | RequestFailure::Abort(error) => error, + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum SharedBufferSlotState { Unused, @@ -522,12 +481,6 @@ fn complete_request( } } -const fn request_failure_error(error: RequestFailure) -> ErrorCode { - match error { - RequestFailure::Respond(error) | RequestFailure::Abort(error) => error, - } -} - fn handle_request( process: &BrokerProcess, operation: BrokerOperation, @@ -636,7 +589,7 @@ fn handle_file_request( buffer, offset, }) => { - let data = read_shared_buffer(shared_buffers, buffer, MAX_FILE_TRANSFER_SIZE)?; + let data = copy_shared_buffer(shared_buffers, buffer, MAX_FILE_TRANSFER_SIZE)?; match litebox_broker_core::fs::write(process, handle, &data, offset) .map_err(RequestFailure::from)? { @@ -808,7 +761,8 @@ fn allocate_zeroed(length: u32) -> RequestResult> { Ok(data) } -fn read_shared_buffer( +/// Copies a validated operation-scoped shared-buffer sequence. +pub fn copy_shared_buffer( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, max_length: u32, @@ -864,7 +818,7 @@ fn read_file_path( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, ) -> RequestResult { - let data = read_shared_buffer(shared_buffers, buffer, SHARED_BUFFER_SLOT_SIZE)?; + let data = copy_shared_buffer(shared_buffers, buffer, SHARED_BUFFER_SLOT_SIZE)?; let path = alloc::string::String::from_utf8(data) .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))?; if !path.starts_with('/') { @@ -895,7 +849,7 @@ fn handle_stdio_request( })) } StdioRequest::Write(WriteStdioRequest { stream, buffer }) => { - let data = read_shared_buffer(shared_buffers, buffer, MAX_STDIO_TRANSFER_SIZE)?; + let data = copy_shared_buffer(shared_buffers, buffer, MAX_STDIO_TRANSFER_SIZE)?; let written = litebox_broker_core::stdio::write(process, stream, &data) .map_err(RequestFailure::from)?; Ok(StdioResponse::Write(WriteStdioResponse { @@ -986,7 +940,7 @@ fn handle_socket_request( return Err(RequestFailure::Abort(ErrorCode::MalformedRequest)); } let data = - read_shared_buffer(shared_buffers, request.buffer, MAX_SOCKET_TRANSFER_SIZE)?; + copy_shared_buffer(shared_buffers, request.buffer, MAX_SOCKET_TRANSFER_SIZE)?; match litebox_broker_core::socket::send(process, request.handle, data, request.flags) .map_err(RequestFailure::from)? { @@ -1005,7 +959,7 @@ fn handle_socket_request( { return Err(RequestFailure::Abort(ErrorCode::MalformedRequest)); } - let data = read_shared_buffer(shared_buffers, request.buffer, MAX_UDP_DATAGRAM_SIZE)?; + let data = copy_shared_buffer(shared_buffers, request.buffer, MAX_UDP_DATAGRAM_SIZE)?; match litebox_broker_core::socket::send_to( process, request.handle, @@ -1182,7 +1136,7 @@ fn handle_pipe_request( })) } PipeRequest::Write(request) => { - let data = read_shared_buffer(shared_buffers, request.buffer, MAX_PIPE_TRANSFER_SIZE)?; + let data = copy_shared_buffer(shared_buffers, request.buffer, MAX_PIPE_TRANSFER_SIZE)?; litebox_broker_core::pipe::write(process, request.handle, &data) .map_err(RequestFailure::from) .and_then(|written| { @@ -1616,7 +1570,7 @@ mod tests { let retained = Mutex::new(None); assert!(matches!( - setup_connection_with_process( + setup_connection( broker, None, None, @@ -2864,6 +2818,7 @@ mod tests { control_channel, shared_buffers, test_readiness_sink(), + |_| false, send_shared_memory, )? { Ok(association) => association, diff --git a/litebox_broker_host/src/test_support.rs b/litebox_broker_host/src/test_support.rs index f79398c0cd..e3df5b9c6a 100644 --- a/litebox_broker_host/src/test_support.rs +++ b/litebox_broker_host/src/test_support.rs @@ -103,6 +103,7 @@ impl LocalSetupChannel for InProcessBrokerSetup { &mut host_setup, shared_buffers, readiness, + |_| false, |_| Ok(()), ) .expect("the in-process broker setup must succeed") diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index fb5a327bab..c84a2cd6a3 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -14,7 +14,7 @@ use std::sync::{ use std::time::{Duration, Instant}; use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; -use litebox_broker_host::{BrokerHostExtensionError, copy_shared_buffer}; +use litebox_broker_host::{RequestFailure, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ @@ -545,7 +545,7 @@ impl RunnerChildren { process: &BrokerProcess, operation: &BrokerOperation, shared_buffers: &SharedBufferPool, - ) -> Option> { + ) -> Option> { match operation { BrokerOperation::StartProcess(request) => Some( copy_shared_buffer( @@ -573,7 +573,7 @@ impl RunnerChildren { Ok(ProcessStartAcknowledgement::Failed(error)) => { Ok(BrokerResult::ProcessStartFailed(error)) } - Err(error) => Err(BrokerHostExtensionError::Abort(error)), + Err(error) => Err(RequestFailure::Abort(error)), }) } BrokerOperation::ProcessReady(request) => Some( @@ -1280,7 +1280,7 @@ impl Drop for WatchdogCompletion { } } -const fn process_extension_error(error: ErrorCode) -> BrokerHostExtensionError { +const fn process_extension_error(error: ErrorCode) -> RequestFailure { match error { ErrorCode::PolicyDenied | ErrorCode::UnknownObject @@ -1289,8 +1289,8 @@ const fn process_extension_error(error: ErrorCode) -> BrokerHostExtensionError { | ErrorCode::WouldBlock | ErrorCode::PeerClosed | ErrorCode::OutOfMemory - | ErrorCode::UnsupportedOperation => BrokerHostExtensionError::Respond(error), - _ => BrokerHostExtensionError::Abort(error), + | ErrorCode::UnsupportedOperation => RequestFailure::Respond(error), + _ => RequestFailure::Abort(error), } } diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index d549ea6eb0..3d8fc4169a 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -28,7 +28,7 @@ use std::time::{Duration, Instant}; use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{ BrokerHostAssociation, BrokerHostError, ConnectionTermination, ProcessStartupData, - setup_connection_with_process, + setup_connection, }; use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::error::ErrorCode; @@ -249,7 +249,7 @@ where let control_ring = ControlRing::new(control_memory) .map_err(|error| IoError::other(format!("failed to create control ring: {error:?}")))?; let readiness = Arc::new(ReadinessPublisherRuntime::new()); - let association = match setup_connection_with_process( + let association = match setup_connection( broker, process, startup, @@ -1069,6 +1069,7 @@ mod tests { &mut control, &shared_buffers, readiness.clone(), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), None)?; channel.send_memfd(control_ring.memory(), None) diff --git a/litebox_broker_userland/tests/notification_runtime.rs b/litebox_broker_userland/tests/notification_runtime.rs index 5ef7eed500..3a0632933f 100644 --- a/litebox_broker_userland/tests/notification_runtime.rs +++ b/litebox_broker_userland/tests/notification_runtime.rs @@ -64,6 +64,7 @@ fn spawn_host( &mut control, &shared_buffers, Arc::new(ReadinessPublisherRuntime::new()), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), None)?; channel.send_memfd(control_ring.memory(), None) @@ -157,6 +158,7 @@ fn host_serves_control_requests_and_notifications_over_shared_rings() { &mut control, &host_shared_buffers, Arc::new(ReadinessPublisherRuntime::new()), + |_| false, |channel| { channel.send_memfd(host_shared_buffers.memory(), None)?; channel.send_memfd(host_control_ring.memory(), None) diff --git a/litebox_runner_linux_userland/tests/run.rs b/litebox_runner_linux_userland/tests/run.rs index 5305174f3e..b40b2b8bf4 100644 --- a/litebox_runner_linux_userland/tests/run.rs +++ b/litebox_runner_linux_userland/tests/run.rs @@ -526,6 +526,7 @@ fn run_test_broker_connection( &mut channel, &shared_buffers, readiness.clone(), + |_| false, |channel| { channel.send_memfd(shared_buffers.memory(), Some(setup_deadline))?; channel.send_memfd(control_ring.memory(), Some(setup_deadline)) From f4019c4f5133f2b8e17835d00603916a1c1827cc Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 12:43:28 -0700 Subject: [PATCH 09/21] Unify runner startup negotiation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_host/src/lib.rs | 38 +++++---------- litebox_broker_local/src/lib.rs | 48 +------------------ litebox_broker_local_userland/src/lib.rs | 5 +- litebox_broker_local_userland/src/linux.rs | 21 ++------ litebox_broker_local_userland/src/windows.rs | 23 ++------- litebox_broker_protocol/src/process.rs | 15 ++++++ litebox_broker_userland/src/runner.rs | 23 ++++----- litebox_broker_userland/src/runtime.rs | 4 +- .../tests/userland_broker.rs | 40 +++++----------- litebox_runner_linux_userland/src/lib.rs | 44 +++++------------ litebox_runner_windows_userland/src/lib.rs | 37 ++++---------- 11 files changed, 83 insertions(+), 215 deletions(-) diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 786186a616..b18c859d10 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -44,8 +44,7 @@ use litebox_broker_protocol::pipe::{ CreatePipeResponse, MAX_PIPE_TRANSFER_SIZE, ReadPipeResponse, WritePipeResponse, }; use litebox_broker_protocol::process::{ - InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartup, + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessStartup, ProcessStartupData, }; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ @@ -62,7 +61,7 @@ use litebox_broker_protocol::stdio::{ IsTerminalStdioRequest, IsTerminalStdioResponse, MAX_STDIO_TRANSFER_SIZE, ReadStdioRequest, ReadStdioResponse, WriteStdioRequest, WriteStdioResponse, }; -use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, ObjectHandle, RequestId}; +use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId}; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use spin::mutex::SpinMutex; @@ -95,27 +94,6 @@ struct AssociationState { shared_buffer_usage: SharedBufferUsage, } -/// Failure classification for broker request handling. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RequestFailure { - /// Return an error result and keep serving the association. - Respond(ErrorCode), - /// Fail the association without publishing a response. - Abort(ErrorCode), -} - -/// Child startup data staged during broker negotiation. -pub struct ProcessStartupData { - /// Platform-defined format. - pub format: ProcessBootstrapFormat, - /// Version within the platform-defined format. - pub version: ProcessBootstrapVersion, - /// Opaque platform bytes. - pub payload: Vec, - /// Child-owned broker handles in the parent's inheritance-manifest order. - pub inherited_objects: Vec, -} - impl<'a, Memory: SharedMemory> BrokerHostAssociation<'a, Memory> { fn new( process: Arc, @@ -283,8 +261,7 @@ where version, buffer, }, - inherited_objects: InheritedProcessObjects::new(&inherited_objects) - .ok_or(BrokerHostError::Broker(ErrorCode::Internal))?, + inherited_objects, }) } None => None, @@ -386,6 +363,15 @@ where type RequestResult = core::result::Result; +/// Failure classification for broker request handling. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RequestFailure { + /// Return an error result and keep serving the association. + Respond(ErrorCode), + /// Fail the association without publishing a response. + Abort(ErrorCode), +} + impl From for RequestFailure { fn from(error: litebox_broker_core::BrokerError) -> Self { match error { diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index b4bd1b5eeb..4f5fa546ee 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -40,9 +40,7 @@ use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, }; -use litebox_broker_protocol::process::{ - InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, -}; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; use litebox_broker_protocol::{ @@ -71,19 +69,6 @@ pub struct BrokerNotifications { channel: Channel, } -/// Child startup data delivered during broker negotiation. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProcessStartupData { - /// Platform-defined format. - pub format: ProcessBootstrapFormat, - /// Version within the platform-defined format. - pub version: ProcessBootstrapVersion, - /// Opaque platform bytes. - pub payload: Vec, - /// Child-owned broker handles in the parent's inheritance-manifest order. - pub inherited_objects: InheritedProcessObjects, -} - impl BrokerLocal { fn new(channel: Channel, process_id: ProcessId, shared_memory: Arc) -> Self { let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) @@ -366,7 +351,6 @@ mod tests { use core::cell::{Cell, RefCell}; use core::convert::Infallible; use litebox_broker_protocol::message::{ReadinessNotification, StdioRequest, StdioResponse}; - use litebox_broker_protocol::process::{ProcessBootstrap, ProcessStartup}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use litebox_broker_protocol::stdio::{ @@ -414,36 +398,6 @@ mod tests { assert_eq!(local.process_id(), test_process_id()); } - #[test] - fn negotiate_returns_child_startup_data() { - let inherited_objects = InheritedProcessObjects::new(&[ObjectHandle(7)]).unwrap(); - let channel = FakeControlChannel::new( - Some(BrokerHandshakeResponse::Negotiated { - broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: test_process_id(), - startup: Some(ProcessStartup { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(3), - version: ProcessBootstrapVersion(4), - buffer: SharedBufferSequence::new(&[SharedBufferSlotIndex(0)], 5).unwrap(), - }, - inherited_objects, - }), - }), - None, - ); - - let (_local, startup, ()) = - BrokerLocal::negotiate(channel, |channel| Ok((channel, noop_shared_memory(), ()))) - .unwrap(); - let startup = startup.unwrap(); - - assert_eq!(startup.format, ProcessBootstrapFormat(3)); - assert_eq!(startup.version, ProcessBootstrapVersion(4)); - assert_eq!(startup.payload, [0; 5]); - assert_eq!(startup.inherited_objects, inherited_objects); - } - #[test] fn close_object_sends_close_object_request() { let handle = ObjectHandle(7); diff --git a/litebox_broker_local_userland/src/lib.rs b/litebox_broker_local_userland/src/lib.rs index ec30a0f860..90c69d2bf5 100644 --- a/litebox_broker_local_userland/src/lib.rs +++ b/litebox_broker_local_userland/src/lib.rs @@ -9,11 +9,10 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::{ - BrokerAssociationFailureCoordinator, BrokerConnection, connect, connect_child, - start_notification_receiver, + BrokerAssociationFailureCoordinator, BrokerConnection, connect, start_notification_receiver, }; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(all(windows, target_arch = "x86_64"))] -pub use windows::{BrokerConnection, connect, connect_child, start_notification_receiver}; +pub use windows::{BrokerConnection, connect, start_notification_receiver}; diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index 84b8f53cc9..a816212b1b 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -12,8 +12,9 @@ use std::{ }; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications, ProcessStartupData}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport_linux_userland::unix_socket::{ @@ -39,23 +40,7 @@ pub struct BrokerConnection { } /// Connects to and negotiates an association with a Linux-userland broker. -pub fn connect(control_socket_path: &Path) -> Result { - let (connection, startup) = connect_with_startup(control_socket_path)?; - if startup.is_some() { - anyhow::bail!("initial broker association returned child startup data"); - } - Ok(connection) -} - -/// Connects a child runner and receives its startup data. -pub fn connect_child(control_socket_path: &Path) -> Result<(BrokerConnection, ProcessStartupData)> { - let (connection, startup) = connect_with_startup(control_socket_path)?; - let startup = - startup.ok_or_else(|| anyhow::anyhow!("child broker association omitted startup data"))?; - Ok((connection, startup)) -} - -fn connect_with_startup( +pub fn connect( control_socket_path: &Path, ) -> Result<(BrokerConnection, Option)> { let setup_deadline = Instant::now() + SETUP_TIMEOUT; diff --git a/litebox_broker_local_userland/src/windows.rs b/litebox_broker_local_userland/src/windows.rs index e122be3506..bd7906b8ce 100644 --- a/litebox_broker_local_userland/src/windows.rs +++ b/litebox_broker_local_userland/src/windows.rs @@ -6,8 +6,9 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{Context as _, Result}; -use litebox_broker_local::{BrokerLocal, BrokerNotifications, ProcessStartupData}; +use litebox_broker_local::{BrokerLocal, BrokerNotifications}; use litebox_broker_protocol::message::BrokerNotification; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_POOL_SIZE; use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport_windows_userland::control_ring::{ @@ -26,25 +27,7 @@ pub struct BrokerConnection { } /// Connects to and negotiates an association with a Windows-userland broker. -pub fn connect(control_pipe: &OsStr) -> Result { - let (connection, startup) = connect_with_startup(control_pipe)?; - if startup.is_some() { - anyhow::bail!("initial broker association returned child startup data"); - } - Ok(connection) -} - -/// Connects a child runner and receives its startup data. -pub fn connect_child(control_pipe: &OsStr) -> Result<(BrokerConnection, ProcessStartupData)> { - let (connection, startup) = connect_with_startup(control_pipe)?; - let startup = - startup.ok_or_else(|| anyhow::anyhow!("child broker association omitted startup data"))?; - Ok((connection, startup)) -} - -fn connect_with_startup( - control_pipe: &OsStr, -) -> Result<(BrokerConnection, Option)> { +pub fn connect(control_pipe: &OsStr) -> Result<(BrokerConnection, Option)> { let deadline = Instant::now() + SETUP_TIMEOUT; let setup = WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(control_pipe, deadline) diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index b25952a41b..9b5b4fef97 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +use alloc::vec::Vec; + use crate::shared_buffer::SharedBufferSequence; use crate::{ObjectHandle, ProcessId, ThreadId}; @@ -82,6 +84,19 @@ pub struct ProcessStartup { pub inherited_objects: InheritedProcessObjects, } +/// Owned child startup data delivered during broker negotiation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProcessStartupData { + /// Platform-defined format. + pub format: ProcessBootstrapFormat, + /// Version within the platform-defined format. + pub version: ProcessBootstrapVersion, + /// Opaque platform bytes. + pub payload: Vec, + /// Child-owned broker handles in the parent's inheritance-manifest order. + pub inherited_objects: InheritedProcessObjects, +} + /// Starts one child process from an opaque platform bootstrap. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartProcessRequest { diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index c84a2cd6a3..eb7d7ec4fb 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -18,10 +18,10 @@ use litebox_broker_host::{RequestFailure, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ - MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartToken, - StartedProcess, + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessStartToken, StartedProcess, }; -use litebox_broker_protocol::{ObjectHandle, ProcessId, ThreadId}; +use litebox_broker_protocol::{ProcessId, ThreadId}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use crate::runtime::AssociationFailureCause; @@ -43,7 +43,6 @@ const PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT: Duration = Duration::fr const PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); const PROCESS_EXIT_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); -const CHILD_ARGUMENT: &str = "--child"; const MAX_PENDING_CHILD_STARTS: usize = crate::WORKER_COUNT - 1; const _: () = assert!(MAX_PENDING_CHILD_STARTS > 0); const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_WORKER_COUNT > MAX_PENDING_CHILD_STARTS); @@ -51,8 +50,8 @@ const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_QUEUE_CAPACITY >= MAX_PE /// Configuration for starting one out-of-process runner. /// -/// Dynamically started descendants use the same executable with the hidden -/// `--child` argument instead of the root arguments. +/// Dynamically started descendants use the same executable without the root +/// arguments and derive their role from broker startup data. #[derive(Clone)] pub struct RunnerConfig { executable: PathBuf, @@ -96,7 +95,7 @@ impl RunnerConfig { fn child(&self) -> Self { Self { executable: self.executable.clone(), - arguments: vec![OsString::from(CHILD_ARGUMENT)], + arguments: Vec::new(), proxy_url: self.proxy_url.clone(), } } @@ -410,7 +409,7 @@ pub(crate) struct RunnerChildren { pub(crate) struct ChildRunner { pub(crate) process: Arc, pub(crate) launch: Arc, - pub(crate) inherited_objects: Vec, + pub(crate) inherited_objects: InheritedProcessObjects, pub(crate) format: ProcessBootstrapFormat, pub(crate) version: ProcessBootstrapVersion, pub(crate) bootstrap: Vec, @@ -559,7 +558,7 @@ impl RunnerChildren { request.bootstrap.format, request.bootstrap.version, bootstrap, - request.inherited_objects.as_slice(), + request.inherited_objects, ) .map_err(process_extension_error) }) @@ -640,14 +639,16 @@ impl RunnerChildren { format: ProcessBootstrapFormat, version: ProcessBootstrapVersion, bootstrap: Vec, - requested_inherited_objects: &[litebox_broker_protocol::ObjectHandle], + requested_inherited_objects: InheritedProcessObjects, ) -> Result { if !parent.is_running() { return Err(ErrorCode::ProtocolState); } let (process, inherited_objects) = parent - .create_child(requested_inherited_objects) + .create_child(requested_inherited_objects.as_slice()) .map_err(ErrorCode::from)?; + let inherited_objects = InheritedProcessObjects::new(&inherited_objects) + .expect("child handle count must match the bounded inheritance request"); let child_id = process.id(); let launch = Arc::new(ChildLaunch { parent_id: parent.id(), diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 3d8fc4169a..daad0cd424 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -27,12 +27,12 @@ use std::time::{Duration, Instant}; use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{ - BrokerHostAssociation, BrokerHostError, ConnectionTermination, ProcessStartupData, - setup_connection, + BrokerHostAssociation, BrokerHostError, ConnectionTermination, setup_connection, }; use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest}; +use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 34a93a05d7..70ae7867ed 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -12,7 +12,7 @@ use std::time::{Duration, Instant}; use litebox_broker_local::{BrokerLocal, BrokerLocalError}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::process::{ - InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartupData, }; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{ @@ -20,13 +20,14 @@ use litebox_broker_protocol::shared_buffer::{ }; use litebox_broker_protocol::socket::{ReceiveFromFlags, SendFlags, SocketConnectionStatus}; use litebox_broker_transport::control_ring::ControlRing; -use litebox_broker_transport_linux_userland::unix_socket::UnixStreamLocalSetupChannel; +use litebox_broker_transport_linux_userland::unix_socket::{ + UnixControlRingLocalCallChannel, UnixStreamLocalSetupChannel, +}; const RUNNER_ARGUMENT: &str = "broker-userland-test-runner"; const NETWORK_RUNNER_ARGUMENT: &str = "broker-userland-network-test-runner"; const CHILD_START_RUNNER_ARGUMENT: &str = "broker-userland-child-start-runner"; const CHILD_CANCEL_RUNNER_ARGUMENT: &str = "broker-userland-child-cancel-runner"; -const CHILD_ARGUMENT: &str = "--child"; const TEST_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x7465_7374); const FAILING_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x6661_696c); const BROKER_PROCESS_TIMEOUT: Duration = Duration::from_secs(30); @@ -145,10 +146,6 @@ fn run_fake_runner(args: &[OsString]) { ); let control_socket_path = args.get(2).unwrap(); - if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_ARGUMENT) { - run_fake_child(Path::new(control_socket_path)); - return; - } let setup_channel = connect_control_with_retry(Path::new(control_socket_path)).unwrap(); let (local, startup, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { let shared_memory = setup.receive_memfd( @@ -167,7 +164,10 @@ fn run_fake_runner(args: &[OsString]) { Ok((call_channel, Arc::new(shared_memory), ())) }) .unwrap(); - assert!(startup.is_none()); + if let Some(bootstrap) = startup { + run_fake_child(local, bootstrap); + return; + } let local = Arc::new(local); if args.get(3).and_then(|argument| argument.to_str()) == Some(NETWORK_RUNNER_ARGUMENT) { @@ -373,26 +373,10 @@ fn run_fake_runner(args: &[OsString]) { drop(local); } -fn run_fake_child(control_socket_path: &Path) { - let setup_channel = connect_control_with_retry(control_socket_path).unwrap(); - let (local, bootstrap, ()) = BrokerLocal::negotiate(setup_channel, |mut setup| { - let shared_memory = setup.receive_memfd( - SHARED_BUFFER_POOL_SIZE, - Some(Instant::now() + Duration::from_secs(5)), - )?; - let control_memory = - setup.receive_control_ring(Some(Instant::now() + Duration::from_secs(5)))?; - let control_ring = ControlRing::new(control_memory).map_err(|error| { - std::io::Error::new( - ErrorKind::InvalidData, - format!("invalid test control ring: {error:?}"), - ) - })?; - let (call_channel, _notifications, _shutdown) = setup.into_active(control_ring, || {})?; - Ok((call_channel, Arc::new(shared_memory), ())) - }) - .unwrap(); - let bootstrap = bootstrap.expect("child negotiation must include startup data"); +fn run_fake_child( + local: BrokerLocal, + bootstrap: ProcessStartupData, +) { if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { local .report_process_start_failure(ErrorCode::UnsupportedOperation) diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 92c7a8d58e..99d53e7851 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -31,11 +31,7 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/usr/bin/python3 --version`). /// /// The program path must be absolute and refer to a file in the broker-owned file system. - #[arg( - required_unless_present = "child", - trailing_var_arg = true, - value_hint = clap::ValueHint::CommandWithArguments - )] + #[arg(trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times) #[arg(long = "env")] @@ -46,14 +42,6 @@ pub struct CliArgs { /// Allow using unstable options #[arg(short = 'Z', long = "unstable")] pub unstable: bool, - /// Start as a dynamically launched child. - #[arg( - long = "child", - hide = true, - requires_all = ["unstable", "broker_control_channel"], - help_heading = "Unstable Options" - )] - pub child: bool, /// Broker-supplied Unix socket path for the local control channel. #[arg( long = "broker-control-channel", @@ -99,15 +87,15 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - if cli_args.child { - if !cli_args.program_and_arguments.is_empty() { - return Err(anyhow!("--child does not accept a root program argument")); - } - let control_socket_path = cli_args - .broker_control_channel - .as_deref() - .context("--child requires --broker-control-channel")?; - let (connection, bootstrap) = broker::connect_child(control_socket_path)?; + let control_socket_path = cli_args + .broker_control_channel + .as_deref() + .context("file operations require --broker-control-channel")?; + let (connection, startup) = + litebox_platform_linux_userland::with_guest_signals_blocked(|| { + broker::connect(control_socket_path) + })?; + if let Some(bootstrap) = startup { connection .local .report_process_start_failure(ErrorCode::UnsupportedOperation)?; @@ -131,19 +119,13 @@ pub fn run(cli_args: CliArgs) -> Result { let mut broker_positional_io_fds = Vec::new(); let mut broker_shutdown_fds = Vec::new(); - let control_socket_path = cli_args - .broker_control_channel - .as_deref() - .context("file operations require --broker-control-channel")?; let broker::BrokerConnection { local: broker_local, notifications: broker_notifications, coordinator: broker_association_coordinator, positional_io_fds, shutdown_fd, - } = litebox_platform_linux_userland::with_guest_signals_blocked(|| { - broker::connect(control_socket_path) - })?; + } = connection; let process_id = i32::try_from(broker_local.process_id().0) .context("process ID does not fit Linux pid_t")?; broker_positional_io_fds.extend(positional_io_fds); @@ -258,17 +240,15 @@ mod tests { } #[test] - fn child_does_not_require_a_root_program() { + fn broker_connection_does_not_require_a_root_program() { let args = CliArgs::try_parse_from([ "runner", "--unstable", "--broker-control-channel", "/tmp/broker.sock", - "--child", ]) .unwrap(); - assert!(args.child); assert!(args.program_and_arguments.is_empty()); } diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 327fa40c0a..9d5a0f8370 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -21,11 +21,7 @@ pub struct CliArgs { /// The program and arguments passed to it (e.g., `/app/program.exe --help`). /// /// The program path refers to a path inside the broker-owned file system. - #[arg( - required_unless_present = "child", - trailing_var_arg = true, - value_hint = clap::ValueHint::CommandWithArguments - )] + #[arg(trailing_var_arg = true, value_hint = clap::ValueHint::CommandWithArguments)] pub program_and_arguments: Vec, /// Environment variables passed to the program (`K=V` pairs; can be invoked multiple times). #[arg(long = "env")] @@ -36,14 +32,6 @@ pub struct CliArgs { /// Allow using unstable options. #[arg(short = 'Z', long = "unstable")] pub unstable: bool, - /// Start as a dynamically launched child. - #[arg( - long = "child", - hide = true, - requires_all = ["unstable", "broker_control_channel"], - help_heading = "Unstable Options" - )] - pub child: bool, /// Broker-supplied Windows named-pipe path for the local control channel. #[arg( long = "broker-control-channel", @@ -67,15 +55,12 @@ pub fn run(cli_args: CliArgs) -> Result { ) .init(); - if cli_args.child { - if !cli_args.program_and_arguments.is_empty() { - anyhow::bail!("--child does not accept a root program argument"); - } - let control_pipe = cli_args - .broker_control_channel - .as_deref() - .context("--child requires --broker-control-channel")?; - let (connection, bootstrap) = broker::connect_child(control_pipe)?; + let control_pipe = cli_args + .broker_control_channel + .as_deref() + .context("file operations require --broker-control-channel")?; + let (connection, startup) = broker::connect(control_pipe)?; + if let Some(bootstrap) = startup { connection .local .report_process_start_failure(ErrorCode::UnsupportedOperation)?; @@ -88,14 +73,10 @@ pub fn run(cli_args: CliArgs) -> Result { let platform = WindowsUserland::new(); WindowsUserland::set_guest_tls_mode(GuestTlsMode::Windows); - let control_pipe = cli_args - .broker_control_channel - .as_deref() - .context("file operations require --broker-control-channel")?; let broker::BrokerConnection { local, notifications, - } = broker::connect(control_pipe)?; + } = connection; let process_id = local.process_id().0 as usize; let litebox = litebox::LiteBox::new_with_broker_local(platform, local); let initial_thread = litebox.create_thread()?; @@ -115,7 +96,7 @@ pub fn run(cli_args: CliArgs) -> Result { let (program_path, program_args) = cli_args .program_and_arguments .split_first() - .context("program path missing — clap should have required at least one argument")?; + .context("program path missing")?; let shim = shim_builder.build(); let argv = std::iter::once(program_path.as_str()) From 1ca51c4db18bbc0f4a2d85df27c485d57280ca38 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 14:48:03 -0700 Subject: [PATCH 10/21] Simplify startup protocol and pending calls Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_host/src/lib.rs | 14 ++- litebox_broker_local/src/lib.rs | 10 +- litebox_broker_local/src/process.rs | 19 ++-- litebox_broker_protocol/src/lib.rs | 2 +- litebox_broker_protocol/src/message.rs | 18 ++-- litebox_broker_protocol/src/process.rs | 29 +----- litebox_broker_protocol/src/wire.rs | 98 +++++++------------ litebox_broker_transport/src/pending_calls.rs | 61 ++++++------ .../src/unix_socket/local.rs | 20 ++-- .../src/local.rs | 4 +- .../src/named_pipe.rs | 16 +-- litebox_broker_userland/src/runner.rs | 32 +++--- 12 files changed, 130 insertions(+), 193 deletions(-) diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index b18c859d10..acd61382cf 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -44,7 +44,7 @@ use litebox_broker_protocol::pipe::{ CreatePipeResponse, MAX_PIPE_TRANSFER_SIZE, ReadPipeResponse, WritePipeResponse, }; use litebox_broker_protocol::process::{ - MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessStartup, ProcessStartupData, + MAX_PROCESS_BOOTSTRAP_SIZE, ProcessStartupData, ProcessStartupDescriptor, }; use litebox_broker_protocol::random::MAX_RANDOM_TRANSFER_SIZE; use litebox_broker_protocol::shared_buffer::{ @@ -255,12 +255,10 @@ where .map_err(|_| BrokerHostError::Broker(ErrorCode::Internal))?; write_shared_buffer(shared_buffers, buffer, &payload, MAX_PROCESS_BOOTSTRAP_SIZE) .map_err(|error| BrokerHostError::Broker(error.into()))?; - Some(ProcessStartup { - bootstrap: ProcessBootstrap { - format, - version, - buffer, - }, + Some(ProcessStartupDescriptor { + format, + version, + buffer, inherited_objects, }) } @@ -517,7 +515,7 @@ fn handle_request( } BrokerOperation::StartProcess(_) | BrokerOperation::AcknowledgeProcessStart(_) - | BrokerOperation::ProcessReady(_) + | BrokerOperation::ReportProcessReady(_) | BrokerOperation::ReportProcessStartFailure(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) } diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 4f5fa546ee..4f48ff491f 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -134,13 +134,13 @@ impl BrokerLocal { Some(startup) => { let mut payload = Vec::new(); payload - .try_reserve_exact(startup.bootstrap.buffer.length() as usize) + .try_reserve_exact(startup.buffer.length() as usize) .map_err(|_| BrokerLocalError::Broker(ErrorCode::OutOfMemory))?; - payload.resize(startup.bootstrap.buffer.length() as usize, 0); - local.read_shared_buffer(startup.bootstrap.buffer, &mut payload); + payload.resize(startup.buffer.length() as usize, 0); + local.read_shared_buffer(startup.buffer, &mut payload); Some(ProcessStartupData { - format: startup.bootstrap.format, - version: startup.bootstrap.version, + format: startup.format, + version: startup.version, payload, inherited_objects: startup.inherited_objects, }) diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs index 926a51676a..3f66942e2f 100644 --- a/litebox_broker_local/src/process.rs +++ b/litebox_broker_local/src/process.rs @@ -5,9 +5,8 @@ use litebox_broker_protocol::ThreadId; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ - InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrap, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessReadyRequest, ProcessStartToken, StartProcessRequest, - StartedProcess, + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessStartToken, ProcessStartupDescriptor, StartedProcess, }; use litebox_broker_protocol::shared_buffer::SharedBufferSequence; use litebox_broker_transport::channel::LocalCallChannel; @@ -37,12 +36,10 @@ impl BrokerLocal { return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); } self.write_shared_buffer(buffer, bootstrap); - match self.request(BrokerOperation::StartProcess(StartProcessRequest { - bootstrap: ProcessBootstrap { - format, - version, - buffer, - }, + match self.request(BrokerOperation::StartProcess(ProcessStartupDescriptor { + format, + version, + buffer, inherited_objects, }))? { BrokerResult::ProcessStarted(started) => Ok(started), @@ -77,9 +74,7 @@ impl BrokerLocal { /// /// Panics if the broker returns a response for another operation. pub fn process_ready(&self, initial_thread_id: Option) -> Result<(), Channel::Error> { - match self.request(BrokerOperation::ProcessReady(ProcessReadyRequest { - initial_thread_id, - }))? { + match self.request(BrokerOperation::ReportProcessReady(initial_thread_id))? { BrokerResult::ProcessReady => Ok(()), BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), response => panic!("broker returned unexpected process-ready response: {response:?}"), diff --git a/litebox_broker_protocol/src/lib.rs b/litebox_broker_protocol/src/lib.rs index ecb6bfdf2f..8806d1c297 100644 --- a/litebox_broker_protocol/src/lib.rs +++ b/litebox_broker_protocol/src/lib.rs @@ -61,4 +61,4 @@ pub struct RequestId(pub u64); pub struct ProtocolVersion(pub u16); /// Current broker protocol version. -pub const BROKER_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion(2); +pub const BROKER_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion(1); diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index 01b0f85039..246a1af44b 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -17,10 +17,7 @@ use crate::pipe::{ CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; -use crate::process::{ - ProcessBootstrap, ProcessReadyRequest, ProcessStartToken, ProcessStartup, StartProcessRequest, - StartedProcess, -}; +use crate::process::{ProcessStartToken, ProcessStartupDescriptor, StartedProcess}; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; use crate::socket::{ @@ -69,11 +66,11 @@ pub enum BrokerOperation { /// File request family. File(FileRequest), /// Start one child process from an opaque platform bootstrap. - StartProcess(StartProcessRequest), + StartProcess(ProcessStartupDescriptor), /// Commit a child process after its start result reaches the parent. AcknowledgeProcessStart(ProcessStartToken), /// Report that this child process is ready to begin guest execution. - ProcessReady(ProcessReadyRequest), + ReportProcessReady(Option), /// Report that this child rejected its startup data before becoming ready. ReportProcessStartFailure(ErrorCode), } @@ -110,10 +107,7 @@ impl BrokerOperation { | FileRequest::Mkdir(MkdirFileRequest { path: buffer, .. }) | FileRequest::Rmdir(RmdirFileRequest { path: buffer, .. }), ) - | Self::StartProcess(StartProcessRequest { - bootstrap: ProcessBootstrap { buffer, .. }, - .. - }) => Some(*buffer), + | Self::StartProcess(ProcessStartupDescriptor { buffer, .. }) => Some(*buffer), Self::CreateThread | Self::ExitThread(_) | Self::CloseObject(_) @@ -136,7 +130,7 @@ impl BrokerOperation { FileRequest::Seek(_) | FileRequest::Truncate(_) | FileRequest::HandleStatus(_), ) | Self::AcknowledgeProcessStart(_) - | Self::ProcessReady(_) + | Self::ReportProcessReady(_) | Self::ReportProcessStartFailure(_) => None, } } @@ -164,7 +158,7 @@ pub enum BrokerHandshakeResponse { /// Assigned process ID. process_id: ProcessId, /// Child startup data, absent for the initial process. - startup: Option, + startup: Option, }, /// Negotiation failed because the requested version is unsupported. /// diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index 9b5b4fef97..f97464d7e9 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -64,23 +64,16 @@ impl InheritedProcessObjects { } } -/// Opaque bootstrap descriptor supplied when starting a process. +/// Child startup descriptor transported through the broker protocol. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ProcessBootstrap { +pub struct ProcessStartupDescriptor { /// Platform-defined format. pub format: ProcessBootstrapFormat, /// Version within the platform-defined format. pub version: ProcessBootstrapVersion, /// Operation-scoped shared-buffer sequence containing the bootstrap bytes. pub buffer: SharedBufferSequence, -} - -/// Child startup data delivered during broker negotiation. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ProcessStartup { - /// Opaque platform bootstrap staged in the child's shared-buffer pool. - pub bootstrap: ProcessBootstrap, - /// Child-owned broker handles in the parent's inheritance-manifest order. + /// Broker handles inherited in manifest order. pub inherited_objects: InheritedProcessObjects, } @@ -97,15 +90,6 @@ pub struct ProcessStartupData { pub inherited_objects: InheritedProcessObjects, } -/// Starts one child process from an opaque platform bootstrap. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct StartProcessRequest { - /// Opaque bootstrap staged in the parent's shared-buffer pool. - pub bootstrap: ProcessBootstrap, - /// Parent-owned broker handles inherited in manifest order. - pub inherited_objects: InheritedProcessObjects, -} - /// Reports a materialized child that is ready for parent acknowledgement. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartedProcess { @@ -116,10 +100,3 @@ pub struct StartedProcess { /// Broker-assigned initial thread ID when it differs from the process ID. pub initial_thread_id: Option, } - -/// Reports that a child finished restoring its initial state. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ProcessReadyRequest { - /// Broker-assigned initial thread ID when it differs from the process ID. - pub initial_thread_id: Option, -} diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 1cbfbc7bab..01aa2946d1 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -23,9 +23,8 @@ use crate::message::{ BrokerRequest, BrokerResponse, BrokerResult, ReadinessNotification, }; use crate::process::{ - InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrap, - ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessReadyRequest, ProcessStartToken, - ProcessStartup, StartProcessRequest, StartedProcess, + InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessStartToken, ProcessStartupDescriptor, StartedProcess, }; use crate::readiness::ReadinessFlags; @@ -201,13 +200,10 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); fs::encode_fs_request(&mut encoder, request); } - BrokerOperation::StartProcess(StartProcessRequest { - bootstrap: - ProcessBootstrap { - format, - version, - buffer, - }, + BrokerOperation::StartProcess(ProcessStartupDescriptor { + format, + version, + buffer, inherited_objects, }) => { encoder.u8(REQUEST_TAG_START_PROCESS); @@ -222,7 +218,7 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); encoder.u64(token.0); } - BrokerOperation::ProcessReady(ProcessReadyRequest { initial_thread_id }) => { + BrokerOperation::ReportProcessReady(initial_thread_id) => { encoder.u8(REQUEST_TAG_PROCESS_READY); encoder.request_id(request_id); encode_optional_thread_id(&mut encoder, initial_thread_id); @@ -270,20 +266,18 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_FILL_RANDOM => BrokerOperation::FillRandom(decoder.shared_buffer_sequence()?), REQUEST_TAG_STDIO => BrokerOperation::Stdio(stdio::decode_stdio_request(&mut decoder)?), REQUEST_TAG_FILE => BrokerOperation::File(fs::decode_fs_request(&mut decoder)?), - REQUEST_TAG_START_PROCESS => BrokerOperation::StartProcess(StartProcessRequest { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(decoder.u32()?), - version: ProcessBootstrapVersion(decoder.u16()?), - buffer: decoder.shared_buffer_sequence()?, - }, + REQUEST_TAG_START_PROCESS => BrokerOperation::StartProcess(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, inherited_objects: decode_inherited_objects(&mut decoder)?, }), REQUEST_TAG_ACKNOWLEDGE_PROCESS_START => { BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(decoder.u64()?)) } - REQUEST_TAG_PROCESS_READY => BrokerOperation::ProcessReady(ProcessReadyRequest { - initial_thread_id: decode_optional_thread_id(&mut decoder)?, - }), + REQUEST_TAG_PROCESS_READY => { + BrokerOperation::ReportProcessReady(decode_optional_thread_id(&mut decoder)?) + } REQUEST_TAG_REPORT_PROCESS_START_FAILURE => { BrokerOperation::ReportProcessStartFailure(decode_error_code(&mut decoder)?) } @@ -312,13 +306,10 @@ pub fn encode_handshake_response(response: BrokerHandshakeResponse) -> Vec { encoder.protocol_version(broker_protocol_version); encoder.process_id(process_id); match startup { - Some(ProcessStartup { - bootstrap: - ProcessBootstrap { - format, - version, - buffer, - }, + Some(ProcessStartupDescriptor { + format, + version, + buffer, inherited_objects, }) => { encoder.u8(1); @@ -354,12 +345,10 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result None, - 1 => Some(ProcessStartup { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(decoder.u32()?), - version: ProcessBootstrapVersion(decoder.u16()?), - buffer: decoder.shared_buffer_sequence()?, - }, + 1 => Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, inherited_objects: decode_inherited_objects(&mut decoder)?, }), _ => return Err(WireError::InvalidTag), @@ -665,9 +654,8 @@ mod tests { WritePipeResponse, }; use crate::process::{ - InheritedProcessObjects, ProcessBootstrap, ProcessBootstrapFormat, ProcessBootstrapVersion, - ProcessReadyRequest, ProcessStartToken, ProcessStartup, StartProcessRequest, - StartedProcess, + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, + ProcessStartToken, ProcessStartupDescriptor, StartedProcess, }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ @@ -1015,12 +1003,10 @@ mod tests { name: TcpOptionName::KeepAlive, })), BrokerOperation::Socket(SocketRequest::Status(SocketStatusRequest { handle })), - BrokerOperation::StartProcess(StartProcessRequest { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(u32::MAX), - version: ProcessBootstrapVersion(u16::MAX), - buffer: largest_sequence, - }, + BrokerOperation::StartProcess(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(u32::MAX), + version: ProcessBootstrapVersion(u16::MAX), + buffer: largest_sequence, inherited_objects: InheritedProcessObjects::new(&[ ObjectHandle(1), ObjectHandle(2), @@ -1030,12 +1016,8 @@ mod tests { .unwrap(), }), BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(u64::MAX)), - BrokerOperation::ProcessReady(ProcessReadyRequest { - initial_thread_id: None, - }), - BrokerOperation::ProcessReady(ProcessReadyRequest { - initial_thread_id: Some(thread_id(19)), - }), + BrokerOperation::ReportProcessReady(None), + BrokerOperation::ReportProcessReady(Some(thread_id(19))), BrokerOperation::ReportProcessStartFailure(ErrorCode::UnsupportedOperation), ]; let mut maximum_encoded_size = 0; @@ -1215,12 +1197,10 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(7), - startup: Some(ProcessStartup { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(0x7465_7374), - version: ProcessBootstrapVersion(1), - buffer: sequence(0, 37), - }, + startup: Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(0x7465_7374), + version: ProcessBootstrapVersion(1), + buffer: sequence(0, 37), inherited_objects: InheritedProcessObjects::new(&[ ObjectHandle(5), ObjectHandle(6), @@ -1932,12 +1912,10 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(2), - startup: Some(ProcessStartup { - bootstrap: ProcessBootstrap { - format: ProcessBootstrapFormat(3), - version: ProcessBootstrapVersion(4), - buffer: sequence(0, 5), - }, + startup: Some(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(3), + version: ProcessBootstrapVersion(4), + buffer: sequence(0, 5), inherited_objects: InheritedProcessObjects::EMPTY, }), }, diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs index 23e7476968..90b5c1f69c 100644 --- a/litebox_broker_transport/src/pending_calls.rs +++ b/litebox_broker_transport/src/pending_calls.rs @@ -13,10 +13,10 @@ use litebox_broker_protocol::message::BrokerResponse; /// Maximum number of active calls waiting for broker responses. pub const MAX_PENDING_CALLS: usize = 64; -/// Pending-call capacity reserved for lifecycle-control operations. -pub const RESERVED_LIFECYCLE_PENDING_CALLS: usize = 8; -/// Maximum active ordinary calls after preserving lifecycle-control capacity. -pub const MAX_ORDINARY_PENDING_CALLS: usize = MAX_PENDING_CALLS - RESERVED_LIFECYCLE_PENDING_CALLS; +/// Pending-call capacity unavailable to ordinary operations. +pub const RESERVED_PENDING_CALL_CAPACITY: usize = 8; +/// Maximum active ordinary calls after preserving reserved capacity. +pub const MAX_ORDINARY_PENDING_CALLS: usize = MAX_PENDING_CALLS - RESERVED_PENDING_CALL_CAPACITY; /// A mutex usable by [`PendingCalls`]. pub trait PendingCallsMutex { @@ -81,30 +81,27 @@ pub enum PendingCallsError { /// Concurrent registry of requests awaiting broker responses. pub struct PendingCalls { - state: Sync::Mutex>, - capacity_available: Sync::Condvar>, + state: Sync::Mutex>, + capacity_available: Sync::Condvar>, } -struct PendingCallsState { - calls: BTreeMap>, +struct PendingCallsInner { + calls: BTreeMap>>, ordinary_calls: usize, failure: Option>, } -struct RegisteredPendingCall { - call: Arc>, - ordinary: bool, -} - /// Completion state for one request awaiting a broker response. pub struct PendingCall { + counts_against_ordinary_limit: bool, result: Sync::Mutex>>>, result_ready: Sync::Condvar>>>, } impl PendingCall { - fn new() -> Self { + fn new(counts_against_ordinary_limit: bool) -> Self { Self { + counts_against_ordinary_limit, result: Sync::mutex(None), result_ready: Sync::condvar(), } @@ -133,7 +130,7 @@ impl PendingCalls { /// Creates an empty live pending-call registry. pub fn new() -> Self { Self { - state: Sync::mutex(PendingCallsState { + state: Sync::mutex(PendingCallsInner { calls: BTreeMap::new(), ordinary_calls: 0, failure: None, @@ -147,26 +144,27 @@ impl PendingCalls { &self, request_id: RequestId, ) -> Result>, PendingCallsError> { - self.register_with_class(request_id, true) + self.register_inner(request_id, true) } - /// Registers lifecycle-control work using capacity ordinary calls cannot consume. - pub fn register_lifecycle( + /// Registers work using capacity ordinary calls cannot consume. + pub fn register_with_reserved_capacity( &self, request_id: RequestId, ) -> Result>, PendingCallsError> { - self.register_with_class(request_id, false) + self.register_inner(request_id, false) } - fn register_with_class( + fn register_inner( &self, request_id: RequestId, - ordinary: bool, + counts_against_ordinary_limit: bool, ) -> Result>, PendingCallsError> { - let pending_call = Arc::new(PendingCall::new()); + let pending_call = Arc::new(PendingCall::new(counts_against_ordinary_limit)); let mut state = self.state.lock(); while (state.calls.len() >= MAX_PENDING_CALLS - || (ordinary && state.ordinary_calls >= MAX_ORDINARY_PENDING_CALLS)) + || (counts_against_ordinary_limit + && state.ordinary_calls >= MAX_ORDINARY_PENDING_CALLS)) && state.failure.is_none() { state = self.capacity_available.wait(state); @@ -176,11 +174,8 @@ impl PendingCalls { } match state.calls.entry(request_id) { Entry::Vacant(entry) => { - entry.insert(RegisteredPendingCall { - call: Arc::clone(&pending_call), - ordinary, - }); - if ordinary { + entry.insert(Arc::clone(&pending_call)); + if counts_against_ordinary_limit { state.ordinary_calls += 1; } } @@ -201,17 +196,17 @@ impl PendingCalls { if let Some(error) = state.failure.as_ref() { return Err(PendingCallsError::AssociationFailed(Arc::clone(error))); } - let Some(registered) = state.calls.remove(&response.request_id) else { + let Some(pending_call) = state.calls.remove(&response.request_id) else { return Err(PendingCallsError::UnknownResponseId); }; - if registered.ordinary { + if pending_call.counts_against_ordinary_limit { state.ordinary_calls = state .ordinary_calls .checked_sub(1) .expect("ordinary pending-call count must remain balanced"); } self.capacity_available.notify_all(); - registered.call + pending_call }; pending_call.resolve(Ok(response)); Ok(()) @@ -232,8 +227,8 @@ impl PendingCalls { self.capacity_available.notify_all(); pending_calls }; - for registered in pending_calls.into_values() { - registered.call.resolve(Err(Arc::clone(&error))); + for pending_call in pending_calls.into_values() { + pending_call.resolve(Err(Arc::clone(&error))); } true } diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index e7b4f90c04..d85ae82e3e 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -326,7 +326,9 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { BrokerOperation::AcknowledgeProcessStart(_) | BrokerOperation::ReportProcessStartFailure(_) ) { - association.pending_calls.register_lifecycle(request_id) + association + .pending_calls + .register_with_reserved_capacity(request_id) } else { association.pending_calls.register(request_id) } @@ -773,9 +775,9 @@ mod control_ring_tests { } #[test] - fn pending_capacity_reserves_lifecycle_calls() { + fn pending_capacity_preserves_process_start_calls() { use litebox_broker_transport::pending_calls::{ - MAX_ORDINARY_PENDING_CALLS, RESERVED_LIFECYCLE_PENDING_CALLS, + MAX_ORDINARY_PENDING_CALLS, RESERVED_PENDING_CALL_CAPACITY, }; let (channel, shutdown, mut responses, mut requests, _peer) = activate_local(|| {}); @@ -791,12 +793,12 @@ mod control_ring_tests { }) }) .collect::>(); - callers.extend((0..RESERVED_LIFECYCLE_PENDING_CALLS).map(|index| { - let lifecycle_channel = Arc::clone(&channel); - let lifecycle_start = Arc::clone(&start); + callers.extend((0..RESERVED_PENDING_CALL_CAPACITY).map(|index| { + let reserved_channel = Arc::clone(&channel); + let reserved_start = Arc::clone(&start); thread::spawn(move || { - lifecycle_start.wait(); - lifecycle_channel.call(BrokerRequest { + reserved_start.wait(); + reserved_channel.call(BrokerRequest { request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), operation: BrokerOperation::AcknowledgeProcessStart( litebox_broker_protocol::process::ProcessStartToken(index as u64), @@ -822,7 +824,7 @@ mod control_ring_tests { BrokerOperation::AcknowledgeProcessStart(_) )) .count(), - RESERVED_LIFECYCLE_PENDING_CALLS + RESERVED_PENDING_CALL_CAPACITY ); let released_request = published .iter() diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 6c33ebd523..71712358fe 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -235,7 +235,9 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { BrokerOperation::AcknowledgeProcessStart(_) | BrokerOperation::ReportProcessStartFailure(_) ) { - association.pending_calls.register_lifecycle(request_id) + association + .pending_calls + .register_with_reserved_capacity(request_id) } else { association.pending_calls.register(request_id) } diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index e863847184..bca8816a4a 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -474,9 +474,9 @@ mod tests { } #[test] - fn pending_capacity_reserves_lifecycle_calls() { + fn pending_capacity_preserves_process_start_calls() { use litebox_broker_transport::pending_calls::{ - MAX_ORDINARY_PENDING_CALLS, RESERVED_LIFECYCLE_PENDING_CALLS, + MAX_ORDINARY_PENDING_CALLS, RESERVED_PENDING_CALL_CAPACITY, }; let control_name = pipe_name("pending-capacity-control"); @@ -517,7 +517,7 @@ mod tests { BrokerOperation::AcknowledgeProcessStart(_) )) .count(), - RESERVED_LIFECYCLE_PENDING_CALLS + RESERVED_PENDING_CALL_CAPACITY ); let released_request = published .iter() @@ -568,12 +568,12 @@ mod tests { }) }) .collect::>(); - callers.extend((0..RESERVED_LIFECYCLE_PENDING_CALLS).map(|index| { - let lifecycle_calls = Arc::clone(&calls); - let lifecycle_start = Arc::clone(&start); + callers.extend((0..RESERVED_PENDING_CALL_CAPACITY).map(|index| { + let reserved_calls = Arc::clone(&calls); + let reserved_start = Arc::clone(&start); std::thread::spawn(move || { - lifecycle_start.wait(); - lifecycle_calls.call(BrokerRequest { + reserved_start.wait(); + reserved_calls.call(BrokerRequest { request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), operation: BrokerOperation::AcknowledgeProcessStart( litebox_broker_protocol::process::ProcessStartToken(index as u64), diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index eb7d7ec4fb..52a00eec5e 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -547,22 +547,18 @@ impl RunnerChildren { ) -> Option> { match operation { BrokerOperation::StartProcess(request) => Some( - copy_shared_buffer( - shared_buffers, - request.bootstrap.buffer, - MAX_PROCESS_BOOTSTRAP_SIZE, - ) - .and_then(|bootstrap| { - self.start_process( - process, - request.bootstrap.format, - request.bootstrap.version, - bootstrap, - request.inherited_objects, - ) - .map_err(process_extension_error) - }) - .map(BrokerResult::ProcessStarted), + copy_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) + .and_then(|bootstrap| { + self.start_process( + process, + request.format, + request.version, + bootstrap, + request.inherited_objects, + ) + .map_err(process_extension_error) + }) + .map(BrokerResult::ProcessStarted), ), BrokerOperation::AcknowledgeProcessStart(token) => { Some(match self.resolve_acknowledgement(process.id(), *token) { @@ -575,8 +571,8 @@ impl RunnerChildren { Err(error) => Err(RequestFailure::Abort(error)), }) } - BrokerOperation::ProcessReady(request) => Some( - self.process_ready(process.id(), request.initial_thread_id) + BrokerOperation::ReportProcessReady(initial_thread_id) => Some( + self.process_ready(process.id(), *initial_thread_id) .map(|()| BrokerResult::ProcessReady) .map_err(process_extension_error), ), From 3c5ed9ab10d70ee4823bd414b4e9c4a6055b4fc2 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 15:43:35 -0700 Subject: [PATCH 11/21] Unify runner process model Model every out-of-process runner with RunnerInstance and represent parent-issued startup through RunnerStartup and ProcessStart coordination. Move the process-start state machine into its own module and share one association-serving path across initial and started runners. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_userland/src/linux.rs | 2 +- litebox_broker_userland/src/runner.rs | 2714 +---------------- litebox_broker_userland/src/runner/linux.rs | 97 +- .../src/runner/process_start.rs | 2474 +++++++++++++++ litebox_broker_userland/src/runner/windows.rs | 101 +- litebox_broker_userland/src/runtime.rs | 191 +- litebox_broker_userland/src/windows.rs | 2 +- 7 files changed, 2749 insertions(+), 2832 deletions(-) create mode 100644 litebox_broker_userland/src/runner/process_start.rs diff --git a/litebox_broker_userland/src/linux.rs b/litebox_broker_userland/src/linux.rs index 3f6add0788..1a229d60d9 100644 --- a/litebox_broker_userland/src/linux.rs +++ b/litebox_broker_userland/src/linux.rs @@ -222,7 +222,7 @@ fn serve_control_stream( ) -> IoResult<()> { let control_channel = UnixStreamHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - litebox_broker_userland::runtime::serve_association( + litebox_broker_userland::runtime::serve_in_process_runner_association( broker, control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 52a00eec5e..38fbbf246e 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -8,45 +8,28 @@ use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::path::PathBuf; use std::process::{Child, Command, ExitStatus}; use std::sync::{ - Arc, Condvar, Mutex, MutexGuard, + Arc, Condvar, Mutex, atomic::{AtomicBool, Ordering}, }; use std::time::{Duration, Instant}; -use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; -use litebox_broker_host::{RequestFailure, copy_shared_buffer}; -use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; -use litebox_broker_protocol::process::{ - InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartToken, StartedProcess, -}; -use litebox_broker_protocol::{ProcessId, ThreadId}; -use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; - -use crate::runtime::AssociationFailureCause; +use litebox_broker_core::BrokerCore; #[cfg(target_os = "linux")] mod linux; +mod process_start; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(target_os = "linux")] use linux::PlatformRunnerEndpoint; +pub(crate) use process_start::{AssociationFailure, RunnerProcessManager, RunnerStartup}; #[cfg(all(windows, target_arch = "x86_64"))] use windows::PlatformRunnerEndpoint; const SETUP_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_RECEIPT_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); const PROCESS_EXIT_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); const ACCEPT_RETRY_DELAY: Duration = Duration::from_millis(10); -const MAX_PENDING_CHILD_STARTS: usize = crate::WORKER_COUNT - 1; -const _: () = assert!(MAX_PENDING_CHILD_STARTS > 0); -const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_WORKER_COUNT > MAX_PENDING_CHILD_STARTS); -const _: () = assert!(crate::runtime::LIFECYCLE_CONTROL_QUEUE_CAPACITY >= MAX_PENDING_CHILD_STARTS); /// Configuration for starting one out-of-process runner. /// @@ -92,7 +75,7 @@ impl RunnerConfig { arguments } - fn child(&self) -> Self { + fn for_process_start(&self) -> Self { Self { executable: self.executable.clone(), arguments: Vec::new(), @@ -109,17 +92,7 @@ pub struct RunnerInstance { runner: Arc>, shutdown: Arc, endpoint: PlatformRunnerEndpoint, - child_config: RunnerConfig, -} - -struct ChildRunResult { - result: IoResult, - runner_success: Option, - runner_signal: Option, - runner_exit_code: Option, - termination_provenance: ChildTerminationProvenance, - association_panicked: bool, - shutdown_observation_failed: bool, + process_start_config: RunnerConfig, } struct RunnerShutdown { @@ -129,33 +102,6 @@ struct RunnerShutdown { termination_dispatched: AtomicBool, } -#[derive(Clone, Copy, Default)] -struct ChildTerminationProvenance(u8); - -impl ChildTerminationProvenance { - const BROKER_TERMINATION: u8 = 1; - const REPORTED_START_FAILURE: u8 = 2; - - const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { - let mut value = 0; - if broker_termination { - value |= Self::BROKER_TERMINATION; - } - if reported_start_failure { - value |= Self::REPORTED_START_FAILURE; - } - Self(value) - } - - const fn broker_termination(self) -> bool { - self.0 & Self::BROKER_TERMINATION != 0 - } - - const fn reported_start_failure(self) -> bool { - self.0 & Self::REPORTED_START_FAILURE != 0 - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum RunnerShutdownState { Active, @@ -256,12 +202,12 @@ impl RunnerInstance { changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), }); - let child_config = config.child(); + let process_start_config = config.for_process_start(); Ok(Self { runner, shutdown, endpoint, - child_config, + process_start_config, }) } @@ -275,8 +221,11 @@ impl RunnerInstance { /// /// Panics if another runner owner poisoned the process mutex. pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { - let children = RunnerChildren::new(self.child_config.clone(), broker.clone()); - let mut association_result = self.endpoint.serve(&self.runner, Arc::clone(&children)); + let process_manager = + RunnerProcessManager::new(self.process_start_config.clone(), broker.clone()); + let mut association_result = + self.endpoint + .serve(&self.runner, None, Arc::clone(&process_manager)); self.endpoint.close(); let runner_exited = if association_result.result.is_ok() { self.shutdown @@ -302,92 +251,12 @@ impl RunnerInstance { if let Some(process) = association_result.process.take() { process.cleanup(!root_abnormal); } - children.wait_for_drain(); + process_manager.wait_for_drain(); let runner_status = runner_status?; runner_exited?; association_result.result?; Ok(runner_status) } - - fn run_child_to_completion( - mut self, - child: ChildRunner, - children: Arc, - ) -> ChildRunResult { - let launch = Arc::clone(&child.launch); - launch.install_shutdown(Arc::clone(&self.shutdown)); - let association_result = self.endpoint.serve_child(&self.runner, child, children); - self.endpoint.close(); - let shutdown_request = launch.shutdown_request(); - let shutdown_was_expected = shutdown_request.was_expected(); - if association_result.abnormal { - launch.mark_abnormal(); - } - let runner_exited = if !shutdown_was_expected - && matches!( - association_result.failure_cause, - AssociationFailureCause::None | AssociationFailureCause::PeerClosed - ) - && !association_result.panicked - { - self.shutdown - .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) - } else { - self.shutdown.has_exited() - }; - let shutdown_observation_failed = match runner_exited { - Ok(true) => { - if association_result.failure_cause == AssociationFailureCause::Other { - launch.mark_abnormal(); - } - false - } - Ok(false) => { - if !shutdown_was_expected - || association_result.failure_cause == AssociationFailureCause::Other - { - launch.mark_abnormal(); - } - launch.mark_shutdown_expected(); - self.shutdown.shutdown(); - false - } - Err(_) => { - launch.mark_abnormal(); - self.shutdown.shutdown(); - true - } - }; - self.shutdown.retire(); - let runner_status = wait_for_runner_exit(&self.runner); - if runner_status.is_err() { - launch.mark_abnormal(); - } - let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); - let runner_signal = runner_status - .as_ref() - .ok() - .copied() - .and_then(runner_exit_signal); - let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); - let termination_provenance = ChildTerminationProvenance::new( - self.shutdown.termination_was_dispatched(), - shutdown_request.expected_start_failure_was_reported(), - ); - let result = runner_status.and_then(|status| { - association_result.result?; - Ok(status) - }); - ChildRunResult { - result, - runner_success, - runner_signal, - runner_exit_code, - termination_provenance, - association_panicked: association_result.panicked, - shutdown_observation_failed, - } - } } impl Drop for RunnerInstance { @@ -399,2469 +268,176 @@ impl Drop for RunnerInstance { } } -pub(crate) struct RunnerChildren { - pub(crate) broker: BrokerCore, - config: RunnerConfig, - state: Mutex, - drained: Condvar, -} - -pub(crate) struct ChildRunner { - pub(crate) process: Arc, - pub(crate) launch: Arc, - pub(crate) inherited_objects: InheritedProcessObjects, - pub(crate) format: ProcessBootstrapFormat, - pub(crate) version: ProcessBootstrapVersion, - pub(crate) bootstrap: Vec, -} - -struct RunnerChildrenState { - launches: Vec<(ProcessStartToken, Arc)>, - associations: Vec<(ProcessId, AssociationFailure)>, - active_instances: usize, - active_watchdogs: usize, -} - -pub(crate) type AssociationFailure = Arc; - -pub(crate) struct ChildLaunch { - parent_id: ProcessId, - process: Arc, - state: Mutex, - changed: Condvar, -} +#[cfg(target_os = "linux")] +fn runner_exit_signal(status: ExitStatus) -> Option { + use std::os::unix::process::ExitStatusExt; -#[derive(Clone, Copy)] -enum ChildLaunchPhase { - Starting, - Ready { initial_thread_id: Option }, - Committing, - Committed, - CompletingStart, - StartComplete, - Aborted(ErrorCode), + status.signal() } -#[derive(Clone, Copy, PartialEq, Eq)] -enum StartResultPublication { - NotStarted, - Publishing, - Delivered, +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_signal(_status: ExitStatus) -> Option { + None } -struct ChildLaunchData { - phase: ChildLaunchPhase, - publication: StartResultPublication, - shutdown: Option>, - association_failure: Option, - shutdown_request: ShutdownRequest, - abnormal: bool, - receipt: ReceiptState, - resolution_watchdog: DeadlineState, - acknowledgement_publication_watchdog: DeadlineState, - start_failure_publication_watchdog: DeadlineState, - active_control_callbacks: usize, - runner_finished: bool, - finalization_taken: bool, +#[cfg(not(any(target_os = "linux", all(windows, target_arch = "x86_64"))))] +const fn runner_exit_signal(_status: ExitStatus) -> Option { + None } -#[derive(Clone, Copy, PartialEq, Eq)] -enum ShutdownRequest { - None, - Expected, - ExpectedStartFailurePending, - ExpectedStartFailure, - Unexpected, +#[cfg(target_os = "linux")] +const fn runner_signal_is_abnormal(signal: Option, broker_termination: bool) -> bool { + matches!(signal, Some(signal) if signal != libc::SIGKILL || !broker_termination) } -impl ShutdownRequest { - const fn was_expected(self) -> bool { - matches!( - self, - Self::Expected | Self::ExpectedStartFailurePending | Self::ExpectedStartFailure - ) - } - - const fn expected_start_failure_was_reported(self) -> bool { - matches!( - self, - Self::ExpectedStartFailurePending | Self::ExpectedStartFailure - ) - } +#[cfg(not(target_os = "linux"))] +const fn runner_signal_is_abnormal(_signal: Option, _broker_termination: bool) -> bool { + false } -#[derive(Clone, Copy, PartialEq, Eq)] -enum ReceiptState { - AwaitingAcknowledgement, - AcknowledgementAdmitted, - TimeoutPending, - Draining, - Resolved, +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_code_is_crash(exit_code: Option) -> bool { + matches!(exit_code, Some(code) if code as u32 >= 0x8000_0000) } -#[derive(Clone, Copy, PartialEq, Eq)] -enum DeadlineState { - Unarmed, - Armed(Instant), - Disarmed, - Fired, +#[cfg(not(all(windows, target_arch = "x86_64")))] +const fn runner_exit_code_is_crash(_exit_code: Option) -> bool { + false } -enum ProcessStartAcknowledgement { - Acknowledged, - Failed(ErrorCode), +#[cfg(all(windows, target_arch = "x86_64"))] +const fn runner_exit_code_is_expected_shutdown( + exit_code: Option, + broker_termination: bool, +) -> bool { + broker_termination && matches!(exit_code, Some(1)) } -struct ReceiptExpiration { - shutdown: Option>, - association_failure: Option, - fail_parent: bool, - commit_supervision_deadline: Option, -} +#[cfg(all(windows, target_arch = "x86_64"))] +const _: () = { + let access_violation = 0xc000_0005_u32 as i32; + let breakpoint = 0x8000_0003_u32 as i32; + assert!(runner_exit_code_is_crash(Some(access_violation))); + assert!(runner_exit_code_is_crash(Some(breakpoint))); + assert!(!runner_exit_code_is_expected_shutdown( + Some(access_violation), + true + )); + assert!(runner_exit_code_is_expected_shutdown(Some(1), true)); +}; -enum ReceiptResolution { - Resolved(Option), - DeferredToDrain, +#[cfg(not(all(windows, target_arch = "x86_64")))] +const fn runner_exit_code_is_expected_shutdown( + _exit_code: Option, + _broker_termination: bool, +) -> bool { + false } -impl RunnerChildren { - fn new(config: RunnerConfig, broker: BrokerCore) -> Arc { - Arc::new(Self { - broker, - config, - state: Mutex::new(RunnerChildrenState { - launches: Vec::new(), - associations: Vec::new(), - active_instances: 0, - active_watchdogs: 0, - }), - drained: Condvar::new(), - }) - } - - pub(crate) fn handle_operation( - self: &Arc, - process: &BrokerProcess, - operation: &BrokerOperation, - shared_buffers: &SharedBufferPool, - ) -> Option> { - match operation { - BrokerOperation::StartProcess(request) => Some( - copy_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) - .and_then(|bootstrap| { - self.start_process( - process, - request.format, - request.version, - bootstrap, - request.inherited_objects, - ) - .map_err(process_extension_error) - }) - .map(BrokerResult::ProcessStarted), - ), - BrokerOperation::AcknowledgeProcessStart(token) => { - Some(match self.resolve_acknowledgement(process.id(), *token) { - Ok(ProcessStartAcknowledgement::Acknowledged) => { - Ok(BrokerResult::ProcessStartAcknowledged) - } - Ok(ProcessStartAcknowledgement::Failed(error)) => { - Ok(BrokerResult::ProcessStartFailed(error)) - } - Err(error) => Err(RequestFailure::Abort(error)), - }) - } - BrokerOperation::ReportProcessReady(initial_thread_id) => Some( - self.process_ready(process.id(), *initial_thread_id) - .map(|()| BrokerResult::ProcessReady) - .map_err(process_extension_error), - ), - BrokerOperation::ReportProcessStartFailure(error) => Some( - self.report_process_start_failure(process.id(), *error) - .map(|()| BrokerResult::ProcessStartFailed(*error)) - .map_err(process_extension_error), - ), - _ => None, - } - } - - pub(crate) fn response_sent( - &self, - process_id: ProcessId, - operation: &BrokerOperation, - result: &BrokerResult, - ) { - match (operation, result) { - (_, BrokerResult::ProcessStarted(started)) => { - let Some(launch) = self.find_launch(started.token) else { - return; - }; - if launch.parent_id == process_id { - launch.mark_start_result_delivered(); - } - } - ( - BrokerOperation::AcknowledgeProcessStart(token), - BrokerResult::ProcessStartAcknowledged | BrokerResult::ProcessStartFailed(_), - ) => { - let Some(launch) = self.find_launch(*token) else { - return; - }; - if launch.parent_id != process_id { - return; - } - if let ReceiptResolution::Resolved(finalization) = launch.resolve_receipt() { - self.remove_launch(*token); - if let Some(abnormal) = finalization { - self.finish_child_launch(&launch, abnormal); - } - } - } - ( - BrokerOperation::ReportProcessStartFailure(error), - BrokerResult::ProcessStartFailed(reported), - ) if error == reported => { - if let Some(launch) = self.find_child_launch(process_id) { - launch.start_failure_response_sent(*error); - } - } - _ => {} - } - } - - fn start_process( - self: &Arc, - parent: &BrokerProcess, - format: ProcessBootstrapFormat, - version: ProcessBootstrapVersion, - bootstrap: Vec, - requested_inherited_objects: InheritedProcessObjects, - ) -> Result { - if !parent.is_running() { - return Err(ErrorCode::ProtocolState); - } - let (process, inherited_objects) = parent - .create_child(requested_inherited_objects.as_slice()) - .map_err(ErrorCode::from)?; - let inherited_objects = InheritedProcessObjects::new(&inherited_objects) - .expect("child handle count must match the bounded inheritance request"); - let child_id = process.id(); - let launch = Arc::new(ChildLaunch { - parent_id: parent.id(), - process: Arc::clone(&process), - state: Mutex::new(ChildLaunchData { - phase: ChildLaunchPhase::Starting, - publication: StartResultPublication::NotStarted, - shutdown: None, - association_failure: None, - shutdown_request: ShutdownRequest::None, - abnormal: false, - receipt: ReceiptState::AwaitingAcknowledgement, - resolution_watchdog: DeadlineState::Unarmed, - acknowledgement_publication_watchdog: DeadlineState::Unarmed, - start_failure_publication_watchdog: DeadlineState::Unarmed, - active_control_callbacks: 0, - runner_finished: false, - finalization_taken: false, - }), - changed: Condvar::new(), - }); - let token = match (|| { - loop { - let mut token = [0; 8]; - getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; - let token = ProcessStartToken(u64::from_ne_bytes(token)); - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state - .launches - .try_reserve(1) - .map_err(|_| ErrorCode::OutOfMemory)?; - if parent.is_cancellation_requested() { - return Err(ErrorCode::PeerClosed); - } - if state.launches.len() >= MAX_PENDING_CHILD_STARTS { - return Err(ErrorCode::ResourceExhausted); - } - if state - .launches - .iter() - .any(|(candidate, _)| *candidate == token) - { - continue; - } - let active_instances = state - .active_instances - .checked_add(1) - .ok_or(ErrorCode::ResourceExhausted)?; - state.launches.push((token, Arc::clone(&launch))); - state.active_instances = active_instances; - return Ok(token); - } - })() { - Ok(token) => token, - Err(error) => { - process.cleanup(true); - return Err(error); - } - }; - - let children = Arc::clone(self); - let config = self.config.clone(); - let thread_launch = Arc::clone(&launch); - let thread = std::thread::Builder::new() - .name(format!("litebox-runner-{}", child_id.0)) - .spawn(move || { - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - RunnerInstance::start(config).map(|instance| { - instance.run_child_to_completion( - ChildRunner { - process, - launch: Arc::clone(&thread_launch), - inherited_objects, - format, - version, - bootstrap, - }, - Arc::clone(&children), - ) - }) - })); - match outcome { - Ok(Ok(result)) => children.child_finished(token, &thread_launch, result, false), - Ok(Err(error)) => children.child_finished( - token, - &thread_launch, - ChildRunResult { - result: Err(error), - runner_success: None, - runner_signal: None, - runner_exit_code: None, - termination_provenance: ChildTerminationProvenance::default(), - association_panicked: false, - shutdown_observation_failed: false, - }, - false, - ), - Err(_) => children.child_finished( - token, - &thread_launch, - ChildRunResult { - result: Err(IoError::other("child runner thread panicked")), - runner_success: None, - runner_signal: None, - runner_exit_code: None, - termination_provenance: ChildTerminationProvenance::default(), - association_panicked: false, - shutdown_observation_failed: false, - }, - true, - ), - } - }); - if thread.is_err() { - self.remove_launch(token); - launch.process.cleanup(true); - self.finish_instance(); - return Err(ErrorCode::OutOfMemory); - } - drop(thread); - - let initial_thread_id = launch.wait_until_ready()?; - let receipt_deadline = launch.begin_start_result_publication()?; - if self - .arm_initial_receipt_deadline(token, &launch, receipt_deadline) - .is_err() - { - if let Some(expiration) = launch.expire_initial_receipt_deadline() { - self.apply_receipt_expiration( - token, - &launch, - expiration, - self.find_association_failure(launch.parent_id), - ); - } - return Err(ErrorCode::OutOfMemory); - } - Ok(StartedProcess { - token, - process_id: child_id, - initial_thread_id, - }) - } - - pub(crate) fn admit_acknowledgement( - self: &Arc, - parent_id: ProcessId, - token: ProcessStartToken, - ) -> Result<(), ErrorCode> { - let launch = self.find_launch(token).ok_or(ErrorCode::UnknownObject)?; - if launch.parent_id != parent_id { - return Err(ErrorCode::UnknownObject); - } - launch.admit_acknowledgement()?; - if self - .arm_internal_resolution_watchdog(token, &launch) - .is_err() - { - if let Some(expiration) = launch.fail_internal_resolution_watchdog() { - self.apply_receipt_expiration( - token, - &launch, - expiration, - self.find_association_failure(launch.parent_id), - ); - } - return Err(ErrorCode::Internal); - } - if self - .arm_acknowledgement_publication_watchdog(token, &launch) - .is_err() - { - if let Some(expiration) = launch.fail_internal_resolution_watchdog() { - self.apply_receipt_expiration( - token, - &launch, - expiration, - self.find_association_failure(launch.parent_id), - ); - } - return Err(ErrorCode::Internal); - } - Ok(()) - } - - fn resolve_acknowledgement( - &self, - parent_id: ProcessId, - token: ProcessStartToken, - ) -> Result { - let launch = self.find_launch(token).ok_or(ErrorCode::PeerClosed)?; - if launch.parent_id != parent_id { - return Err(ErrorCode::UnknownObject); - } - launch.resolve_acknowledgement() - } - - fn process_ready( - &self, - child_id: ProcessId, - initial_thread_id: Option, - ) -> Result<(), ErrorCode> { - let launch = self - .find_child_launch(child_id) - .ok_or(ErrorCode::PeerClosed)?; - launch.ready_and_wait(initial_thread_id) - } - - fn report_process_start_failure( - self: &Arc, - child_id: ProcessId, - error: ErrorCode, - ) -> Result<(), ErrorCode> { - let (token, launch) = self - .find_child_launch_entry(child_id) - .ok_or(ErrorCode::PeerClosed)?; - launch.report_start_failure(error)?; - if self - .arm_start_failure_publication_watchdog(token, &launch) - .is_err() - { - if let Some(expiration) = launch.fail_start_failure_publication_watchdog() { - self.apply_receipt_expiration(token, &launch, expiration, None); - } - return Err(ErrorCode::Internal); - } - Ok(()) - } - - pub(crate) fn association_ending(&self, process_id: ProcessId) -> Vec> { - let draining = self - .state - .lock() - .expect("runner child state mutex poisoned") - .launches - .iter() - .filter(|(_, launch)| launch.parent_id == process_id) - .map(|(_, launch)| Arc::clone(launch)) - .collect::>(); - for launch in &draining { - launch.abort(ErrorCode::PeerClosed, false, true); - launch.begin_receipt_drain(); - } - - let child_launch = { - self.state - .lock() - .expect("runner child state mutex poisoned") - .launches - .iter() - .find_map(|(_, launch)| { - (launch.process.id() == process_id).then(|| Arc::clone(launch)) - }) - }; - if let Some(launch) = child_launch { - launch.association_closed(); - } - draining - } - - pub(crate) fn association_ended(&self, process_id: ProcessId, draining: Vec>) { - self.unregister_association(process_id); - for launch in draining { - self.remove_launch_by_process_id(launch.process.id()); - if let Some(abnormal) = launch.finish_receipt_drain() { - self.finish_child_launch(&launch, abnormal); - } +fn accept_runner_channel( + deadline: Instant, + channel_name: &'static str, + mut runner_status: impl FnMut() -> IoResult>, + mut try_accept: impl FnMut() -> IoResult, +) -> IoResult { + loop { + if let Some(status) = runner_status()? { + return Err(IoError::new( + ErrorKind::BrokenPipe, + format!("runner {status} before connecting its {channel_name} channel"), + )); } - } - - pub(crate) fn register_association( - &self, - process_id: ProcessId, - failure: AssociationFailure, - ) -> IoResult<()> { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state - .associations - .try_reserve(1) - .map_err(|_| IoError::other("failed to reserve broker association registration"))?; - if state - .associations - .iter() - .any(|(candidate, _)| *candidate == process_id) - { - return Err(IoError::other( - "a broker process already has a live association", + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(IoError::new( + ErrorKind::TimedOut, + format!("timed out waiting for runner {channel_name} channel"), )); } - state.associations.push((process_id, failure)); - Ok(()) - } - - pub(crate) fn install_child_association_failure( - &self, - process_id: ProcessId, - failure: AssociationFailure, - ) { - if let Some(launch) = self.find_child_launch(process_id) { - launch.install_association_failure(failure); + match try_accept() { + Ok(channel) => return Ok(channel), + Err(error) if error.kind() == ErrorKind::WouldBlock => {} + Err(error) => return Err(error), } + std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); } +} + +fn runner_has_exited(runner: &Arc>) -> IoResult { + // A pre-authentication caller stops accepting before acting on `true`; + // post-authentication callers no longer rely on PID-based authentication. + runner + .lock() + .expect("runner process mutex poisoned") + .try_wait() + .map(|status| status.is_some()) +} - fn unregister_association(&self, process_id: ProcessId) { - let mut state = self - .state +fn wait_for_runner_exit(runner: &Arc>) -> IoResult { + loop { + if let Some(status) = runner .lock() - .expect("runner child state mutex poisoned"); - if let Some(index) = state - .associations - .iter() - .position(|(candidate, _)| *candidate == process_id) + .expect("runner process mutex poisoned") + .try_wait()? { - state.associations.swap_remove(index); + return Ok(status); } + std::thread::sleep(ACCEPT_RETRY_DELAY); } +} - fn find_association_failure(&self, process_id: ProcessId) -> Option { - self.state - .lock() - .expect("runner child state mutex poisoned") - .associations - .iter() - .find_map(|(candidate, failure)| { - (*candidate == process_id).then(|| Arc::clone(failure)) - }) - } - - fn arm_initial_receipt_deadline( - self: &Arc, - token: ProcessStartToken, - launch: &Arc, - deadline: Instant, - ) -> Result<(), ()> { - let launch = Arc::clone(launch); - let parent_failure = self.find_association_failure(launch.parent_id); - self.spawn_watchdog( - format!("litebox-start-receipt-{}", launch.process.id().0), - move |children| { - let Some(expiration) = launch.wait_for_initial_receipt_deadline(deadline) else { - return; - }; - children.apply_receipt_expiration(token, &launch, expiration, parent_failure); - }, - ) - } +#[cfg(test)] +mod tests { + use std::sync::{Arc, Condvar, Mutex, atomic::AtomicBool, mpsc}; + use std::time::Duration; - fn arm_internal_resolution_watchdog( - self: &Arc, - token: ProcessStartToken, - launch: &Arc, - ) -> Result<(), ()> { - let launch = Arc::clone(launch); - let parent_failure = self.find_association_failure(launch.parent_id); - self.spawn_watchdog( - format!("litebox-start-resolution-{}", launch.process.id().0), - move |children| { - let Some(expiration) = launch.wait_for_internal_resolution_timeout() else { - return; - }; - children.apply_receipt_expiration(token, &launch, expiration, parent_failure); - }, - ) - } + #[cfg(target_os = "linux")] + #[test] + fn shutdown_can_terminate_while_runner_waits_for_exit() { + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; + use std::process::Command; - fn arm_acknowledgement_publication_watchdog( - self: &Arc, - token: ProcessStartToken, - launch: &Arc, - ) -> Result<(), ()> { - let launch = Arc::clone(launch); - let parent_failure = self.find_association_failure(launch.parent_id); - self.spawn_watchdog( - format!("litebox-start-ack-publication-{}", launch.process.id().0), - move |children| { - let Some(expiration) = launch.wait_for_acknowledgement_publication_timeout() else { - return; - }; - children.apply_receipt_expiration(token, &launch, expiration, parent_failure); - }, - ) - } + let runner = Arc::new(Mutex::new( + Command::new("sh") + .args(["-c", "exec sleep 30"]) + .spawn() + .unwrap(), + )); + let shutdown = RunnerShutdown { + runner: Arc::clone(&runner), + state: Mutex::new(RunnerShutdownState::Active), + changed: Condvar::new(), + termination_dispatched: AtomicBool::new(false), + }; + let waiting = Arc::clone(&runner); + let (finished, completion) = mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || { + let status = wait_for_runner_exit(&waiting).unwrap(); + finished.send(status).unwrap(); + }); - fn arm_start_failure_publication_watchdog( - self: &Arc, - token: ProcessStartToken, - launch: &Arc, - ) -> Result<(), ()> { - let launch = Arc::clone(launch); - self.spawn_watchdog( - format!( - "litebox-start-failure-publication-{}", - launch.process.id().0 - ), - move |children| { - let Some(expiration) = launch.wait_for_start_failure_publication_timeout() else { - return; - }; - children.apply_receipt_expiration(token, &launch, expiration, None); - }, - ) + shutdown.shutdown(); + assert!(shutdown.termination_was_dispatched()); + assert!( + !completion + .recv_timeout(Duration::from_secs(1)) + .unwrap() + .success() + ); + waiter.join().unwrap(); } - fn spawn_watchdog( - self: &Arc, - name: String, - watchdog: impl FnOnce(&Arc) + Send + 'static, - ) -> Result<(), ()> { - { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state.active_watchdogs = state.active_watchdogs.checked_add(1).ok_or(())?; - } - let children = Arc::clone(self); - if let Ok(thread) = std::thread::Builder::new().name(name).spawn(move || { - let _completion = WatchdogCompletion { - children: Arc::clone(&children), - }; - watchdog(&children); - }) { - drop(thread); - Ok(()) - } else { - self.finish_watchdog(); - Err(()) - } - } + #[cfg(target_os = "linux")] + #[test] + fn shutdown_after_observed_exit_records_no_termination() { + use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; + use std::process::Command; - fn apply_receipt_expiration( - &self, - token: ProcessStartToken, - launch: &ChildLaunch, - expiration: ReceiptExpiration, - parent_failure: Option, - ) { - let fail_parent = expiration.fail_parent; - let commit_supervision_deadline = expiration.commit_supervision_deadline; - if let Some(association_failure) = expiration.association_failure { - association_failure(); - } - if let Some(shutdown) = expiration.shutdown { - shutdown.shutdown(); - } - let parent_failed = if fail_parent { - if let Some(parent_failure) = parent_failure { - parent_failure(); - true - } else { - false - } - } else { - true - }; - if let Some(abnormal) = launch.complete_timeout_callback() { - self.finish_child_launch(launch, abnormal); - } - if fail_parent && !parent_failed { - self.remove_launch(token); - if let Some(abnormal) = launch.finish_receipt_drain() { - self.finish_child_launch(launch, abnormal); - } - } - if let Some(deadline) = commit_supervision_deadline - && !launch.wait_for_commit_resolution(deadline) - { - std::process::abort(); - } - } - - fn find_launch(&self, token: ProcessStartToken) -> Option> { - self.state - .lock() - .expect("runner child state mutex poisoned") - .launches - .iter() - .find_map(|(candidate, launch)| (*candidate == token).then(|| Arc::clone(launch))) - } - - fn find_child_launch(&self, child_id: ProcessId) -> Option> { - self.find_child_launch_entry(child_id) - .map(|(_, launch)| launch) - } - - fn find_child_launch_entry( - &self, - child_id: ProcessId, - ) -> Option<(ProcessStartToken, Arc)> { - self.state - .lock() - .expect("runner child state mutex poisoned") - .launches - .iter() - .find_map(|(token, launch)| { - (launch.process.id() == child_id).then(|| (*token, Arc::clone(launch))) - }) - } - - fn remove_launch(&self, token: ProcessStartToken) { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - if let Some(index) = state - .launches - .iter() - .position(|(candidate, _)| *candidate == token) - { - state.launches.swap_remove(index); - } - } - - fn remove_launch_by_process_id(&self, process_id: ProcessId) { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - if let Some(index) = state - .launches - .iter() - .position(|(_, launch)| launch.process.id() == process_id) - { - state.launches.swap_remove(index); - } - } - - fn child_finished( - &self, - token: ProcessStartToken, - launch: &ChildLaunch, - result: ChildRunResult, - thread_panicked: bool, - ) { - let unexpected_runner_failure = result.runner_success == Some(false) - && result.runner_signal.is_none() - && !launch.commit_was_claimed() - && !result.termination_provenance.reported_start_failure() - && !runner_exit_code_is_expected_shutdown( - result.runner_exit_code, - result.termination_provenance.broker_termination(), - ); - let unexpected_crash = runner_signal_is_abnormal( - result.runner_signal, - result.termination_provenance.broker_termination(), - ); - let abnormal = thread_panicked - || result.association_panicked - || result.shutdown_observation_failed - || unexpected_crash - || runner_exit_code_is_crash(result.runner_exit_code) - || unexpected_runner_failure; - if result.result.is_err() || result.runner_success != Some(true) { - launch.abort(ErrorCode::PeerClosed, abnormal, false); - } else { - // Any clean host exit before commit still aborts creation. - launch.abort(ErrorCode::PeerClosed, false, false); - } - if !launch.retains_published_receipt() { - self.remove_launch(token); - if let ReceiptResolution::Resolved(finalization) = launch.resolve_receipt() { - debug_assert!(finalization.is_none()); - } - } - if let Some(abnormal) = launch.runner_finished(abnormal) { - self.finish_child_launch(launch, abnormal); - } - } - - fn finish_child_launch(&self, launch: &ChildLaunch, abnormal: bool) { - launch.process.cleanup(!abnormal); - self.finish_instance(); - } - - fn finish_instance(&self) { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state.active_instances = state - .active_instances - .checked_sub(1) - .expect("runner child count must remain balanced"); - if state.active_instances == 0 && state.active_watchdogs == 0 { - self.drained.notify_all(); - } - } - - fn finish_watchdog(&self) { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - state.active_watchdogs = state - .active_watchdogs - .checked_sub(1) - .expect("runner watchdog count must remain balanced"); - if state.active_instances == 0 && state.active_watchdogs == 0 { - self.drained.notify_all(); - } - } - - fn wait_for_drain(&self) { - let mut state = self - .state - .lock() - .expect("runner child state mutex poisoned"); - while state.active_instances != 0 || state.active_watchdogs != 0 { - state = self - .drained - .wait(state) - .expect("runner child state mutex poisoned"); - } - } -} - -struct WatchdogCompletion { - children: Arc, -} - -impl Drop for WatchdogCompletion { - fn drop(&mut self) { - self.children.finish_watchdog(); - } -} - -const fn process_extension_error(error: ErrorCode) -> RequestFailure { - match error { - ErrorCode::PolicyDenied - | ErrorCode::UnknownObject - | ErrorCode::InvalidRights - | ErrorCode::ResourceExhausted - | ErrorCode::WouldBlock - | ErrorCode::PeerClosed - | ErrorCode::OutOfMemory - | ErrorCode::UnsupportedOperation => RequestFailure::Respond(error), - _ => RequestFailure::Abort(error), - } -} - -const fn process_start_failure_is_expected(error: ErrorCode) -> bool { - matches!( - error, - ErrorCode::UnsupportedOperation - | ErrorCode::PolicyDenied - | ErrorCode::InvalidRights - | ErrorCode::ResourceExhausted - | ErrorCode::WouldBlock - | ErrorCode::OutOfMemory - ) -} - -impl ChildLaunch { - fn wait_until_ready(&self) -> Result, ErrorCode> { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match state.phase { - ChildLaunchPhase::Starting => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - ChildLaunchPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), - ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete => { - return Err(ErrorCode::ProtocolState); - } - ChildLaunchPhase::Aborted(error) => return Err(error), - } - } - } - - fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { - self.process - .mark_start_ready(initial_thread_id) - .map_err(|error| match error { - BrokerError::UnknownObject => ErrorCode::ProtocolState, - error => ErrorCode::from(error), - })?; - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if !matches!(state.phase, ChildLaunchPhase::Starting) { - return Err(match state.phase { - ChildLaunchPhase::Aborted(error) => error, - _ => ErrorCode::ProtocolState, - }); - } - state.phase = ChildLaunchPhase::Ready { initial_thread_id }; - self.changed.notify_all(); - loop { - match state.phase { - ChildLaunchPhase::StartComplete if state.active_control_callbacks == 0 => { - return Ok(()); - } - ChildLaunchPhase::Ready { .. } - | ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - ChildLaunchPhase::Aborted(error) => return Err(error), - ChildLaunchPhase::Starting => unreachable!("ready state cannot regress"), - } - } - } - - fn begin_start_result_publication(&self) -> Result { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - match state.phase { - ChildLaunchPhase::Ready { .. } - if state.publication == StartResultPublication::NotStarted => - { - state.publication = StartResultPublication::Publishing; - Ok(Instant::now() + PROCESS_START_RECEIPT_TIMEOUT) - } - ChildLaunchPhase::Aborted(error) => Err(error), - _ => Err(ErrorCode::ProtocolState), - } - } - - fn report_start_failure(&self, error: ErrorCode) -> Result<(), ErrorCode> { - if !process_start_failure_is_expected(error) { - return Err(ErrorCode::ProtocolState); - } - let mut state = self.state.lock().expect("child launch mutex poisoned"); - match state.phase { - ChildLaunchPhase::Starting => {} - ChildLaunchPhase::Aborted(error) => return Err(error), - _ => return Err(ErrorCode::ProtocolState), - } - state.phase = ChildLaunchPhase::Aborted(error); - state.shutdown_request = ShutdownRequest::ExpectedStartFailurePending; - arm_deadline( - &mut state.start_failure_publication_watchdog, - PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, - ); - self.changed.notify_all(); - Ok(()) - } - - fn start_failure_response_sent(&self, error: ErrorCode) { - let (association_failure, shutdown) = { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - complete_deadline(&mut state.start_failure_publication_watchdog); - let actions = if matches!(state.phase, ChildLaunchPhase::Aborted(cause) if cause == error) - && state.shutdown_request == ShutdownRequest::ExpectedStartFailurePending - { - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } else { - (None, None) - }; - self.changed.notify_all(); - actions - }; - if let Some(association_failure) = association_failure { - association_failure(); - } - if let Some(shutdown) = shutdown { - shutdown.shutdown(); - } - } - - fn admit_acknowledgement(&self) -> Result<(), ErrorCode> { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if state.publication == StartResultPublication::NotStarted { - return Err(ErrorCode::ProtocolState); - } - match state.receipt { - ReceiptState::AwaitingAcknowledgement => {} - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { - return Err(ErrorCode::PeerClosed); - } - ReceiptState::AcknowledgementAdmitted => return Err(ErrorCode::UnknownObject), - } - state.receipt = ReceiptState::AcknowledgementAdmitted; - if state.publication == StartResultPublication::Delivered - && matches!( - state.phase, - ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) - ) - { - arm_deadline( - &mut state.resolution_watchdog, - PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, - ); - self.changed.notify_all(); - } - Ok(()) - } - - fn resolve_acknowledgement(&self) -> Result { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match state.receipt { - ReceiptState::AcknowledgementAdmitted => {} - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { - return Err(ErrorCode::PeerClosed); - } - ReceiptState::AwaitingAcknowledgement => return Err(ErrorCode::ProtocolState), - } - if state.publication == StartResultPublication::NotStarted { - return Err(ErrorCode::ProtocolState); - } - match (state.phase, state.publication) { - ( - ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_), - StartResultPublication::Publishing, - ) => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - (ChildLaunchPhase::Ready { .. }, StartResultPublication::Delivered) => { - debug_assert!(matches!(state.resolution_watchdog, DeadlineState::Armed(_))); - state.phase = ChildLaunchPhase::Committing; - drop(state); - let commit_result = self.process.commit_start().map_err(ErrorCode::from); - state = self.state.lock().expect("child launch mutex poisoned"); - match commit_result { - Ok(()) => { - state.phase = ChildLaunchPhase::Committed; - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Acknowledged); - } - Err(error) => { - state.phase = ChildLaunchPhase::Aborted(error); - state.abnormal |= error == ErrorCode::Internal; - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Failed(error)); - } - } - } - (ChildLaunchPhase::Aborted(error), StartResultPublication::Delivered) => { - debug_assert!(matches!( - state.resolution_watchdog, - DeadlineState::Armed(_) | DeadlineState::Fired - )); - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Failed(error)); - } - (ChildLaunchPhase::Starting, _) => return Err(ErrorCode::ProtocolState), - ( - ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete, - _, - ) => { - return Err(ErrorCode::ProtocolState); - } - (_, StartResultPublication::NotStarted) => { - unreachable!("publication state was checked before phase dispatch") - } - } - } - } - - fn mark_start_result_delivered(&self) { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if state.publication == StartResultPublication::Publishing { - state.publication = StartResultPublication::Delivered; - if state.receipt == ReceiptState::AcknowledgementAdmitted - && matches!( - state.phase, - ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) - ) - { - arm_deadline( - &mut state.resolution_watchdog, - PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, - ); - } - self.changed.notify_all(); - } - } - - fn install_shutdown(&self, shutdown: Arc) { - let shutdown = { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.shutdown = Some(Arc::clone(&shutdown)); - (state.shutdown_request != ShutdownRequest::None).then_some(shutdown) - }; - if let Some(shutdown) = shutdown { - shutdown.shutdown(); - } - } - - fn install_association_failure(&self, failure: AssociationFailure) { - let failure = { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.association_failure = Some(Arc::clone(&failure)); - (state.shutdown_request != ShutdownRequest::None).then_some(failure) - }; - if let Some(failure) = failure { - failure(); - } - } - - fn abort(&self, error: ErrorCode, abnormal: bool, expected_shutdown: bool) { - let (association_failure, shutdown) = { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.abnormal |= abnormal; - match state.phase { - ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } => { - state.phase = ChildLaunchPhase::Aborted(error); - let newly_requested = state.shutdown_request == ShutdownRequest::None; - if newly_requested { - state.shutdown_request = if expected_shutdown { - ShutdownRequest::Expected - } else { - ShutdownRequest::Unexpected - }; - } - self.changed.notify_all(); - ( - newly_requested - .then(|| state.association_failure.as_ref().map(Arc::clone)) - .flatten(), - state.shutdown.clone(), - ) - } - ChildLaunchPhase::Aborted(_) => match state.shutdown_request { - ShutdownRequest::None => { - state.shutdown_request = if expected_shutdown { - ShutdownRequest::Expected - } else { - ShutdownRequest::Unexpected - }; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } - ShutdownRequest::ExpectedStartFailurePending => { - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } - ShutdownRequest::Expected - | ShutdownRequest::ExpectedStartFailure - | ShutdownRequest::Unexpected => (None, None), - }, - ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete => (None, None), - } - }; - if let Some(association_failure) = association_failure { - association_failure(); - } - if let Some(shutdown) = shutdown { - shutdown.shutdown(); - } - } - - fn association_closed(&self) { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.association_failure = None; - if matches!( - state.phase, - ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } - ) { - state.phase = ChildLaunchPhase::Aborted(ErrorCode::PeerClosed); - self.changed.notify_all(); - } - } - - fn mark_shutdown_expected(&self) { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - } - - fn mark_abnormal(&self) { - self.state - .lock() - .expect("child launch mutex poisoned") - .abnormal = true; - } - - fn shutdown_request(&self) -> ShutdownRequest { - self.state - .lock() - .expect("child launch mutex poisoned") - .shutdown_request - } - - fn commit_was_claimed(&self) -> bool { - matches!( - self.state - .lock() - .expect("child launch mutex poisoned") - .phase, - ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete - ) - } - - fn retains_published_receipt(&self) -> bool { - let state = self.state.lock().expect("child launch mutex poisoned"); - state.publication != StartResultPublication::NotStarted - && state.receipt != ReceiptState::Resolved - } - - fn resolve_receipt(&self) -> ReceiptResolution { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if matches!( - state.receipt, - ReceiptState::TimeoutPending | ReceiptState::Draining - ) { - return ReceiptResolution::DeferredToDrain; - } - state.receipt = ReceiptState::Resolved; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); - ReceiptResolution::Resolved(self.complete_process_start_and_take_finalization(state)) - } - - fn begin_receipt_drain(&self) { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if state.receipt != ReceiptState::Resolved { - state.receipt = ReceiptState::Draining; - if !matches!(state.phase, ChildLaunchPhase::Committing) { - complete_deadline(&mut state.resolution_watchdog); - } - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); - } - } - - fn finish_receipt_drain(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.receipt = ReceiptState::Resolved; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); - self.complete_process_start_and_take_finalization(state) - } - - fn expire_initial_receipt_deadline(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - let (error, abnormal) = match (state.receipt, state.publication) { - (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), - (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { - (ErrorCode::Internal, true) - } - _ => return None, - }; - expire_launch(&mut state, error, abnormal, true, &self.changed) - } - - fn wait_for_initial_receipt_deadline(&self, deadline: Instant) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - if !matches!( - (state.receipt, state.publication), - (ReceiptState::AwaitingAcknowledgement, _) - | ( - ReceiptState::AcknowledgementAdmitted, - StartResultPublication::Publishing - ) - ) { - return None; - } - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - break; - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("child launch mutex poisoned"); - state = next; - } - let (error, abnormal) = match (state.receipt, state.publication) { - (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), - (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { - (ErrorCode::Internal, true) - } - _ => return None, - }; - expire_launch(&mut state, error, abnormal, true, &self.changed) - } - - fn wait_for_internal_resolution_timeout(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match state.resolution_watchdog { - DeadlineState::Unarmed => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.resolution_watchdog = DeadlineState::Fired; - return expire_internal_resolution(&mut state, &self.changed); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("child launch mutex poisoned"); - state = next; - } - DeadlineState::Disarmed | DeadlineState::Fired => return None, - } - } - } - - fn wait_for_acknowledgement_publication_timeout(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match state.acknowledgement_publication_watchdog { - DeadlineState::Unarmed => { - state = self - .changed - .wait(state) - .expect("child launch mutex poisoned"); - } - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.acknowledgement_publication_watchdog = DeadlineState::Fired; - return expire_launch( - &mut state, - ErrorCode::PeerClosed, - false, - true, - &self.changed, - ); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("child launch mutex poisoned"); - state = next; - } - DeadlineState::Disarmed | DeadlineState::Fired => return None, - } - } - } - - fn wait_for_start_failure_publication_timeout(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - loop { - match state.start_failure_publication_watchdog { - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.start_failure_publication_watchdog = DeadlineState::Fired; - return expire_start_failure_publication(&mut state, &self.changed); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("child launch mutex poisoned"); - state = next; - } - DeadlineState::Unarmed | DeadlineState::Disarmed | DeadlineState::Fired => { - return None; - } - } - } - } - - fn fail_start_failure_publication_watchdog(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - if !matches!( - state.start_failure_publication_watchdog, - DeadlineState::Armed(_) - ) { - return None; - } - state.start_failure_publication_watchdog = DeadlineState::Fired; - expire_start_failure_publication(&mut state, &self.changed) - } - - fn fail_internal_resolution_watchdog(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - expire_launch(&mut state, ErrorCode::Internal, true, true, &self.changed) - } - - fn complete_timeout_callback(&self) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.active_control_callbacks = state - .active_control_callbacks - .checked_sub(1) - .expect("process-start timeout callback count must remain balanced"); - self.changed.notify_all(); - take_finalization(&mut state) - } - - fn wait_for_commit_resolution(&self, deadline: Instant) -> bool { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - while matches!(state.phase, ChildLaunchPhase::Committing) { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return false; - } - let (next, wait_result) = self - .changed - .wait_timeout(state, remaining) - .expect("child launch mutex poisoned"); - state = next; - if wait_result.timed_out() && matches!(state.phase, ChildLaunchPhase::Committing) { - return false; - } - } - true - } - - fn complete_process_start_and_take_finalization( - &self, - mut state: MutexGuard<'_, ChildLaunchData>, - ) -> Option { - if !matches!(state.phase, ChildLaunchPhase::Committed) { - return take_finalization(&mut state); - } - state.phase = ChildLaunchPhase::CompletingStart; - state.active_control_callbacks = state - .active_control_callbacks - .checked_add(1) - .expect("process-start control callback count must remain bounded"); - drop(state); - - let completion_failed = self.process.complete_start().is_err(); - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.active_control_callbacks = state - .active_control_callbacks - .checked_sub(1) - .expect("process-start control callback count must remain balanced"); - state.abnormal |= completion_failed; - state.phase = ChildLaunchPhase::StartComplete; - self.changed.notify_all(); - take_finalization(&mut state) - } - - fn runner_finished(&self, abnormal: bool) -> Option { - let mut state = self.state.lock().expect("child launch mutex poisoned"); - state.abnormal |= abnormal; - state.runner_finished = true; - take_finalization(&mut state) - } -} - -fn expire_launch( - state: &mut ChildLaunchData, - error: ErrorCode, - abnormal: bool, - fail_parent: bool, - changed: &Condvar, -) -> Option { - if matches!( - state.receipt, - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved - ) { - return None; - } - state.active_control_callbacks += 1; - state.receipt = ReceiptState::TimeoutPending; - state.abnormal |= abnormal; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - let (association_failure, shutdown) = match state.phase { - ChildLaunchPhase::Starting | ChildLaunchPhase::Ready { .. } => { - state.phase = ChildLaunchPhase::Aborted(error); - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } - ChildLaunchPhase::Aborted(_) => { - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } else { - (None, None) - } - } - ChildLaunchPhase::Committing - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete => (None, None), - }; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown, - association_failure, - fail_parent, - commit_supervision_deadline: None, - }) -} - -fn expire_start_failure_publication( - state: &mut ChildLaunchData, - changed: &Condvar, -) -> Option { - if !matches!(state.phase, ChildLaunchPhase::Aborted(_)) - || state.shutdown_request != ShutdownRequest::ExpectedStartFailurePending - { - return None; - } - state.active_control_callbacks += 1; - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown: state.shutdown.clone(), - association_failure: state.association_failure.as_ref().map(Arc::clone), - fail_parent: false, - commit_supervision_deadline: None, - }) -} - -fn expire_internal_resolution( - state: &mut ChildLaunchData, - changed: &Condvar, -) -> Option { - let committing_drain = state.receipt == ReceiptState::Draining - && matches!(state.phase, ChildLaunchPhase::Committing); - if state.receipt != ReceiptState::AcknowledgementAdmitted && !committing_drain { - return None; - } - state.active_control_callbacks += 1; - state.abnormal = true; - let (association_failure, shutdown, fail_parent, commit_supervision_deadline) = - match state.phase { - ChildLaunchPhase::Ready { .. } | ChildLaunchPhase::Aborted(_) => { - state.phase = ChildLaunchPhase::Aborted(ErrorCode::Internal); - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - false, - None, - ) - } - ChildLaunchPhase::Committing => { - state.receipt = ReceiptState::TimeoutPending; - complete_deadline(&mut state.acknowledgement_publication_watchdog); - ( - None, - None, - true, - Some(Instant::now() + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT), - ) - } - ChildLaunchPhase::Starting - | ChildLaunchPhase::Committed - | ChildLaunchPhase::CompletingStart - | ChildLaunchPhase::StartComplete => { - state.active_control_callbacks -= 1; - return None; - } - }; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown, - association_failure, - fail_parent, - commit_supervision_deadline, - }) -} - -fn arm_deadline(state: &mut DeadlineState, timeout: Duration) { - if *state == DeadlineState::Unarmed { - *state = DeadlineState::Armed(Instant::now() + timeout); - } -} - -fn complete_deadline(state: &mut DeadlineState) { - if matches!(state, DeadlineState::Unarmed | DeadlineState::Armed(_)) { - *state = DeadlineState::Disarmed; - } -} - -fn complete_acknowledgement_resolution(state: &mut ChildLaunchData) { - complete_deadline(&mut state.resolution_watchdog); - if state.receipt == ReceiptState::AcknowledgementAdmitted { - arm_deadline( - &mut state.acknowledgement_publication_watchdog, - PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, - ); - } -} - -fn take_finalization(state: &mut ChildLaunchData) -> Option { - if state.finalization_taken - || state.active_control_callbacks != 0 - || state.receipt != ReceiptState::Resolved - { - return None; - } - if !state.runner_finished { - return None; - } - state.runner_finished = false; - state.finalization_taken = true; - Some(state.abnormal) -} - -#[cfg(target_os = "linux")] -fn runner_exit_signal(status: ExitStatus) -> Option { - use std::os::unix::process::ExitStatusExt; - - status.signal() -} - -#[cfg(all(windows, target_arch = "x86_64"))] -const fn runner_exit_signal(_status: ExitStatus) -> Option { - None -} - -#[cfg(not(any(target_os = "linux", all(windows, target_arch = "x86_64"))))] -const fn runner_exit_signal(_status: ExitStatus) -> Option { - None -} - -#[cfg(target_os = "linux")] -const fn runner_signal_is_abnormal(signal: Option, broker_termination: bool) -> bool { - matches!(signal, Some(signal) if signal != libc::SIGKILL || !broker_termination) -} - -#[cfg(not(target_os = "linux"))] -const fn runner_signal_is_abnormal(_signal: Option, _broker_termination: bool) -> bool { - false -} - -#[cfg(all(windows, target_arch = "x86_64"))] -const fn runner_exit_code_is_crash(exit_code: Option) -> bool { - matches!(exit_code, Some(code) if code as u32 >= 0x8000_0000) -} - -#[cfg(not(all(windows, target_arch = "x86_64")))] -const fn runner_exit_code_is_crash(_exit_code: Option) -> bool { - false -} - -#[cfg(all(windows, target_arch = "x86_64"))] -const fn runner_exit_code_is_expected_shutdown( - exit_code: Option, - broker_termination: bool, -) -> bool { - broker_termination && matches!(exit_code, Some(1)) -} - -#[cfg(all(windows, target_arch = "x86_64"))] -const _: () = { - let access_violation = 0xc000_0005_u32 as i32; - let breakpoint = 0x8000_0003_u32 as i32; - assert!(runner_exit_code_is_crash(Some(access_violation))); - assert!(runner_exit_code_is_crash(Some(breakpoint))); - assert!(!runner_exit_code_is_expected_shutdown( - Some(access_violation), - true - )); - assert!(runner_exit_code_is_expected_shutdown(Some(1), true)); -}; - -#[cfg(not(all(windows, target_arch = "x86_64")))] -const fn runner_exit_code_is_expected_shutdown( - _exit_code: Option, - _broker_termination: bool, -) -> bool { - false -} - -fn accept_runner_channel( - deadline: Instant, - channel_name: &'static str, - mut runner_status: impl FnMut() -> IoResult>, - mut try_accept: impl FnMut() -> IoResult, -) -> IoResult { - loop { - if let Some(status) = runner_status()? { - return Err(IoError::new( - ErrorKind::BrokenPipe, - format!("runner {status} before connecting its {channel_name} channel"), - )); - } - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Err(IoError::new( - ErrorKind::TimedOut, - format!("timed out waiting for runner {channel_name} channel"), - )); - } - match try_accept() { - Ok(channel) => return Ok(channel), - Err(error) if error.kind() == ErrorKind::WouldBlock => {} - Err(error) => return Err(error), - } - std::thread::sleep(remaining.min(ACCEPT_RETRY_DELAY)); - } -} - -fn runner_has_exited(runner: &Arc>) -> IoResult { - // A pre-authentication caller stops accepting before acting on `true`; - // post-authentication callers no longer rely on PID-based authentication. - runner - .lock() - .expect("runner process mutex poisoned") - .try_wait() - .map(|status| status.is_some()) -} - -fn wait_for_runner_exit(runner: &Arc>) -> IoResult { - loop { - if let Some(status) = runner - .lock() - .expect("runner process mutex poisoned") - .try_wait()? - { - return Ok(status); - } - std::thread::sleep(ACCEPT_RETRY_DELAY); - } -} - -#[cfg(test)] -mod tests { - use super::{ - ChildLaunch, ChildLaunchData, ChildLaunchPhase, DeadlineState, - PROCESS_START_RECEIPT_TIMEOUT, PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT, - ProcessStartAcknowledgement, ReceiptResolution, ReceiptState, RunnerChildren, - RunnerChildrenState, RunnerConfig, ShutdownRequest, StartResultPublication, - }; - use litebox_broker_core::test_support::TestBrokerCoreBuilder; - use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; - use litebox_broker_protocol::ProcessId; - use litebox_broker_protocol::error::ErrorCode; - use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; - use litebox_broker_protocol::process::ProcessStartToken; - use std::path::PathBuf; - use std::sync::{ - Arc, Condvar, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc, - }; - use std::time::{Duration, Instant}; - - fn launch_with_broker_in_phase( - publication: StartResultPublication, - phase: ChildLaunchPhase, - ) -> (BrokerCore, Arc) { - let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( - ObjectRights::all(), - )) - .build() - .unwrap(); - let parent = broker - .create_process(CallerCredential::Unauthenticated) - .unwrap(); - let (process, _) = parent.create_child(&[]).unwrap(); - parent.cleanup(true); - if let ChildLaunchPhase::Ready { initial_thread_id } = phase { - process.mark_start_ready(initial_thread_id).unwrap(); - } - ( - broker, - Arc::new(ChildLaunch { - parent_id: ProcessId(1), - process, - state: Mutex::new(ChildLaunchData { - phase, - publication, - shutdown: None, - association_failure: None, - shutdown_request: ShutdownRequest::None, - abnormal: false, - receipt: ReceiptState::AwaitingAcknowledgement, - resolution_watchdog: DeadlineState::Unarmed, - acknowledgement_publication_watchdog: DeadlineState::Unarmed, - start_failure_publication_watchdog: DeadlineState::Unarmed, - active_control_callbacks: 0, - runner_finished: false, - finalization_taken: false, - }), - changed: Condvar::new(), - }), - ) - } - - fn launch_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { - launch_with_broker_in_phase( - publication, - ChildLaunchPhase::Ready { - initial_thread_id: None, - }, - ) - } - - fn launch(publication: StartResultPublication) -> Arc { - launch_with_broker(publication).1 - } - - fn starting_launch(publication: StartResultPublication) -> Arc { - launch_with_broker_in_phase(publication, ChildLaunchPhase::Starting).1 - } - - #[test] - fn publication_captures_an_absolute_receipt_deadline() { - let launch = launch(StartResultPublication::NotStarted); - let before = Instant::now(); - - let deadline = launch.begin_start_result_publication().unwrap(); - let after = Instant::now(); - - assert!(deadline >= before + PROCESS_START_RECEIPT_TIMEOUT); - assert!(deadline <= after + PROCESS_START_RECEIPT_TIMEOUT); - } - - #[test] - fn acknowledgement_ingress_claims_receipt_before_worker_resolution() { - let (broker, launch) = launch_with_broker(StartResultPublication::Delivered); - let token = ProcessStartToken(7); - let parent_id = launch.parent_id; - let children = Arc::new(RunnerChildren { - broker, - config: RunnerConfig::new(PathBuf::new(), Vec::new()), - state: Mutex::new(RunnerChildrenState { - launches: vec![(token, Arc::clone(&launch))], - associations: Vec::new(), - active_instances: 1, - active_watchdogs: 0, - }), - drained: Condvar::new(), - }); - - children.admit_acknowledgement(parent_id, token).unwrap(); - - assert_eq!(children.state.lock().unwrap().active_watchdogs, 2); - assert!(matches!( - launch.state.lock().unwrap().receipt, - ReceiptState::AcknowledgementAdmitted - )); - assert!(launch.expire_initial_receipt_deadline().is_none()); - assert!(matches!( - children.resolve_acknowledgement(parent_id, token), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - children.response_sent( - parent_id, - &BrokerOperation::AcknowledgeProcessStart(token), - &BrokerResult::ProcessStartAcknowledged, - ); - launch.process.cleanup(true); - children.finish_instance(); - children.wait_for_drain(); - assert_eq!(children.state.lock().unwrap().active_watchdogs, 0); - } - - #[test] - fn reported_bootstrap_rejection_selects_normal_rollback() { - let launch = starting_launch(StartResultPublication::NotStarted); - - launch - .report_start_failure(ErrorCode::UnsupportedOperation) - .unwrap(); - - assert!(matches!( - launch.wait_until_ready(), - Err(ErrorCode::UnsupportedOperation) - )); - assert!( - launch - .shutdown_request() - .expected_start_failure_was_reported() - ); - assert!(!launch.state.lock().unwrap().abnormal); - launch.start_failure_response_sent(ErrorCode::UnsupportedOperation); - assert!(launch.shutdown_request().was_expected()); - assert_eq!(launch.runner_finished(false), None); - assert!(matches!( - launch.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - launch.process.cleanup(true); - } - - #[test] - fn acknowledgement_waits_for_publication_bookkeeping() { - let launch = launch(StartResultPublication::Publishing); - launch.admit_acknowledgement().unwrap(); - let waiting = Arc::clone(&launch); - let (sender, receiver) = mpsc::sync_channel(1); - let worker = - std::thread::spawn(move || sender.send(waiting.resolve_acknowledgement()).unwrap()); - - assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); - assert!(matches!( - launch.state.lock().unwrap().resolution_watchdog, - DeadlineState::Unarmed - )); - launch.mark_start_result_delivered(); - assert!(matches!( - receiver.recv_timeout(Duration::from_secs(1)).unwrap(), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - worker.join().unwrap(); - assert!(launch.process.is_running()); - assert!(matches!( - launch.state.lock().unwrap().resolution_watchdog, - DeadlineState::Disarmed - )); - assert!(matches!( - launch - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog, - DeadlineState::Armed(_) - )); - - launch.resolve_receipt(); - launch.process.cleanup(true); - } - - #[test] - fn acknowledgement_interrupted_by_drain_returns_peer_closed() { - let launch = launch(StartResultPublication::Publishing); - launch.admit_acknowledgement().unwrap(); - let waiting = Arc::clone(&launch); - let worker = std::thread::spawn(move || waiting.resolve_acknowledgement()); - - std::thread::sleep(Duration::from_millis(20)); - launch.begin_receipt_drain(); - - assert!(matches!(worker.join().unwrap(), Err(ErrorCode::PeerClosed))); - launch.finish_receipt_drain(); - launch.process.cleanup(true); - } - - #[test] - fn process_ready_interrupted_by_abort_returns_the_abort_cause() { - let ready = starting_launch(StartResultPublication::NotStarted); - ready.abort(ErrorCode::PeerClosed, false, true); - assert!(matches!( - ready.ready_and_wait(None), - Err(ErrorCode::PeerClosed) - )); - ready.resolve_receipt(); - ready.process.cleanup(true); - } - - #[test] - fn failure_report_interrupted_by_abort_returns_the_abort_cause() { - let failed = starting_launch(StartResultPublication::NotStarted); - failed.abort(ErrorCode::PeerClosed, false, true); - assert!(matches!( - failed.report_start_failure(ErrorCode::UnsupportedOperation), - Err(ErrorCode::PeerClosed) - )); - failed.resolve_receipt(); - failed.process.cleanup(true); - } - - #[test] - fn delivery_arms_the_resolution_watchdog_before_the_worker_resumes() { - let launch = launch(StartResultPublication::Publishing); - launch.admit_acknowledgement().unwrap(); - - assert!(matches!( - launch.state.lock().unwrap().resolution_watchdog, - DeadlineState::Unarmed - )); - launch.mark_start_result_delivered(); - assert!(matches!( - launch.state.lock().unwrap().resolution_watchdog, - DeadlineState::Armed(_) - )); - - launch.begin_receipt_drain(); - launch.finish_receipt_drain(); - launch.process.cleanup(true); - } - - #[test] - fn acknowledgement_publication_timeout_retains_receipt_for_drain() { - let launch = launch(StartResultPublication::Delivered); - launch.admit_acknowledgement().unwrap(); - assert!(matches!( - launch.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - launch - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = launch - .wait_for_acknowledgement_publication_timeout() - .unwrap(); - assert!(expiration.fail_parent); - let state = launch.state.lock().unwrap(); - assert!(matches!(state.receipt, ReceiptState::TimeoutPending)); - assert!(matches!( - state.acknowledgement_publication_watchdog, - DeadlineState::Fired - )); - assert!(!state.abnormal); - drop(state); - - assert_eq!(launch.complete_timeout_callback(), None); - launch.begin_receipt_drain(); - launch.finish_receipt_drain(); - launch.process.cleanup(true); - } - - #[test] - fn start_failure_publication_timeout_terminates_the_child() { - let launch = starting_launch(StartResultPublication::NotStarted); - launch - .report_start_failure(ErrorCode::UnsupportedOperation) - .unwrap(); - launch - .state - .lock() - .unwrap() - .start_failure_publication_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = launch.wait_for_start_failure_publication_timeout().unwrap(); - - assert!(!expiration.fail_parent); - assert!(launch.shutdown_request().was_expected()); - assert!(matches!( - launch - .state - .lock() - .unwrap() - .start_failure_publication_watchdog, - DeadlineState::Fired - )); - assert_eq!(launch.complete_timeout_callback(), None); - assert_eq!(launch.runner_finished(false), None); - assert!(matches!( - launch.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - launch.process.cleanup(true); - } - - #[test] - fn precommit_resolution_timeout_returns_typed_internal_failure() { - let launch = launch(StartResultPublication::Delivered); - launch.admit_acknowledgement().unwrap(); - launch.state.lock().unwrap().resolution_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = launch.wait_for_internal_resolution_timeout().unwrap(); - assert!(!expiration.fail_parent); - let state = launch.state.lock().unwrap(); - assert!(matches!( - state.phase, - ChildLaunchPhase::Aborted(ErrorCode::Internal) - )); - assert!(matches!( - state.receipt, - ReceiptState::AcknowledgementAdmitted - )); - assert!(matches!( - state.acknowledgement_publication_watchdog, - DeadlineState::Unarmed - )); - assert!(state.abnormal); - drop(state); - assert_eq!(launch.complete_timeout_callback(), None); - - assert!(matches!( - launch.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Failed(ErrorCode::Internal)) - )); - assert!(matches!( - launch - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog, - DeadlineState::Armed(_) - )); - launch.resolve_receipt(); - launch.process.cleanup(false); - } - - #[test] - fn child_abort_fails_an_installed_active_association() { - let launch = launch(StartResultPublication::NotStarted); - let failed = Arc::new(AtomicBool::new(false)); - let recorded = Arc::clone(&failed); - launch.install_association_failure(Arc::new(move || { - recorded.store(true, Ordering::Release); - })); - - launch.abort(ErrorCode::PeerClosed, false, true); - - assert!(failed.load(Ordering::Acquire)); - launch.process.cleanup(true); - } - - #[test] - fn supervisor_wait_observes_commit_resolution() { - let launch = launch(StartResultPublication::Delivered); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Committing; - let waiting = Arc::clone(&launch); - let worker = std::thread::spawn(move || { - waiting.wait_for_commit_resolution(Instant::now() + Duration::from_secs(1)) - }); - - std::thread::sleep(Duration::from_millis(20)); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Committed; - launch.changed.notify_all(); - - assert!(worker.join().unwrap()); - launch.process.cleanup(true); - } - - #[test] - fn receipt_drain_keeps_commit_supervision_armed() { - let launch = launch(StartResultPublication::Delivered); - launch.admit_acknowledgement().unwrap(); - { - let mut state = launch.state.lock().unwrap(); - state.phase = ChildLaunchPhase::Committing; - state.resolution_watchdog = DeadlineState::Armed(Instant::now()); - } - launch.begin_receipt_drain(); - assert!(matches!( - launch.state.lock().unwrap().resolution_watchdog, - DeadlineState::Armed(_) - )); - - let before = Instant::now(); - let expiration = launch.wait_for_internal_resolution_timeout().unwrap(); - let after = Instant::now(); - let deadline = expiration.commit_supervision_deadline.unwrap(); - assert!(deadline >= before + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); - assert!(deadline <= after + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); - { - let mut state = launch.state.lock().unwrap(); - state.phase = ChildLaunchPhase::Aborted(ErrorCode::Internal); - } - launch.changed.notify_all(); - assert_eq!(launch.complete_timeout_callback(), None); - launch.finish_receipt_drain(); - launch.process.cleanup(false); - } - - #[test] - fn published_abort_returns_typed_start_failure() { - let launch = launch(StartResultPublication::Publishing); - launch.abort(ErrorCode::PeerClosed, false, false); - launch.mark_start_result_delivered(); - launch.admit_acknowledgement().unwrap(); - - assert!(matches!( - launch.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Failed(ErrorCode::PeerClosed)) - )); - launch.resolve_receipt(); - launch.process.cleanup(true); - } - - #[test] - fn acknowledgement_before_publication_is_protocol_violation() { - let launch = launch(StartResultPublication::NotStarted); - - assert!(matches!( - launch.admit_acknowledgement(), - Err(ErrorCode::ProtocolState) - )); - launch.resolve_receipt(); - launch.process.cleanup(true); - } - - #[test] - fn published_receipt_defers_process_finalization() { - let launch = launch(StartResultPublication::Delivered); - launch.abort(ErrorCode::PeerClosed, false, false); - - assert_eq!(launch.runner_finished(false), None); - assert!(matches!( - launch.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - launch.process.cleanup(true); - } - - #[test] - fn receipt_resolution_completes_process_start() { - let launch = starting_launch(StartResultPublication::Delivered); - let thread_id = launch.process.create_thread().unwrap(); - launch.process.mark_start_ready(Some(thread_id)).unwrap(); - launch.process.commit_start().unwrap(); - launch.state.lock().unwrap().phase = ChildLaunchPhase::Committed; - - launch.resolve_receipt(); - launch.resolve_receipt(); - - let state = launch.state.lock().unwrap(); - assert!(matches!(state.phase, ChildLaunchPhase::StartComplete)); - assert!(!state.abnormal); - drop(state); - assert_eq!(launch.process.exit_thread(thread_id), Ok(())); - launch.process.cleanup(true); - } - - #[test] - fn process_ready_waits_until_process_start_is_complete() { - let launch = starting_launch(StartResultPublication::Delivered); - let thread_id = launch.process.create_thread().unwrap(); - let waiting = Arc::clone(&launch); - let (sender, receiver) = mpsc::sync_channel(1); - let worker = std::thread::spawn(move || { - sender - .send(waiting.ready_and_wait(Some(thread_id))) - .unwrap(); - }); - - let mut state = launch.state.lock().unwrap(); - while !matches!(state.phase, ChildLaunchPhase::Ready { .. }) { - state = launch.changed.wait(state).unwrap(); - } - launch.process.commit_start().unwrap(); - state.phase = ChildLaunchPhase::Committed; - launch.changed.notify_all(); - drop(state); - assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); - - launch.resolve_receipt(); - assert_eq!( - receiver.recv_timeout(Duration::from_secs(1)).unwrap(), - Ok(()) - ); - worker.join().unwrap(); - assert_eq!(launch.process.exit_thread(thread_id), Ok(())); - launch.process.cleanup(true); - } - - #[test] - fn acknowledgement_admission_preserves_the_publication_deadline() { - let launch = launch(StartResultPublication::Publishing); - launch.admit_acknowledgement().unwrap(); - - let _expiration = launch.expire_initial_receipt_deadline().unwrap(); - let state = launch.state.lock().unwrap(); - assert!(matches!( - state.phase, - ChildLaunchPhase::Aborted(ErrorCode::Internal) - )); - assert!(state.abnormal); - drop(state); - assert_eq!(launch.complete_timeout_callback(), None); - launch.process.cleanup(true); - } - - #[test] - fn receipt_timeout_defers_finalization_until_association_drain() { - let launch = launch(StartResultPublication::Delivered); - - assert_eq!(launch.runner_finished(false), None); - let _expiration = launch.expire_initial_receipt_deadline().unwrap(); - assert!(matches!( - launch.resolve_receipt(), - ReceiptResolution::DeferredToDrain - )); - assert_eq!(launch.complete_timeout_callback(), None); - launch.begin_receipt_drain(); - assert_eq!(launch.finish_receipt_drain(), Some(false)); - launch.process.cleanup(true); - } - - #[test] - fn unpublished_launch_waits_for_receipt_drain_before_finalization() { - let launch = launch(StartResultPublication::NotStarted); - - launch.begin_receipt_drain(); - assert_eq!(launch.runner_finished(false), None); - assert_eq!(launch.finish_receipt_drain(), Some(false)); - launch.process.cleanup(true); - } - - #[test] - fn deferred_finalization_uses_the_latest_abnormal_disposition() { - let launch = launch(StartResultPublication::Publishing); - launch.admit_acknowledgement().unwrap(); - - assert_eq!(launch.runner_finished(false), None); - let _expiration = launch.expire_initial_receipt_deadline().unwrap(); - assert_eq!(launch.complete_timeout_callback(), None); - launch.begin_receipt_drain(); - assert_eq!(launch.finish_receipt_drain(), Some(true)); - launch.process.cleanup(false); - } - - #[test] - fn acknowledgement_send_does_not_steal_a_timed_out_receipt_from_drain() { - let (broker, launch) = launch_with_broker(StartResultPublication::Delivered); - let token = ProcessStartToken(7); - let parent_id = launch.parent_id; - let children = RunnerChildren { - broker, - config: RunnerConfig::new(PathBuf::new(), Vec::new()), - state: Mutex::new(RunnerChildrenState { - launches: vec![(token, Arc::clone(&launch))], - associations: Vec::new(), - active_instances: 1, - active_watchdogs: 0, - }), - drained: Condvar::new(), - }; - - assert_eq!(launch.runner_finished(false), None); - let _expiration = launch.expire_initial_receipt_deadline().unwrap(); - children.response_sent( - parent_id, - &BrokerOperation::AcknowledgeProcessStart(token), - &BrokerResult::ProcessStartAcknowledged, - ); - - assert!(children.find_launch(token).is_some()); - assert_eq!(launch.complete_timeout_callback(), None); - let draining = children.association_ending(parent_id); - assert_eq!(draining.len(), 1); - children.association_ended(parent_id, draining); - assert!(children.find_launch(token).is_none()); - } - - #[cfg(target_os = "linux")] - #[test] - fn shutdown_can_terminate_while_the_launch_owner_waits() { - use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; - use std::process::Command; - - let child = Arc::new(Mutex::new( - Command::new("sh") - .args(["-c", "exec sleep 30"]) - .spawn() - .unwrap(), - )); - let shutdown = RunnerShutdown { - runner: Arc::clone(&child), - state: Mutex::new(RunnerShutdownState::Active), - changed: Condvar::new(), - termination_dispatched: AtomicBool::new(false), - }; - let waiting = Arc::clone(&child); - let (finished, completion) = mpsc::sync_channel(1); - let waiter = std::thread::spawn(move || { - let status = wait_for_runner_exit(&waiting).unwrap(); - finished.send(status).unwrap(); - }); - - shutdown.shutdown(); - assert!(shutdown.termination_was_dispatched()); - assert!( - !completion - .recv_timeout(Duration::from_secs(1)) - .unwrap() - .success() - ); - waiter.join().unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn shutdown_after_observed_exit_records_no_termination() { - use super::{RunnerShutdown, RunnerShutdownState, wait_for_runner_exit}; - use std::process::Command; - - let child = Arc::new(Mutex::new( + let runner = Arc::new(Mutex::new( Command::new("sh").args(["-c", "exit 1"]).spawn().unwrap(), )); let shutdown = RunnerShutdown { - runner: Arc::clone(&child), + runner: Arc::clone(&runner), state: Mutex::new(RunnerShutdownState::Active), changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), @@ -2872,7 +448,7 @@ mod tests { shutdown.retire(); assert!(!shutdown.termination_was_dispatched()); - assert!(!wait_for_runner_exit(&child).unwrap().success()); + assert!(!wait_for_runner_exit(&runner).unwrap().success()); } #[cfg(target_os = "linux")] diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index db2d34548f..dec0a59cae 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -15,7 +15,9 @@ use litebox_broker_transport_linux_userland::unix_socket::{ UnixStreamHostSetupChannel, validate_peer_process, }; -use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited}; +use super::{ + RunnerProcessManager, RunnerStartup, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited, +}; use crate::runtime::{AssociationFailureCause, AssociationRunResult}; pub(super) struct PlatformRunnerEndpoint { @@ -46,30 +48,16 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, runner: &Arc>, - children: Arc, - ) -> AssociationRunResult { - serve_runner_process( - self.listener - .as_ref() - .expect("a live runner instance must own its control listener"), - runner, - children, - ) - } - - pub(super) fn serve_child( - &mut self, - runner: &Arc>, - child: ChildRunner, - children: Arc, + startup: Option, + process_manager: Arc, ) -> AssociationRunResult { - serve_child_runner_process( + serve_association( self.listener .as_ref() .expect("a live runner instance must own its control listener"), runner, - child, - children, + startup, + process_manager, ) } @@ -79,61 +67,27 @@ impl PlatformRunnerEndpoint { } } -fn serve_runner_process( - control_listener: &UnixListener, - runner: &Arc>, - children: Arc, -) -> AssociationRunResult { - let (control_channel, setup_deadline) = match accept_control_channel(control_listener, runner) { - Ok(connection) => connection, - Err(error) => { - let failure_cause = if runner_has_exited(runner).unwrap_or(false) { - AssociationFailureCause::RunnerExit - } else { - AssociationFailureCause::Other - }; - return AssociationRunResult { - result: Err(error), - process: None, - panicked: false, - abnormal: failure_cause == AssociationFailureCause::Other, - failure_cause, - }; - } - }; - crate::runtime::serve_runner_association( - None, - control_channel, - || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), - MemfdSharedMemory::create_control_ring, - |channel, shared_memory, control_memory| { - channel.send_memfd(shared_memory, Some(setup_deadline))?; - channel.send_memfd(control_memory, Some(setup_deadline))?; - Ok(()) - }, - UnixStreamHostSetupChannel::into_active, - children, - ) -} - -fn serve_child_runner_process( +fn serve_association( control_listener: &UnixListener, runner: &Arc>, - child: ChildRunner, - children: Arc, + startup: Option, + process_manager: Arc, ) -> AssociationRunResult { + let is_started_process = startup.is_some(); let (control_channel, setup_deadline) = match accept_control_channel(control_listener, runner) { Ok(connection) => connection, Err(error) => { let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit - } else if matches!( - error.kind(), - std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - ) { + } else if is_started_process + && matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + ) + { AssociationFailureCause::PeerClosed } else { AssociationFailureCause::Other @@ -147,8 +101,8 @@ fn serve_child_runner_process( }; } }; - let mut result = crate::runtime::serve_runner_association( - Some(child), + let mut result = crate::runtime::serve_out_of_process_runner_association( + startup, control_channel, || MemfdSharedMemory::create(SHARED_BUFFER_POOL_SIZE), MemfdSharedMemory::create_control_ring, @@ -158,9 +112,10 @@ fn serve_child_runner_process( Ok(()) }, UnixStreamHostSetupChannel::into_active, - children, + process_manager, ); - if result.failure_cause == AssociationFailureCause::Other + if is_started_process + && result.failure_cause == AssociationFailureCause::Other && result .result .as_ref() diff --git a/litebox_broker_userland/src/runner/process_start.rs b/litebox_broker_userland/src/runner/process_start.rs new file mode 100644 index 0000000000..23dac8c1d2 --- /dev/null +++ b/litebox_broker_userland/src/runner/process_start.rs @@ -0,0 +1,2474 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Coordination for broker-requested runner process starts. + +use std::io::{Error as IoError, Result as IoResult}; +use std::process::ExitStatus; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; +use litebox_broker_host::{RequestFailure, copy_shared_buffer}; +use litebox_broker_protocol::error::ErrorCode; +use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessStartToken, ProcessStartupData, StartedProcess, +}; +use litebox_broker_protocol::{ProcessId, ThreadId}; +use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; + +use super::{ + PROCESS_EXIT_OBSERVATION_TIMEOUT, RunnerConfig, RunnerInstance, RunnerShutdown, + runner_exit_code_is_crash, runner_exit_code_is_expected_shutdown, runner_exit_signal, + runner_signal_is_abnormal, wait_for_runner_exit, +}; +use crate::runtime::AssociationFailureCause; + +const PROCESS_START_RECEIPT_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT: Duration = Duration::from_secs(5); +const PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_PENDING_PROCESS_STARTS: usize = crate::WORKER_COUNT - 1; +const _: () = assert!(MAX_PENDING_PROCESS_STARTS > 0); +const _: () = + assert!(crate::runtime::PROCESS_START_CONTROL_WORKER_COUNT > MAX_PENDING_PROCESS_STARTS); +const _: () = + assert!(crate::runtime::PROCESS_START_CONTROL_QUEUE_CAPACITY >= MAX_PENDING_PROCESS_STARTS); + +/// Shared coordination for runner process starts and active associations. +pub(crate) struct RunnerProcessManager { + broker: BrokerCore, + process_start_config: RunnerConfig, + state: Mutex, + drained: Condvar, +} + +/// Startup context for a runner whose broker process was created by its parent. +pub(crate) struct RunnerStartup { + process: Arc, + start: Arc, + data: ProcessStartupData, +} + +impl RunnerStartup { + pub(crate) fn into_process_and_data(self) -> (Arc, ProcessStartupData) { + (self.process, self.data) + } +} + +struct RunnerCompletion { + result: IoResult, + runner_success: Option, + runner_signal: Option, + runner_exit_code: Option, + termination_provenance: TerminationProvenance, + association_panicked: bool, + shutdown_observation_failed: bool, +} + +#[derive(Clone, Copy, Default)] +struct TerminationProvenance(u8); + +impl TerminationProvenance { + const BROKER_TERMINATION: u8 = 1; + const REPORTED_START_FAILURE: u8 = 2; + + const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { + let mut value = 0; + if broker_termination { + value |= Self::BROKER_TERMINATION; + } + if reported_start_failure { + value |= Self::REPORTED_START_FAILURE; + } + Self(value) + } + + const fn broker_termination(self) -> bool { + self.0 & Self::BROKER_TERMINATION != 0 + } + + const fn reported_start_failure(self) -> bool { + self.0 & Self::REPORTED_START_FAILURE != 0 + } +} + +struct RunnerProcessManagerInner { + starts: Vec<(ProcessStartToken, Arc)>, + associations: Vec<(ProcessId, AssociationFailure)>, + active_instances: usize, + active_watchdogs: usize, +} + +pub(crate) type AssociationFailure = Arc; + +pub(crate) struct ProcessStartDrain { + starts: Vec>, +} + +struct ProcessStart { + parent_id: ProcessId, + process: Arc, + state: Mutex, + changed: Condvar, +} + +#[derive(Clone, Copy)] +enum ProcessStartPhase { + Starting, + Ready { initial_thread_id: Option }, + Committing, + Committed, + CompletingStart, + StartComplete, + Aborted(ErrorCode), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum StartResultPublication { + NotStarted, + Publishing, + Delivered, +} + +struct ProcessStartInner { + phase: ProcessStartPhase, + publication: StartResultPublication, + shutdown: Option>, + association_failure: Option, + shutdown_request: ShutdownRequest, + abnormal: bool, + receipt: ReceiptState, + resolution_watchdog: DeadlineState, + acknowledgement_publication_watchdog: DeadlineState, + start_failure_publication_watchdog: DeadlineState, + active_control_callbacks: usize, + runner_finished: bool, + finalization_taken: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ShutdownRequest { + None, + Expected, + ExpectedStartFailurePending, + ExpectedStartFailure, + Unexpected, +} + +impl ShutdownRequest { + const fn was_expected(self) -> bool { + matches!( + self, + Self::Expected | Self::ExpectedStartFailurePending | Self::ExpectedStartFailure + ) + } + + const fn expected_start_failure_was_reported(self) -> bool { + matches!( + self, + Self::ExpectedStartFailurePending | Self::ExpectedStartFailure + ) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ReceiptState { + AwaitingAcknowledgement, + AcknowledgementAdmitted, + TimeoutPending, + Draining, + Resolved, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum DeadlineState { + Unarmed, + Armed(Instant), + Disarmed, + Fired, +} + +enum ProcessStartAcknowledgement { + Acknowledged, + Failed(ErrorCode), +} + +struct ReceiptExpiration { + shutdown: Option>, + association_failure: Option, + fail_parent: bool, + commit_supervision_deadline: Option, +} + +enum ReceiptResolution { + Resolved(Option), + DeferredToDrain, +} + +impl RunnerInstance { + fn run_started_process_to_completion( + mut self, + startup: RunnerStartup, + process_manager: Arc, + ) -> RunnerCompletion { + let start = Arc::clone(&startup.start); + start.install_shutdown(Arc::clone(&self.shutdown)); + let association_result = self + .endpoint + .serve(&self.runner, Some(startup), process_manager); + self.endpoint.close(); + let shutdown_request = start.shutdown_request(); + let shutdown_was_expected = shutdown_request.was_expected(); + if association_result.abnormal { + start.mark_abnormal(); + } + let runner_exited = if !shutdown_was_expected + && matches!( + association_result.failure_cause, + AssociationFailureCause::None | AssociationFailureCause::PeerClosed + ) + && !association_result.panicked + { + self.shutdown + .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) + } else { + self.shutdown.has_exited() + }; + let shutdown_observation_failed = match runner_exited { + Ok(true) => { + if association_result.failure_cause == AssociationFailureCause::Other { + start.mark_abnormal(); + } + false + } + Ok(false) => { + if !shutdown_was_expected + || association_result.failure_cause == AssociationFailureCause::Other + { + start.mark_abnormal(); + } + start.mark_shutdown_expected(); + self.shutdown.shutdown(); + false + } + Err(_) => { + start.mark_abnormal(); + self.shutdown.shutdown(); + true + } + }; + self.shutdown.retire(); + let runner_status = wait_for_runner_exit(&self.runner); + if runner_status.is_err() { + start.mark_abnormal(); + } + let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); + let runner_signal = runner_status + .as_ref() + .ok() + .copied() + .and_then(runner_exit_signal); + let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); + let termination_provenance = TerminationProvenance::new( + self.shutdown.termination_was_dispatched(), + shutdown_request.expected_start_failure_was_reported(), + ); + let result = runner_status.and_then(|status| { + association_result.result?; + Ok(status) + }); + RunnerCompletion { + result, + runner_success, + runner_signal, + runner_exit_code, + termination_provenance, + association_panicked: association_result.panicked, + shutdown_observation_failed, + } + } +} + +impl RunnerProcessManager { + pub(super) fn new(process_start_config: RunnerConfig, broker: BrokerCore) -> Arc { + Arc::new(Self { + broker, + process_start_config, + state: Mutex::new(RunnerProcessManagerInner { + starts: Vec::new(), + associations: Vec::new(), + active_instances: 0, + active_watchdogs: 0, + }), + drained: Condvar::new(), + }) + } + + pub(crate) fn broker(&self) -> BrokerCore { + self.broker.clone() + } + + pub(crate) fn handle_operation( + self: &Arc, + process: &BrokerProcess, + operation: &BrokerOperation, + shared_buffers: &SharedBufferPool, + ) -> Option> { + match operation { + BrokerOperation::StartProcess(request) => Some( + copy_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) + .and_then(|bootstrap| { + self.start_process( + process, + request.format, + request.version, + bootstrap, + request.inherited_objects, + ) + .map_err(process_extension_error) + }) + .map(BrokerResult::ProcessStarted), + ), + BrokerOperation::AcknowledgeProcessStart(token) => { + Some(match self.resolve_acknowledgement(process.id(), *token) { + Ok(ProcessStartAcknowledgement::Acknowledged) => { + Ok(BrokerResult::ProcessStartAcknowledged) + } + Ok(ProcessStartAcknowledgement::Failed(error)) => { + Ok(BrokerResult::ProcessStartFailed(error)) + } + Err(error) => Err(RequestFailure::Abort(error)), + }) + } + BrokerOperation::ReportProcessReady(initial_thread_id) => Some( + self.process_ready(process.id(), *initial_thread_id) + .map(|()| BrokerResult::ProcessReady) + .map_err(process_extension_error), + ), + BrokerOperation::ReportProcessStartFailure(error) => Some( + self.report_process_start_failure(process.id(), *error) + .map(|()| BrokerResult::ProcessStartFailed(*error)) + .map_err(process_extension_error), + ), + _ => None, + } + } + + pub(crate) fn response_sent( + &self, + process_id: ProcessId, + operation: &BrokerOperation, + result: &BrokerResult, + ) { + match (operation, result) { + (_, BrokerResult::ProcessStarted(started)) => { + let Some(start) = self.find_start(started.token) else { + return; + }; + if start.parent_id == process_id { + start.mark_start_result_delivered(); + } + } + ( + BrokerOperation::AcknowledgeProcessStart(token), + BrokerResult::ProcessStartAcknowledged | BrokerResult::ProcessStartFailed(_), + ) => { + let Some(start) = self.find_start(*token) else { + return; + }; + if start.parent_id != process_id { + return; + } + if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { + self.remove_start(*token); + if let Some(abnormal) = finalization { + self.finish_process_start(&start, abnormal); + } + } + } + ( + BrokerOperation::ReportProcessStartFailure(error), + BrokerResult::ProcessStartFailed(reported), + ) if error == reported => { + if let Some(start) = self.find_process_start(process_id) { + start.start_failure_response_sent(*error); + } + } + _ => {} + } + } + + fn start_process( + self: &Arc, + parent: &BrokerProcess, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + bootstrap: Vec, + requested_inherited_objects: InheritedProcessObjects, + ) -> Result { + if !parent.is_running() { + return Err(ErrorCode::ProtocolState); + } + let (process, inherited_objects) = parent + .create_child(requested_inherited_objects.as_slice()) + .map_err(ErrorCode::from)?; + let inherited_objects = InheritedProcessObjects::new(&inherited_objects) + .expect("child handle count must match the bounded inheritance request"); + let process_id = process.id(); + let start = Arc::new(ProcessStart { + parent_id: parent.id(), + process: Arc::clone(&process), + state: Mutex::new(ProcessStartInner { + phase: ProcessStartPhase::Starting, + publication: StartResultPublication::NotStarted, + shutdown: None, + association_failure: None, + shutdown_request: ShutdownRequest::None, + abnormal: false, + receipt: ReceiptState::AwaitingAcknowledgement, + resolution_watchdog: DeadlineState::Unarmed, + acknowledgement_publication_watchdog: DeadlineState::Unarmed, + start_failure_publication_watchdog: DeadlineState::Unarmed, + active_control_callbacks: 0, + runner_finished: false, + finalization_taken: false, + }), + changed: Condvar::new(), + }); + let token = match (|| { + loop { + let mut token = [0; 8]; + getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; + let token = ProcessStartToken(u64::from_ne_bytes(token)); + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state + .starts + .try_reserve(1) + .map_err(|_| ErrorCode::OutOfMemory)?; + if parent.is_cancellation_requested() { + return Err(ErrorCode::PeerClosed); + } + if state.starts.len() >= MAX_PENDING_PROCESS_STARTS { + return Err(ErrorCode::ResourceExhausted); + } + if state + .starts + .iter() + .any(|(candidate, _)| *candidate == token) + { + continue; + } + let active_instances = state + .active_instances + .checked_add(1) + .ok_or(ErrorCode::ResourceExhausted)?; + state.starts.push((token, Arc::clone(&start))); + state.active_instances = active_instances; + return Ok(token); + } + })() { + Ok(token) => token, + Err(error) => { + process.cleanup(true); + return Err(error); + } + }; + + let process_manager = Arc::clone(self); + let config = self.process_start_config.clone(); + let thread_start = Arc::clone(&start); + let thread = std::thread::Builder::new() + .name(format!("litebox-runner-{}", process_id.0)) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + RunnerInstance::start(config).map(|instance| { + instance.run_started_process_to_completion( + RunnerStartup { + process, + start: Arc::clone(&thread_start), + data: ProcessStartupData { + format, + version, + payload: bootstrap, + inherited_objects, + }, + }, + Arc::clone(&process_manager), + ) + }) + })); + match outcome { + Ok(Ok(result)) => { + process_manager.runner_finished(token, &thread_start, result, false); + } + Ok(Err(error)) => process_manager.runner_finished( + token, + &thread_start, + RunnerCompletion { + result: Err(error), + runner_success: None, + runner_signal: None, + runner_exit_code: None, + termination_provenance: TerminationProvenance::default(), + association_panicked: false, + shutdown_observation_failed: false, + }, + false, + ), + Err(_) => process_manager.runner_finished( + token, + &thread_start, + RunnerCompletion { + result: Err(IoError::other("runner process thread panicked")), + runner_success: None, + runner_signal: None, + runner_exit_code: None, + termination_provenance: TerminationProvenance::default(), + association_panicked: false, + shutdown_observation_failed: false, + }, + true, + ), + } + }); + if thread.is_err() { + self.remove_start(token); + start.process.cleanup(true); + self.finish_instance(); + return Err(ErrorCode::OutOfMemory); + } + drop(thread); + + let initial_thread_id = start.wait_until_ready()?; + let receipt_deadline = start.begin_start_result_publication()?; + if self + .arm_initial_receipt_deadline(token, &start, receipt_deadline) + .is_err() + { + if let Some(expiration) = start.expire_initial_receipt_deadline() { + self.apply_receipt_expiration( + token, + &start, + expiration, + self.find_association_failure(start.parent_id), + ); + } + return Err(ErrorCode::OutOfMemory); + } + Ok(StartedProcess { + token, + process_id, + initial_thread_id, + }) + } + + pub(crate) fn admit_acknowledgement( + self: &Arc, + parent_id: ProcessId, + token: ProcessStartToken, + ) -> Result<(), ErrorCode> { + let start = self.find_start(token).ok_or(ErrorCode::UnknownObject)?; + if start.parent_id != parent_id { + return Err(ErrorCode::UnknownObject); + } + start.admit_acknowledgement()?; + if self + .arm_internal_resolution_watchdog(token, &start) + .is_err() + { + if let Some(expiration) = start.fail_internal_resolution_watchdog() { + self.apply_receipt_expiration( + token, + &start, + expiration, + self.find_association_failure(start.parent_id), + ); + } + return Err(ErrorCode::Internal); + } + if self + .arm_acknowledgement_publication_watchdog(token, &start) + .is_err() + { + if let Some(expiration) = start.fail_internal_resolution_watchdog() { + self.apply_receipt_expiration( + token, + &start, + expiration, + self.find_association_failure(start.parent_id), + ); + } + return Err(ErrorCode::Internal); + } + Ok(()) + } + + fn resolve_acknowledgement( + &self, + parent_id: ProcessId, + token: ProcessStartToken, + ) -> Result { + let start = self.find_start(token).ok_or(ErrorCode::PeerClosed)?; + if start.parent_id != parent_id { + return Err(ErrorCode::UnknownObject); + } + start.resolve_acknowledgement() + } + + fn process_ready( + &self, + process_id: ProcessId, + initial_thread_id: Option, + ) -> Result<(), ErrorCode> { + let start = self + .find_process_start(process_id) + .ok_or(ErrorCode::PeerClosed)?; + start.ready_and_wait(initial_thread_id) + } + + fn report_process_start_failure( + self: &Arc, + process_id: ProcessId, + error: ErrorCode, + ) -> Result<(), ErrorCode> { + let (token, start) = self + .find_process_start_entry(process_id) + .ok_or(ErrorCode::PeerClosed)?; + start.report_start_failure(error)?; + if self + .arm_start_failure_publication_watchdog(token, &start) + .is_err() + { + if let Some(expiration) = start.fail_start_failure_publication_watchdog() { + self.apply_receipt_expiration(token, &start, expiration, None); + } + return Err(ErrorCode::Internal); + } + Ok(()) + } + + pub(crate) fn association_ending(&self, process_id: ProcessId) -> ProcessStartDrain { + let draining = self + .state + .lock() + .expect("runner process manager state mutex poisoned") + .starts + .iter() + .filter(|(_, start)| start.parent_id == process_id) + .map(|(_, start)| Arc::clone(start)) + .collect::>(); + for start in &draining { + start.abort(ErrorCode::PeerClosed, false, true); + start.begin_receipt_drain(); + } + + let process_start = { + self.state + .lock() + .expect("runner process manager state mutex poisoned") + .starts + .iter() + .find_map(|(_, start)| { + (start.process.id() == process_id).then(|| Arc::clone(start)) + }) + }; + if let Some(start) = process_start { + start.association_closed(); + } + ProcessStartDrain { starts: draining } + } + + pub(crate) fn association_ended(&self, process_id: ProcessId, draining: ProcessStartDrain) { + self.unregister_association(process_id); + for start in draining.starts { + self.remove_start_by_process_id(start.process.id()); + if let Some(abnormal) = start.finish_receipt_drain() { + self.finish_process_start(&start, abnormal); + } + } + } + + pub(crate) fn register_association( + &self, + process_id: ProcessId, + failure: AssociationFailure, + ) -> IoResult<()> { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state + .associations + .try_reserve(1) + .map_err(|_| IoError::other("failed to reserve broker association registration"))?; + if state + .associations + .iter() + .any(|(candidate, _)| *candidate == process_id) + { + return Err(IoError::other( + "a broker process already has a live association", + )); + } + state.associations.push((process_id, failure)); + Ok(()) + } + + pub(crate) fn install_process_association_failure( + &self, + process_id: ProcessId, + failure: AssociationFailure, + ) { + if let Some(start) = self.find_process_start(process_id) { + start.install_association_failure(failure); + } + } + + fn unregister_association(&self, process_id: ProcessId) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + if let Some(index) = state + .associations + .iter() + .position(|(candidate, _)| *candidate == process_id) + { + state.associations.swap_remove(index); + } + } + + fn find_association_failure(&self, process_id: ProcessId) -> Option { + self.state + .lock() + .expect("runner process manager state mutex poisoned") + .associations + .iter() + .find_map(|(candidate, failure)| { + (*candidate == process_id).then(|| Arc::clone(failure)) + }) + } + + fn arm_initial_receipt_deadline( + self: &Arc, + token: ProcessStartToken, + start: &Arc, + deadline: Instant, + ) -> Result<(), ()> { + let start = Arc::clone(start); + let parent_failure = self.find_association_failure(start.parent_id); + self.spawn_watchdog( + format!("litebox-start-receipt-{}", start.process.id().0), + move |process_manager| { + let Some(expiration) = start.wait_for_initial_receipt_deadline(deadline) else { + return; + }; + process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); + }, + ) + } + + fn arm_internal_resolution_watchdog( + self: &Arc, + token: ProcessStartToken, + start: &Arc, + ) -> Result<(), ()> { + let start = Arc::clone(start); + let parent_failure = self.find_association_failure(start.parent_id); + self.spawn_watchdog( + format!("litebox-start-resolution-{}", start.process.id().0), + move |process_manager| { + let Some(expiration) = start.wait_for_internal_resolution_timeout() else { + return; + }; + process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); + }, + ) + } + + fn arm_acknowledgement_publication_watchdog( + self: &Arc, + token: ProcessStartToken, + start: &Arc, + ) -> Result<(), ()> { + let start = Arc::clone(start); + let parent_failure = self.find_association_failure(start.parent_id); + self.spawn_watchdog( + format!("litebox-start-ack-publication-{}", start.process.id().0), + move |process_manager| { + let Some(expiration) = start.wait_for_acknowledgement_publication_timeout() else { + return; + }; + process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); + }, + ) + } + + fn arm_start_failure_publication_watchdog( + self: &Arc, + token: ProcessStartToken, + start: &Arc, + ) -> Result<(), ()> { + let start = Arc::clone(start); + self.spawn_watchdog( + format!("litebox-start-failure-publication-{}", start.process.id().0), + move |process_manager| { + let Some(expiration) = start.wait_for_start_failure_publication_timeout() else { + return; + }; + process_manager.apply_receipt_expiration(token, &start, expiration, None); + }, + ) + } + + fn spawn_watchdog( + self: &Arc, + name: String, + watchdog: impl FnOnce(&Arc) + Send + 'static, + ) -> Result<(), ()> { + { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state.active_watchdogs = state.active_watchdogs.checked_add(1).ok_or(())?; + } + let process_manager = Arc::clone(self); + if let Ok(thread) = std::thread::Builder::new().name(name).spawn(move || { + let _completion = WatchdogCompletion { + process_manager: Arc::clone(&process_manager), + }; + watchdog(&process_manager); + }) { + drop(thread); + Ok(()) + } else { + self.finish_watchdog(); + Err(()) + } + } + + fn apply_receipt_expiration( + &self, + token: ProcessStartToken, + start: &ProcessStart, + expiration: ReceiptExpiration, + parent_failure: Option, + ) { + let fail_parent = expiration.fail_parent; + let commit_supervision_deadline = expiration.commit_supervision_deadline; + if let Some(association_failure) = expiration.association_failure { + association_failure(); + } + if let Some(shutdown) = expiration.shutdown { + shutdown.shutdown(); + } + let parent_failed = if fail_parent { + if let Some(parent_failure) = parent_failure { + parent_failure(); + true + } else { + false + } + } else { + true + }; + if let Some(abnormal) = start.complete_timeout_callback() { + self.finish_process_start(start, abnormal); + } + if fail_parent && !parent_failed { + self.remove_start(token); + if let Some(abnormal) = start.finish_receipt_drain() { + self.finish_process_start(start, abnormal); + } + } + if let Some(deadline) = commit_supervision_deadline + && !start.wait_for_commit_resolution(deadline) + { + std::process::abort(); + } + } + + fn find_start(&self, token: ProcessStartToken) -> Option> { + self.state + .lock() + .expect("runner process manager state mutex poisoned") + .starts + .iter() + .find_map(|(candidate, start)| (*candidate == token).then(|| Arc::clone(start))) + } + + fn find_process_start(&self, process_id: ProcessId) -> Option> { + self.find_process_start_entry(process_id) + .map(|(_, start)| start) + } + + fn find_process_start_entry( + &self, + process_id: ProcessId, + ) -> Option<(ProcessStartToken, Arc)> { + self.state + .lock() + .expect("runner process manager state mutex poisoned") + .starts + .iter() + .find_map(|(token, start)| { + (start.process.id() == process_id).then(|| (*token, Arc::clone(start))) + }) + } + + fn remove_start(&self, token: ProcessStartToken) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + if let Some(index) = state + .starts + .iter() + .position(|(candidate, _)| *candidate == token) + { + state.starts.swap_remove(index); + } + } + + fn remove_start_by_process_id(&self, process_id: ProcessId) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + if let Some(index) = state + .starts + .iter() + .position(|(_, start)| start.process.id() == process_id) + { + state.starts.swap_remove(index); + } + } + + fn runner_finished( + &self, + token: ProcessStartToken, + start: &ProcessStart, + result: RunnerCompletion, + thread_panicked: bool, + ) { + let unexpected_runner_failure = result.runner_success == Some(false) + && result.runner_signal.is_none() + && !start.commit_was_claimed() + && !result.termination_provenance.reported_start_failure() + && !runner_exit_code_is_expected_shutdown( + result.runner_exit_code, + result.termination_provenance.broker_termination(), + ); + let unexpected_crash = runner_signal_is_abnormal( + result.runner_signal, + result.termination_provenance.broker_termination(), + ); + let abnormal = thread_panicked + || result.association_panicked + || result.shutdown_observation_failed + || unexpected_crash + || runner_exit_code_is_crash(result.runner_exit_code) + || unexpected_runner_failure; + if result.result.is_err() || result.runner_success != Some(true) { + start.abort(ErrorCode::PeerClosed, abnormal, false); + } else { + // Any clean host exit before commit still aborts creation. + start.abort(ErrorCode::PeerClosed, false, false); + } + if !start.retains_published_receipt() { + self.remove_start(token); + if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { + debug_assert!(finalization.is_none()); + } + } + if let Some(abnormal) = start.runner_finished(abnormal) { + self.finish_process_start(start, abnormal); + } + } + + fn finish_process_start(&self, start: &ProcessStart, abnormal: bool) { + start.process.cleanup(!abnormal); + self.finish_instance(); + } + + fn finish_instance(&self) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state.active_instances = state + .active_instances + .checked_sub(1) + .expect("runner instance count must remain balanced"); + if state.active_instances == 0 && state.active_watchdogs == 0 { + self.drained.notify_all(); + } + } + + fn finish_watchdog(&self) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state.active_watchdogs = state + .active_watchdogs + .checked_sub(1) + .expect("runner watchdog count must remain balanced"); + if state.active_instances == 0 && state.active_watchdogs == 0 { + self.drained.notify_all(); + } + } + + pub(super) fn wait_for_drain(&self) { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + while state.active_instances != 0 || state.active_watchdogs != 0 { + state = self + .drained + .wait(state) + .expect("runner process manager state mutex poisoned"); + } + } +} + +struct WatchdogCompletion { + process_manager: Arc, +} + +impl Drop for WatchdogCompletion { + fn drop(&mut self) { + self.process_manager.finish_watchdog(); + } +} + +const fn process_extension_error(error: ErrorCode) -> RequestFailure { + match error { + ErrorCode::PolicyDenied + | ErrorCode::UnknownObject + | ErrorCode::InvalidRights + | ErrorCode::ResourceExhausted + | ErrorCode::WouldBlock + | ErrorCode::PeerClosed + | ErrorCode::OutOfMemory + | ErrorCode::UnsupportedOperation => RequestFailure::Respond(error), + _ => RequestFailure::Abort(error), + } +} + +const fn process_start_failure_is_expected(error: ErrorCode) -> bool { + matches!( + error, + ErrorCode::UnsupportedOperation + | ErrorCode::PolicyDenied + | ErrorCode::InvalidRights + | ErrorCode::ResourceExhausted + | ErrorCode::WouldBlock + | ErrorCode::OutOfMemory + ) +} + +impl ProcessStart { + fn wait_until_ready(&self) -> Result, ErrorCode> { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + match state.phase { + ProcessStartPhase::Starting => { + state = self + .changed + .wait(state) + .expect("process start mutex poisoned"); + } + ProcessStartPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), + ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete => { + return Err(ErrorCode::ProtocolState); + } + ProcessStartPhase::Aborted(error) => return Err(error), + } + } + } + + fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { + self.process + .mark_start_ready(initial_thread_id) + .map_err(|error| match error { + BrokerError::UnknownObject => ErrorCode::ProtocolState, + error => ErrorCode::from(error), + })?; + let mut state = self.state.lock().expect("process start mutex poisoned"); + if !matches!(state.phase, ProcessStartPhase::Starting) { + return Err(match state.phase { + ProcessStartPhase::Aborted(error) => error, + _ => ErrorCode::ProtocolState, + }); + } + state.phase = ProcessStartPhase::Ready { initial_thread_id }; + self.changed.notify_all(); + loop { + match state.phase { + ProcessStartPhase::StartComplete if state.active_control_callbacks == 0 => { + return Ok(()); + } + ProcessStartPhase::Ready { .. } + | ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete => { + state = self + .changed + .wait(state) + .expect("process start mutex poisoned"); + } + ProcessStartPhase::Aborted(error) => return Err(error), + ProcessStartPhase::Starting => unreachable!("ready state cannot regress"), + } + } + } + + fn begin_start_result_publication(&self) -> Result { + let mut state = self.state.lock().expect("process start mutex poisoned"); + match state.phase { + ProcessStartPhase::Ready { .. } + if state.publication == StartResultPublication::NotStarted => + { + state.publication = StartResultPublication::Publishing; + Ok(Instant::now() + PROCESS_START_RECEIPT_TIMEOUT) + } + ProcessStartPhase::Aborted(error) => Err(error), + _ => Err(ErrorCode::ProtocolState), + } + } + + fn report_start_failure(&self, error: ErrorCode) -> Result<(), ErrorCode> { + if !process_start_failure_is_expected(error) { + return Err(ErrorCode::ProtocolState); + } + let mut state = self.state.lock().expect("process start mutex poisoned"); + match state.phase { + ProcessStartPhase::Starting => {} + ProcessStartPhase::Aborted(error) => return Err(error), + _ => return Err(ErrorCode::ProtocolState), + } + state.phase = ProcessStartPhase::Aborted(error); + state.shutdown_request = ShutdownRequest::ExpectedStartFailurePending; + arm_deadline( + &mut state.start_failure_publication_watchdog, + PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, + ); + self.changed.notify_all(); + Ok(()) + } + + fn start_failure_response_sent(&self, error: ErrorCode) { + let (association_failure, shutdown) = { + let mut state = self.state.lock().expect("process start mutex poisoned"); + complete_deadline(&mut state.start_failure_publication_watchdog); + let actions = if matches!(state.phase, ProcessStartPhase::Aborted(cause) if cause == error) + && state.shutdown_request == ShutdownRequest::ExpectedStartFailurePending + { + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } else { + (None, None) + }; + self.changed.notify_all(); + actions + }; + if let Some(association_failure) = association_failure { + association_failure(); + } + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn admit_acknowledgement(&self) -> Result<(), ErrorCode> { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if state.publication == StartResultPublication::NotStarted { + return Err(ErrorCode::ProtocolState); + } + match state.receipt { + ReceiptState::AwaitingAcknowledgement => {} + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { + return Err(ErrorCode::PeerClosed); + } + ReceiptState::AcknowledgementAdmitted => return Err(ErrorCode::UnknownObject), + } + state.receipt = ReceiptState::AcknowledgementAdmitted; + if state.publication == StartResultPublication::Delivered + && matches!( + state.phase, + ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) + ) + { + arm_deadline( + &mut state.resolution_watchdog, + PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, + ); + self.changed.notify_all(); + } + Ok(()) + } + + fn resolve_acknowledgement(&self) -> Result { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + match state.receipt { + ReceiptState::AcknowledgementAdmitted => {} + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { + return Err(ErrorCode::PeerClosed); + } + ReceiptState::AwaitingAcknowledgement => return Err(ErrorCode::ProtocolState), + } + if state.publication == StartResultPublication::NotStarted { + return Err(ErrorCode::ProtocolState); + } + match (state.phase, state.publication) { + ( + ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_), + StartResultPublication::Publishing, + ) => { + state = self + .changed + .wait(state) + .expect("process start mutex poisoned"); + } + (ProcessStartPhase::Ready { .. }, StartResultPublication::Delivered) => { + debug_assert!(matches!(state.resolution_watchdog, DeadlineState::Armed(_))); + state.phase = ProcessStartPhase::Committing; + drop(state); + let commit_result = self.process.commit_start().map_err(ErrorCode::from); + state = self.state.lock().expect("process start mutex poisoned"); + match commit_result { + Ok(()) => { + state.phase = ProcessStartPhase::Committed; + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Acknowledged); + } + Err(error) => { + state.phase = ProcessStartPhase::Aborted(error); + state.abnormal |= error == ErrorCode::Internal; + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Failed(error)); + } + } + } + (ProcessStartPhase::Aborted(error), StartResultPublication::Delivered) => { + debug_assert!(matches!( + state.resolution_watchdog, + DeadlineState::Armed(_) | DeadlineState::Fired + )); + complete_acknowledgement_resolution(&mut state); + self.changed.notify_all(); + return Ok(ProcessStartAcknowledgement::Failed(error)); + } + (ProcessStartPhase::Starting, _) => return Err(ErrorCode::ProtocolState), + ( + ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete, + _, + ) => { + return Err(ErrorCode::ProtocolState); + } + (_, StartResultPublication::NotStarted) => { + unreachable!("publication state was checked before phase dispatch") + } + } + } + } + + fn mark_start_result_delivered(&self) { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if state.publication == StartResultPublication::Publishing { + state.publication = StartResultPublication::Delivered; + if state.receipt == ReceiptState::AcknowledgementAdmitted + && matches!( + state.phase, + ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) + ) + { + arm_deadline( + &mut state.resolution_watchdog, + PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, + ); + } + self.changed.notify_all(); + } + } + + fn install_shutdown(&self, shutdown: Arc) { + let shutdown = { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.shutdown = Some(Arc::clone(&shutdown)); + (state.shutdown_request != ShutdownRequest::None).then_some(shutdown) + }; + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn install_association_failure(&self, failure: AssociationFailure) { + let failure = { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.association_failure = Some(Arc::clone(&failure)); + (state.shutdown_request != ShutdownRequest::None).then_some(failure) + }; + if let Some(failure) = failure { + failure(); + } + } + + fn abort(&self, error: ErrorCode, abnormal: bool, expected_shutdown: bool) { + let (association_failure, shutdown) = { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.abnormal |= abnormal; + match state.phase { + ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { + state.phase = ProcessStartPhase::Aborted(error); + let newly_requested = state.shutdown_request == ShutdownRequest::None; + if newly_requested { + state.shutdown_request = if expected_shutdown { + ShutdownRequest::Expected + } else { + ShutdownRequest::Unexpected + }; + } + self.changed.notify_all(); + ( + newly_requested + .then(|| state.association_failure.as_ref().map(Arc::clone)) + .flatten(), + state.shutdown.clone(), + ) + } + ProcessStartPhase::Aborted(_) => match state.shutdown_request { + ShutdownRequest::None => { + state.shutdown_request = if expected_shutdown { + ShutdownRequest::Expected + } else { + ShutdownRequest::Unexpected + }; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ShutdownRequest::ExpectedStartFailurePending => { + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ShutdownRequest::Expected + | ShutdownRequest::ExpectedStartFailure + | ShutdownRequest::Unexpected => (None, None), + }, + ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete => (None, None), + } + }; + if let Some(association_failure) = association_failure { + association_failure(); + } + if let Some(shutdown) = shutdown { + shutdown.shutdown(); + } + } + + fn association_closed(&self) { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.association_failure = None; + if matches!( + state.phase, + ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } + ) { + state.phase = ProcessStartPhase::Aborted(ErrorCode::PeerClosed); + self.changed.notify_all(); + } + } + + fn mark_shutdown_expected(&self) { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + } + + fn mark_abnormal(&self) { + self.state + .lock() + .expect("process start mutex poisoned") + .abnormal = true; + } + + fn shutdown_request(&self) -> ShutdownRequest { + self.state + .lock() + .expect("process start mutex poisoned") + .shutdown_request + } + + fn commit_was_claimed(&self) -> bool { + matches!( + self.state + .lock() + .expect("process start mutex poisoned") + .phase, + ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete + ) + } + + fn retains_published_receipt(&self) -> bool { + let state = self.state.lock().expect("process start mutex poisoned"); + state.publication != StartResultPublication::NotStarted + && state.receipt != ReceiptState::Resolved + } + + fn resolve_receipt(&self) -> ReceiptResolution { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if matches!( + state.receipt, + ReceiptState::TimeoutPending | ReceiptState::Draining + ) { + return ReceiptResolution::DeferredToDrain; + } + state.receipt = ReceiptState::Resolved; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + ReceiptResolution::Resolved(self.complete_process_start_and_take_finalization(state)) + } + + fn begin_receipt_drain(&self) { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if state.receipt != ReceiptState::Resolved { + state.receipt = ReceiptState::Draining; + if !matches!(state.phase, ProcessStartPhase::Committing) { + complete_deadline(&mut state.resolution_watchdog); + } + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + } + } + + fn finish_receipt_drain(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.receipt = ReceiptState::Resolved; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + self.changed.notify_all(); + self.complete_process_start_and_take_finalization(state) + } + + fn expire_initial_receipt_deadline(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + let (error, abnormal) = match (state.receipt, state.publication) { + (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), + (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { + (ErrorCode::Internal, true) + } + _ => return None, + }; + expire_start(&mut state, error, abnormal, true, &self.changed) + } + + fn wait_for_initial_receipt_deadline(&self, deadline: Instant) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + if !matches!( + (state.receipt, state.publication), + (ReceiptState::AwaitingAcknowledgement, _) + | ( + ReceiptState::AcknowledgementAdmitted, + StartResultPublication::Publishing + ) + ) { + return None; + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break; + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("process start mutex poisoned"); + state = next; + } + let (error, abnormal) = match (state.receipt, state.publication) { + (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), + (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { + (ErrorCode::Internal, true) + } + _ => return None, + }; + expire_start(&mut state, error, abnormal, true, &self.changed) + } + + fn wait_for_internal_resolution_timeout(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + match state.resolution_watchdog { + DeadlineState::Unarmed => { + state = self + .changed + .wait(state) + .expect("process start mutex poisoned"); + } + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.resolution_watchdog = DeadlineState::Fired; + return expire_internal_resolution(&mut state, &self.changed); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("process start mutex poisoned"); + state = next; + } + DeadlineState::Disarmed | DeadlineState::Fired => return None, + } + } + } + + fn wait_for_acknowledgement_publication_timeout(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + match state.acknowledgement_publication_watchdog { + DeadlineState::Unarmed => { + state = self + .changed + .wait(state) + .expect("process start mutex poisoned"); + } + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.acknowledgement_publication_watchdog = DeadlineState::Fired; + return expire_start( + &mut state, + ErrorCode::PeerClosed, + false, + true, + &self.changed, + ); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("process start mutex poisoned"); + state = next; + } + DeadlineState::Disarmed | DeadlineState::Fired => return None, + } + } + } + + fn wait_for_start_failure_publication_timeout(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + loop { + match state.start_failure_publication_watchdog { + DeadlineState::Armed(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + state.start_failure_publication_watchdog = DeadlineState::Fired; + return expire_start_failure_publication(&mut state, &self.changed); + } + let (next, _) = self + .changed + .wait_timeout(state, remaining) + .expect("process start mutex poisoned"); + state = next; + } + DeadlineState::Unarmed | DeadlineState::Disarmed | DeadlineState::Fired => { + return None; + } + } + } + } + + fn fail_start_failure_publication_watchdog(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + if !matches!( + state.start_failure_publication_watchdog, + DeadlineState::Armed(_) + ) { + return None; + } + state.start_failure_publication_watchdog = DeadlineState::Fired; + expire_start_failure_publication(&mut state, &self.changed) + } + + fn fail_internal_resolution_watchdog(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + expire_start(&mut state, ErrorCode::Internal, true, true, &self.changed) + } + + fn complete_timeout_callback(&self) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.active_control_callbacks = state + .active_control_callbacks + .checked_sub(1) + .expect("process-start timeout callback count must remain balanced"); + self.changed.notify_all(); + take_finalization(&mut state) + } + + fn wait_for_commit_resolution(&self, deadline: Instant) -> bool { + let mut state = self.state.lock().expect("process start mutex poisoned"); + while matches!(state.phase, ProcessStartPhase::Committing) { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return false; + } + let (next, wait_result) = self + .changed + .wait_timeout(state, remaining) + .expect("process start mutex poisoned"); + state = next; + if wait_result.timed_out() && matches!(state.phase, ProcessStartPhase::Committing) { + return false; + } + } + true + } + + fn complete_process_start_and_take_finalization( + &self, + mut state: MutexGuard<'_, ProcessStartInner>, + ) -> Option { + if !matches!(state.phase, ProcessStartPhase::Committed) { + return take_finalization(&mut state); + } + state.phase = ProcessStartPhase::CompletingStart; + state.active_control_callbacks = state + .active_control_callbacks + .checked_add(1) + .expect("process-start control callback count must remain bounded"); + drop(state); + + let completion_failed = self.process.complete_start().is_err(); + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.active_control_callbacks = state + .active_control_callbacks + .checked_sub(1) + .expect("process-start control callback count must remain balanced"); + state.abnormal |= completion_failed; + state.phase = ProcessStartPhase::StartComplete; + self.changed.notify_all(); + take_finalization(&mut state) + } + + fn runner_finished(&self, abnormal: bool) -> Option { + let mut state = self.state.lock().expect("process start mutex poisoned"); + state.abnormal |= abnormal; + state.runner_finished = true; + take_finalization(&mut state) + } +} + +fn expire_start( + state: &mut ProcessStartInner, + error: ErrorCode, + abnormal: bool, + fail_parent: bool, + changed: &Condvar, +) -> Option { + if matches!( + state.receipt, + ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved + ) { + return None; + } + state.active_control_callbacks += 1; + state.receipt = ReceiptState::TimeoutPending; + state.abnormal |= abnormal; + complete_deadline(&mut state.resolution_watchdog); + complete_deadline(&mut state.acknowledgement_publication_watchdog); + complete_deadline(&mut state.start_failure_publication_watchdog); + let (association_failure, shutdown) = match state.phase { + ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { + state.phase = ProcessStartPhase::Aborted(error); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } + ProcessStartPhase::Aborted(_) => { + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } else { + (None, None) + } + } + ProcessStartPhase::Committing + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete => (None, None), + }; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown, + association_failure, + fail_parent, + commit_supervision_deadline: None, + }) +} + +fn expire_start_failure_publication( + state: &mut ProcessStartInner, + changed: &Condvar, +) -> Option { + if !matches!(state.phase, ProcessStartPhase::Aborted(_)) + || state.shutdown_request != ShutdownRequest::ExpectedStartFailurePending + { + return None; + } + state.active_control_callbacks += 1; + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown: state.shutdown.clone(), + association_failure: state.association_failure.as_ref().map(Arc::clone), + fail_parent: false, + commit_supervision_deadline: None, + }) +} + +fn expire_internal_resolution( + state: &mut ProcessStartInner, + changed: &Condvar, +) -> Option { + let committing_drain = state.receipt == ReceiptState::Draining + && matches!(state.phase, ProcessStartPhase::Committing); + if state.receipt != ReceiptState::AcknowledgementAdmitted && !committing_drain { + return None; + } + state.active_control_callbacks += 1; + state.abnormal = true; + let (association_failure, shutdown, fail_parent, commit_supervision_deadline) = + match state.phase { + ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) => { + state.phase = ProcessStartPhase::Aborted(ErrorCode::Internal); + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + false, + None, + ) + } + ProcessStartPhase::Committing => { + state.receipt = ReceiptState::TimeoutPending; + complete_deadline(&mut state.acknowledgement_publication_watchdog); + ( + None, + None, + true, + Some(Instant::now() + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT), + ) + } + ProcessStartPhase::Starting + | ProcessStartPhase::Committed + | ProcessStartPhase::CompletingStart + | ProcessStartPhase::StartComplete => { + state.active_control_callbacks -= 1; + return None; + } + }; + changed.notify_all(); + Some(ReceiptExpiration { + shutdown, + association_failure, + fail_parent, + commit_supervision_deadline, + }) +} + +fn arm_deadline(state: &mut DeadlineState, timeout: Duration) { + if *state == DeadlineState::Unarmed { + *state = DeadlineState::Armed(Instant::now() + timeout); + } +} + +fn complete_deadline(state: &mut DeadlineState) { + if matches!(state, DeadlineState::Unarmed | DeadlineState::Armed(_)) { + *state = DeadlineState::Disarmed; + } +} + +fn complete_acknowledgement_resolution(state: &mut ProcessStartInner) { + complete_deadline(&mut state.resolution_watchdog); + if state.receipt == ReceiptState::AcknowledgementAdmitted { + arm_deadline( + &mut state.acknowledgement_publication_watchdog, + PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, + ); + } +} + +fn take_finalization(state: &mut ProcessStartInner) -> Option { + if state.finalization_taken + || state.active_control_callbacks != 0 + || state.receipt != ReceiptState::Resolved + { + return None; + } + if !state.runner_finished { + return None; + } + state.runner_finished = false; + state.finalization_taken = true; + Some(state.abnormal) +} + +#[cfg(test)] +mod tests { + use super::{ + DeadlineState, PROCESS_START_RECEIPT_TIMEOUT, PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT, + ProcessStart, ProcessStartAcknowledgement, ProcessStartInner, ProcessStartPhase, + ReceiptResolution, ReceiptState, RunnerConfig, RunnerProcessManager, + RunnerProcessManagerInner, ShutdownRequest, StartResultPublication, + }; + use litebox_broker_core::test_support::TestBrokerCoreBuilder; + use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; + use litebox_broker_protocol::ProcessId; + use litebox_broker_protocol::error::ErrorCode; + use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; + use litebox_broker_protocol::process::ProcessStartToken; + use std::path::PathBuf; + use std::sync::{ + Arc, Condvar, Mutex, + atomic::{AtomicBool, Ordering}, + mpsc, + }; + use std::time::{Duration, Instant}; + + fn start_with_broker_in_phase( + publication: StartResultPublication, + phase: ProcessStartPhase, + ) -> (BrokerCore, Arc) { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let (process, _) = parent.create_child(&[]).unwrap(); + parent.cleanup(true); + if let ProcessStartPhase::Ready { initial_thread_id } = phase { + process.mark_start_ready(initial_thread_id).unwrap(); + } + ( + broker, + Arc::new(ProcessStart { + parent_id: ProcessId(1), + process, + state: Mutex::new(ProcessStartInner { + phase, + publication, + shutdown: None, + association_failure: None, + shutdown_request: ShutdownRequest::None, + abnormal: false, + receipt: ReceiptState::AwaitingAcknowledgement, + resolution_watchdog: DeadlineState::Unarmed, + acknowledgement_publication_watchdog: DeadlineState::Unarmed, + start_failure_publication_watchdog: DeadlineState::Unarmed, + active_control_callbacks: 0, + runner_finished: false, + finalization_taken: false, + }), + changed: Condvar::new(), + }), + ) + } + + fn start_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { + start_with_broker_in_phase( + publication, + ProcessStartPhase::Ready { + initial_thread_id: None, + }, + ) + } + + fn start(publication: StartResultPublication) -> Arc { + start_with_broker(publication).1 + } + + fn starting_process(publication: StartResultPublication) -> Arc { + start_with_broker_in_phase(publication, ProcessStartPhase::Starting).1 + } + + #[test] + fn publication_captures_an_absolute_receipt_deadline() { + let start = start(StartResultPublication::NotStarted); + let before = Instant::now(); + + let deadline = start.begin_start_result_publication().unwrap(); + let after = Instant::now(); + + assert!(deadline >= before + PROCESS_START_RECEIPT_TIMEOUT); + assert!(deadline <= after + PROCESS_START_RECEIPT_TIMEOUT); + } + + #[test] + fn acknowledgement_ingress_claims_receipt_before_worker_resolution() { + let (broker, start) = start_with_broker(StartResultPublication::Delivered); + let token = ProcessStartToken(7); + let parent_id = start.parent_id; + let process_manager = Arc::new(RunnerProcessManager { + broker, + process_start_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerInner { + starts: vec![(token, Arc::clone(&start))], + associations: Vec::new(), + active_instances: 1, + active_watchdogs: 0, + }), + drained: Condvar::new(), + }); + + process_manager + .admit_acknowledgement(parent_id, token) + .unwrap(); + + assert_eq!(process_manager.state.lock().unwrap().active_watchdogs, 2); + assert!(matches!( + start.state.lock().unwrap().receipt, + ReceiptState::AcknowledgementAdmitted + )); + assert!(start.expire_initial_receipt_deadline().is_none()); + assert!(matches!( + process_manager.resolve_acknowledgement(parent_id, token), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + process_manager.response_sent( + parent_id, + &BrokerOperation::AcknowledgeProcessStart(token), + &BrokerResult::ProcessStartAcknowledged, + ); + start.process.cleanup(true); + process_manager.finish_instance(); + process_manager.wait_for_drain(); + assert_eq!(process_manager.state.lock().unwrap().active_watchdogs, 0); + } + + #[test] + fn reported_bootstrap_rejection_selects_normal_rollback() { + let start = starting_process(StartResultPublication::NotStarted); + + start + .report_start_failure(ErrorCode::UnsupportedOperation) + .unwrap(); + + assert!(matches!( + start.wait_until_ready(), + Err(ErrorCode::UnsupportedOperation) + )); + assert!( + start + .shutdown_request() + .expected_start_failure_was_reported() + ); + assert!(!start.state.lock().unwrap().abnormal); + start.start_failure_response_sent(ErrorCode::UnsupportedOperation); + assert!(start.shutdown_request().was_expected()); + assert_eq!(start.runner_finished(false), None); + assert!(matches!( + start.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + start.process.cleanup(true); + } + + #[test] + fn acknowledgement_waits_for_publication_bookkeeping() { + let start = start(StartResultPublication::Publishing); + start.admit_acknowledgement().unwrap(); + let waiting = Arc::clone(&start); + let (sender, receiver) = mpsc::sync_channel(1); + let worker = + std::thread::spawn(move || sender.send(waiting.resolve_acknowledgement()).unwrap()); + + assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); + assert!(matches!( + start.state.lock().unwrap().resolution_watchdog, + DeadlineState::Unarmed + )); + start.mark_start_result_delivered(); + assert!(matches!( + receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + worker.join().unwrap(); + assert!(start.process.is_running()); + assert!(matches!( + start.state.lock().unwrap().resolution_watchdog, + DeadlineState::Disarmed + )); + assert!(matches!( + start + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog, + DeadlineState::Armed(_) + )); + + start.resolve_receipt(); + start.process.cleanup(true); + } + + #[test] + fn acknowledgement_interrupted_by_drain_returns_peer_closed() { + let start = start(StartResultPublication::Publishing); + start.admit_acknowledgement().unwrap(); + let waiting = Arc::clone(&start); + let worker = std::thread::spawn(move || waiting.resolve_acknowledgement()); + + std::thread::sleep(Duration::from_millis(20)); + start.begin_receipt_drain(); + + assert!(matches!(worker.join().unwrap(), Err(ErrorCode::PeerClosed))); + start.finish_receipt_drain(); + start.process.cleanup(true); + } + + #[test] + fn process_ready_interrupted_by_abort_returns_the_abort_cause() { + let ready = starting_process(StartResultPublication::NotStarted); + ready.abort(ErrorCode::PeerClosed, false, true); + assert!(matches!( + ready.ready_and_wait(None), + Err(ErrorCode::PeerClosed) + )); + ready.resolve_receipt(); + ready.process.cleanup(true); + } + + #[test] + fn failure_report_interrupted_by_abort_returns_the_abort_cause() { + let failed = starting_process(StartResultPublication::NotStarted); + failed.abort(ErrorCode::PeerClosed, false, true); + assert!(matches!( + failed.report_start_failure(ErrorCode::UnsupportedOperation), + Err(ErrorCode::PeerClosed) + )); + failed.resolve_receipt(); + failed.process.cleanup(true); + } + + #[test] + fn delivery_arms_the_resolution_watchdog_before_the_worker_resumes() { + let start = start(StartResultPublication::Publishing); + start.admit_acknowledgement().unwrap(); + + assert!(matches!( + start.state.lock().unwrap().resolution_watchdog, + DeadlineState::Unarmed + )); + start.mark_start_result_delivered(); + assert!(matches!( + start.state.lock().unwrap().resolution_watchdog, + DeadlineState::Armed(_) + )); + + start.begin_receipt_drain(); + start.finish_receipt_drain(); + start.process.cleanup(true); + } + + #[test] + fn acknowledgement_publication_timeout_retains_receipt_for_drain() { + let start = start(StartResultPublication::Delivered); + start.admit_acknowledgement().unwrap(); + assert!(matches!( + start.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Acknowledged) + )); + start + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = start + .wait_for_acknowledgement_publication_timeout() + .unwrap(); + assert!(expiration.fail_parent); + let state = start.state.lock().unwrap(); + assert!(matches!(state.receipt, ReceiptState::TimeoutPending)); + assert!(matches!( + state.acknowledgement_publication_watchdog, + DeadlineState::Fired + )); + assert!(!state.abnormal); + drop(state); + + assert_eq!(start.complete_timeout_callback(), None); + start.begin_receipt_drain(); + start.finish_receipt_drain(); + start.process.cleanup(true); + } + + #[test] + fn start_failure_publication_timeout_terminates_the_runner() { + let start = starting_process(StartResultPublication::NotStarted); + start + .report_start_failure(ErrorCode::UnsupportedOperation) + .unwrap(); + start + .state + .lock() + .unwrap() + .start_failure_publication_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = start.wait_for_start_failure_publication_timeout().unwrap(); + + assert!(!expiration.fail_parent); + assert!(start.shutdown_request().was_expected()); + assert!(matches!( + start + .state + .lock() + .unwrap() + .start_failure_publication_watchdog, + DeadlineState::Fired + )); + assert_eq!(start.complete_timeout_callback(), None); + assert_eq!(start.runner_finished(false), None); + assert!(matches!( + start.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + start.process.cleanup(true); + } + + #[test] + fn precommit_resolution_timeout_returns_typed_internal_failure() { + let start = start(StartResultPublication::Delivered); + start.admit_acknowledgement().unwrap(); + start.state.lock().unwrap().resolution_watchdog = DeadlineState::Armed(Instant::now()); + + let expiration = start.wait_for_internal_resolution_timeout().unwrap(); + assert!(!expiration.fail_parent); + let state = start.state.lock().unwrap(); + assert!(matches!( + state.phase, + ProcessStartPhase::Aborted(ErrorCode::Internal) + )); + assert!(matches!( + state.receipt, + ReceiptState::AcknowledgementAdmitted + )); + assert!(matches!( + state.acknowledgement_publication_watchdog, + DeadlineState::Unarmed + )); + assert!(state.abnormal); + drop(state); + assert_eq!(start.complete_timeout_callback(), None); + + assert!(matches!( + start.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Failed(ErrorCode::Internal)) + )); + assert!(matches!( + start + .state + .lock() + .unwrap() + .acknowledgement_publication_watchdog, + DeadlineState::Armed(_) + )); + start.resolve_receipt(); + start.process.cleanup(false); + } + + #[test] + fn process_start_abort_fails_an_installed_active_association() { + let start = start(StartResultPublication::NotStarted); + let failed = Arc::new(AtomicBool::new(false)); + let recorded = Arc::clone(&failed); + start.install_association_failure(Arc::new(move || { + recorded.store(true, Ordering::Release); + })); + + start.abort(ErrorCode::PeerClosed, false, true); + + assert!(failed.load(Ordering::Acquire)); + start.process.cleanup(true); + } + + #[test] + fn supervisor_wait_observes_commit_resolution() { + let start = start(StartResultPublication::Delivered); + start.state.lock().unwrap().phase = ProcessStartPhase::Committing; + let waiting = Arc::clone(&start); + let worker = std::thread::spawn(move || { + waiting.wait_for_commit_resolution(Instant::now() + Duration::from_secs(1)) + }); + + std::thread::sleep(Duration::from_millis(20)); + start.state.lock().unwrap().phase = ProcessStartPhase::Committed; + start.changed.notify_all(); + + assert!(worker.join().unwrap()); + start.process.cleanup(true); + } + + #[test] + fn receipt_drain_keeps_commit_supervision_armed() { + let start = start(StartResultPublication::Delivered); + start.admit_acknowledgement().unwrap(); + { + let mut state = start.state.lock().unwrap(); + state.phase = ProcessStartPhase::Committing; + state.resolution_watchdog = DeadlineState::Armed(Instant::now()); + } + start.begin_receipt_drain(); + assert!(matches!( + start.state.lock().unwrap().resolution_watchdog, + DeadlineState::Armed(_) + )); + + let before = Instant::now(); + let expiration = start.wait_for_internal_resolution_timeout().unwrap(); + let after = Instant::now(); + let deadline = expiration.commit_supervision_deadline.unwrap(); + assert!(deadline >= before + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); + assert!(deadline <= after + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); + { + let mut state = start.state.lock().unwrap(); + state.phase = ProcessStartPhase::Aborted(ErrorCode::Internal); + } + start.changed.notify_all(); + assert_eq!(start.complete_timeout_callback(), None); + start.finish_receipt_drain(); + start.process.cleanup(false); + } + + #[test] + fn published_abort_returns_typed_start_failure() { + let start = start(StartResultPublication::Publishing); + start.abort(ErrorCode::PeerClosed, false, false); + start.mark_start_result_delivered(); + start.admit_acknowledgement().unwrap(); + + assert!(matches!( + start.resolve_acknowledgement(), + Ok(ProcessStartAcknowledgement::Failed(ErrorCode::PeerClosed)) + )); + start.resolve_receipt(); + start.process.cleanup(true); + } + + #[test] + fn acknowledgement_before_publication_is_protocol_violation() { + let start = start(StartResultPublication::NotStarted); + + assert!(matches!( + start.admit_acknowledgement(), + Err(ErrorCode::ProtocolState) + )); + start.resolve_receipt(); + start.process.cleanup(true); + } + + #[test] + fn published_receipt_defers_process_finalization() { + let start = start(StartResultPublication::Delivered); + start.abort(ErrorCode::PeerClosed, false, false); + + assert_eq!(start.runner_finished(false), None); + assert!(matches!( + start.resolve_receipt(), + ReceiptResolution::Resolved(Some(false)) + )); + start.process.cleanup(true); + } + + #[test] + fn receipt_resolution_completes_process_start() { + let start = starting_process(StartResultPublication::Delivered); + let thread_id = start.process.create_thread().unwrap(); + start.process.mark_start_ready(Some(thread_id)).unwrap(); + start.process.commit_start().unwrap(); + start.state.lock().unwrap().phase = ProcessStartPhase::Committed; + + start.resolve_receipt(); + start.resolve_receipt(); + + let state = start.state.lock().unwrap(); + assert!(matches!(state.phase, ProcessStartPhase::StartComplete)); + assert!(!state.abnormal); + drop(state); + assert_eq!(start.process.exit_thread(thread_id), Ok(())); + start.process.cleanup(true); + } + + #[test] + fn process_ready_waits_until_process_start_is_complete() { + let start = starting_process(StartResultPublication::Delivered); + let thread_id = start.process.create_thread().unwrap(); + let waiting = Arc::clone(&start); + let (sender, receiver) = mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + sender + .send(waiting.ready_and_wait(Some(thread_id))) + .unwrap(); + }); + + let mut state = start.state.lock().unwrap(); + while !matches!(state.phase, ProcessStartPhase::Ready { .. }) { + state = start.changed.wait(state).unwrap(); + } + start.process.commit_start().unwrap(); + state.phase = ProcessStartPhase::Committed; + start.changed.notify_all(); + drop(state); + assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); + + start.resolve_receipt(); + assert_eq!( + receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(()) + ); + worker.join().unwrap(); + assert_eq!(start.process.exit_thread(thread_id), Ok(())); + start.process.cleanup(true); + } + + #[test] + fn acknowledgement_admission_preserves_the_publication_deadline() { + let start = start(StartResultPublication::Publishing); + start.admit_acknowledgement().unwrap(); + + let _expiration = start.expire_initial_receipt_deadline().unwrap(); + let state = start.state.lock().unwrap(); + assert!(matches!( + state.phase, + ProcessStartPhase::Aborted(ErrorCode::Internal) + )); + assert!(state.abnormal); + drop(state); + assert_eq!(start.complete_timeout_callback(), None); + start.process.cleanup(true); + } + + #[test] + fn receipt_timeout_defers_finalization_until_association_drain() { + let start = start(StartResultPublication::Delivered); + + assert_eq!(start.runner_finished(false), None); + let _expiration = start.expire_initial_receipt_deadline().unwrap(); + assert!(matches!( + start.resolve_receipt(), + ReceiptResolution::DeferredToDrain + )); + assert_eq!(start.complete_timeout_callback(), None); + start.begin_receipt_drain(); + assert_eq!(start.finish_receipt_drain(), Some(false)); + start.process.cleanup(true); + } + + #[test] + fn unpublished_start_waits_for_receipt_drain_before_finalization() { + let start = start(StartResultPublication::NotStarted); + + start.begin_receipt_drain(); + assert_eq!(start.runner_finished(false), None); + assert_eq!(start.finish_receipt_drain(), Some(false)); + start.process.cleanup(true); + } + + #[test] + fn deferred_finalization_uses_the_latest_abnormal_disposition() { + let start = start(StartResultPublication::Publishing); + start.admit_acknowledgement().unwrap(); + + assert_eq!(start.runner_finished(false), None); + let _expiration = start.expire_initial_receipt_deadline().unwrap(); + assert_eq!(start.complete_timeout_callback(), None); + start.begin_receipt_drain(); + assert_eq!(start.finish_receipt_drain(), Some(true)); + start.process.cleanup(false); + } + + #[test] + fn acknowledgement_send_does_not_steal_a_timed_out_receipt_from_drain() { + let (broker, start) = start_with_broker(StartResultPublication::Delivered); + let token = ProcessStartToken(7); + let parent_id = start.parent_id; + let process_manager = RunnerProcessManager { + broker, + process_start_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerInner { + starts: vec![(token, Arc::clone(&start))], + associations: Vec::new(), + active_instances: 1, + active_watchdogs: 0, + }), + drained: Condvar::new(), + }; + + assert_eq!(start.runner_finished(false), None); + let _expiration = start.expire_initial_receipt_deadline().unwrap(); + process_manager.response_sent( + parent_id, + &BrokerOperation::AcknowledgeProcessStart(token), + &BrokerResult::ProcessStartAcknowledged, + ); + + assert!(process_manager.find_start(token).is_some()); + assert_eq!(start.complete_timeout_callback(), None); + let draining = process_manager.association_ending(parent_id); + assert_eq!(draining.starts.len(), 1); + process_manager.association_ended(parent_id, draining); + assert!(process_manager.find_start(token).is_none()); + } +} diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 2ae86250e0..19c7fee23f 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -14,7 +14,9 @@ use litebox_broker_transport_windows_userland::named_pipe::{ }; use litebox_broker_transport_windows_userland::shared_memory::WindowsSharedMemory; -use super::{ChildRunner, RunnerChildren, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited}; +use super::{ + RunnerProcessManager, RunnerStartup, SETUP_TIMEOUT, accept_runner_channel, runner_has_exited, +}; use crate::runtime::{AssociationFailureCause, AssociationRunResult}; pub(super) struct PlatformRunnerEndpoint { @@ -39,30 +41,16 @@ impl PlatformRunnerEndpoint { pub(super) fn serve( &mut self, runner: &Arc>, - children: Arc, - ) -> AssociationRunResult { - serve_runner_process( - self.listener - .as_mut() - .expect("a live runner instance must own its control listener"), - runner, - children, - ) - } - - pub(super) fn serve_child( - &mut self, - runner: &Arc>, - child: ChildRunner, - children: Arc, + startup: Option, + process_manager: Arc, ) -> AssociationRunResult { - serve_child_runner_process( + serve_association( self.listener .as_mut() .expect("a live runner instance must own its control listener"), runner, - child, - children, + startup, + process_manager, ) } @@ -71,66 +59,28 @@ impl PlatformRunnerEndpoint { } } -fn serve_runner_process( - control_listener: &mut WindowsNamedPipeListener, - runner: &Arc>, - children: Arc, -) -> AssociationRunResult { - let (control_channel, _setup_deadline) = match accept_control_channel(control_listener, runner) - { - Ok(connection) => connection, - Err(error) => { - let failure_cause = if runner_has_exited(runner).unwrap_or(false) { - AssociationFailureCause::RunnerExit - } else { - AssociationFailureCause::Other - }; - return AssociationRunResult { - result: Err(error), - process: None, - panicked: false, - abnormal: failure_cause == AssociationFailureCause::Other, - failure_cause, - }; - } - }; - let runner_process = runner - .lock() - .expect("runner process mutex poisoned") - .as_raw_handle(); - crate::runtime::serve_runner_association( - None, - control_channel, - || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), - WindowsSharedMemory::create_control_ring, - |channel, shared_memory, control_memory| { - channel.send_shared_memory(shared_memory, runner_process)?; - channel.send_shared_memory(control_memory, runner_process) - }, - WindowsNamedPipeHostSetupChannel::into_active, - children, - ) -} - -fn serve_child_runner_process( +fn serve_association( control_listener: &mut WindowsNamedPipeListener, runner: &Arc>, - child: ChildRunner, - children: Arc, + startup: Option, + process_manager: Arc, ) -> AssociationRunResult { + let is_started_process = startup.is_some(); let (control_channel, _setup_deadline) = match accept_control_channel(control_listener, runner) { Ok(connection) => connection, Err(error) => { let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit - } else if matches!( - error.kind(), - std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - ) { + } else if is_started_process + && matches!( + error.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + ) + { AssociationFailureCause::PeerClosed } else { AssociationFailureCause::Other @@ -148,8 +98,8 @@ fn serve_child_runner_process( .lock() .expect("runner process mutex poisoned") .as_raw_handle(); - let mut result = crate::runtime::serve_runner_association( - Some(child), + let mut result = crate::runtime::serve_out_of_process_runner_association( + startup, control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), WindowsSharedMemory::create_control_ring, @@ -158,9 +108,10 @@ fn serve_child_runner_process( channel.send_shared_memory(control_memory, runner_process) }, WindowsNamedPipeHostSetupChannel::into_active, - children, + process_manager, ); - if result.failure_cause == AssociationFailureCause::Other + if is_started_process + && result.failure_cause == AssociationFailureCause::Other && result .result .as_ref() diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index daad0cd424..83eee81a3f 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -14,8 +14,8 @@ //! transport. //! //! Concurrency and worker sizing are deliberately not part of the public -//! surface: [`serve_association`] is the only entry point, and it owns how -//! many workers dispatch requests and how deeply the request queue may buffer. +//! surface. Both association entry points delegate to one internal runtime +//! that owns worker counts and request queue capacity. use std::io::{Error as IoError, ErrorKind, Result as IoResult}; use std::sync::mpsc::{Receiver, SyncSender, TrySendError, sync_channel}; @@ -32,7 +32,6 @@ use litebox_broker_host::{ use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest}; -use litebox_broker_protocol::process::ProcessStartupData; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, @@ -42,11 +41,11 @@ use litebox_broker_transport::control_ring::ControlRing; use litebox_broker_transport::shared_memory::{ControlRingMemory, SharedBufferPool, SharedMemory}; use crate::readiness::ReadinessPublisherRuntime; -use crate::runner::{ChildRunner, RunnerChildren}; +use crate::runner::{RunnerProcessManager, RunnerStartup}; const REQUEST_QUEUE_CAPACITY: usize = 64; -pub(crate) const LIFECYCLE_CONTROL_WORKER_COUNT: usize = crate::WORKER_COUNT; -pub(crate) const LIFECYCLE_CONTROL_QUEUE_CAPACITY: usize = crate::WORKER_COUNT; +pub(crate) const PROCESS_START_CONTROL_WORKER_COUNT: usize = crate::WORKER_COUNT; +pub(crate) const PROCESS_START_CONTROL_QUEUE_CAPACITY: usize = crate::WORKER_COUNT; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); const REQUEST_QUEUE_STALL_TIMEOUT: Duration = Duration::from_secs(5); @@ -66,7 +65,7 @@ pub(crate) enum AssociationFailureCause { Other, } -/// Serves one broker association from setup through teardown. +/// Serves an association for the development-only in-process runner. /// /// `control_channel` must already be configured with whatever deadline and /// peer authentication its transport requires; this function only negotiates @@ -79,7 +78,7 @@ pub(crate) enum AssociationFailureCause { /// Returns once the association ends, whether by a clean peer close or a /// failure. Layout mismatches and broker setup failures are mapped to a /// precise [`std::io::Error`] rather than left as an opaque boxed error. -pub fn serve_association< +pub fn serve_in_process_runner_association< Memory, SetupChannel, RequestSource, @@ -105,7 +104,7 @@ where NotificationChannel: HostNotificationChannel + Send, Shutdown: HostAssociationShutdown + Send + Sync + 'static, { - serve_association_with_children( + serve_association_inner( broker, control_channel, create_shared_memory, @@ -120,7 +119,11 @@ where ) } -pub(crate) fn serve_runner_association< +/// Serves an initial or parent-started association for an out-of-process runner. +/// +/// Unlike the in-process path, this reports process ownership and failure +/// details to `RunnerInstance`, which owns runner termination and cleanup. +pub(crate) fn serve_out_of_process_runner_association< Memory, SetupChannel, RequestSource, @@ -128,7 +131,7 @@ pub(crate) fn serve_runner_association< NotificationChannel, Shutdown, >( - child: Option, + startup: Option, control_channel: SetupChannel, create_shared_memory: impl FnOnce() -> IoResult, create_control_memory: impl FnOnce() -> IoResult, @@ -137,7 +140,7 @@ pub(crate) fn serve_runner_association< SetupChannel, ControlRing, ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, - children: Arc, + process_manager: Arc, ) -> AssociationRunResult where Memory: ControlRingMemory, @@ -150,17 +153,17 @@ where let panicked = AtomicBool::new(false); let abnormal = AtomicBool::new(false); let process = Mutex::new(None); - let defer_process_finish = child.is_none(); - let broker = children.broker.clone(); - let result = serve_association_with_children( + let defer_process_finish = startup.is_none(); + let broker = process_manager.broker(); + let result = serve_association_inner( &broker, control_channel, create_shared_memory, create_control_memory, send_shared_memory, activate, - Some(children), - child, + Some(process_manager), + startup, Some(&panicked), Some(&abnormal), defer_process_finish.then_some(&process), @@ -191,7 +194,7 @@ where } #[allow(clippy::too_many_arguments)] -fn serve_association_with_children< +fn serve_association_inner< Memory, SetupChannel, RequestSource, @@ -208,8 +211,8 @@ fn serve_association_with_children< SetupChannel, ControlRing, ) -> IoResult<(RequestSource, ResponseSink, NotificationChannel, Shutdown)>, - children: Option>, - child: Option, + process_manager: Option>, + startup: Option, panicked_out: Option<&AtomicBool>, abnormal_out: Option<&AtomicBool>, process_out: Option<&Mutex>>>, @@ -222,24 +225,12 @@ where NotificationChannel: HostNotificationChannel + Send, Shutdown: HostAssociationShutdown + Send + Sync + 'static, { - let is_child = child.is_some(); - let (process, startup) = match child { - Some(ChildRunner { - process, - launch: _, - inherited_objects, - format, - version, - bootstrap, - }) => ( - Some(process), - Some(ProcessStartupData { - format, - version, - payload: bootstrap, - inherited_objects, - }), - ), + let is_started_process = startup.is_some(); + let (process, startup) = match startup { + Some(startup) => { + let (process, data) = startup.into_process_and_data(); + (Some(process), Some(data)) + } None => (None, None), }; let shared_memory = create_shared_memory()?; @@ -291,21 +282,21 @@ where match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { - if !is_child && process_out.is_none() { + if !is_started_process && process_out.is_none() { association.finish(); } return Err(error); } }; - dispatch_requests_with_children( + dispatch_requests( association, readiness, request_source, response_sink, notification_channel, shutdown, - children, - !is_child && process_out.is_none(), + process_manager, + !is_started_process && process_out.is_none(), panicked_out, abnormal_out, ) @@ -487,51 +478,15 @@ impl Drop for AssociationCancellationGuard<'_, '_, Memory> /// `readiness` is created by the caller rather than here so readiness sources /// can record into the same runtime this publishes from. The Linux network /// reactor is currently its production source. -#[cfg(all(test, target_os = "linux"))] -fn dispatch_requests( - association: BrokerHostAssociation<'_, Memory>, - readiness: Arc, - request_source: RequestSource, - response_sink: ResponseSink, - notification_channel: NotificationChannel, - shutdown: Shutdown, -) -> IoResult<()> -where - Memory: SharedMemory, - RequestSource: HostRequestSource, - ResponseSink: HostResponseSink + Clone + Send, - NotificationChannel: HostNotificationChannel + Send, - Shutdown: HostAssociationShutdown + Send + Sync + 'static, -{ - dispatch_requests_with_children( - association, - readiness, - request_source, - response_sink, - notification_channel, - shutdown, - None, - true, - None, - None, - ) -} - #[allow(clippy::too_many_arguments)] -fn dispatch_requests_with_children< - Memory, - RequestSource, - ResponseSink, - NotificationChannel, - Shutdown, ->( +fn dispatch_requests( association: BrokerHostAssociation<'_, Memory>, readiness: Arc, mut request_source: RequestSource, response_sink: ResponseSink, mut notification_channel: NotificationChannel, shutdown: Shutdown, - children: Option>, + process_manager: Option>, finish_process: bool, panicked_out: Option<&AtomicBool>, abnormal_out: Option<&AtomicBool>, @@ -546,29 +501,30 @@ where let association = Arc::new(association); let process_id = association.process_id(); let failure_coordinator = Arc::new(HostAssociationFailureCoordinator::new(shutdown)); - if let Some(children) = &children { + if let Some(process_manager) = &process_manager { let association_failure = Arc::clone(&failure_coordinator); let association_failure: crate::runner::AssociationFailure = Arc::new(move || { association_failure.report(IoError::new( ErrorKind::ConnectionAborted, - "process association terminated by lifecycle control", + "process association terminated by process-start control", )); }); - if let Err(error) = children + if let Err(error) = process_manager .register_association(association.process_id(), Arc::clone(&association_failure)) { failure_coordinator.report(error); } else { - children - .install_child_association_failure(association.process_id(), association_failure); + process_manager + .install_process_association_failure(association.process_id(), association_failure); } } let (request_sender, request_receiver) = sync_channel(REQUEST_QUEUE_CAPACITY); let request_receiver = Arc::new(Mutex::new(request_receiver)); - let (control_sender, control_receiver) = sync_channel(LIFECYCLE_CONTROL_QUEUE_CAPACITY); - let control_receiver = Arc::new(Mutex::new(control_receiver)); + let (process_start_sender, process_start_receiver) = + sync_channel(PROCESS_START_CONTROL_QUEUE_CAPACITY); + let process_start_receiver = Arc::new(Mutex::new(process_start_receiver)); - let draining_launches = std::thread::scope(|scope| { + let draining_starts = std::thread::scope(|scope| { let publisher_readiness = Arc::clone(&readiness); let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() @@ -605,22 +561,23 @@ where association: &association, }; - let mut workers = Vec::with_capacity(crate::WORKER_COUNT + LIFECYCLE_CONTROL_WORKER_COUNT); - for worker_id in 0..LIFECYCLE_CONTROL_WORKER_COUNT { + let mut workers = + Vec::with_capacity(crate::WORKER_COUNT + PROCESS_START_CONTROL_WORKER_COUNT); + for worker_id in 0..PROCESS_START_CONTROL_WORKER_COUNT { let association = Arc::clone(&association); - let control_receiver = Arc::clone(&control_receiver); + let process_start_receiver = Arc::clone(&process_start_receiver); let response_sink = response_sink.clone(); let worker_failure_coordinator = Arc::clone(&failure_coordinator); - let worker_children = children.clone(); + let process_manager_for_worker = process_manager.clone(); match std::thread::Builder::new() - .name(format!("litebox-broker-lifecycle-worker-{worker_id}")) + .name(format!("litebox-broker-process-start-worker-{worker_id}")) .spawn_scoped(scope, move || { run_worker( &association, - &control_receiver, + &process_start_receiver, &response_sink, &worker_failure_coordinator, - worker_children.as_ref(), + process_manager_for_worker.as_ref(), ); }) { Ok(worker) => workers.push(worker), @@ -635,7 +592,7 @@ where let request_receiver = Arc::clone(&request_receiver); let response_sink = response_sink.clone(); let worker_failure_coordinator = Arc::clone(&failure_coordinator); - let worker_children = children.clone(); + let process_manager_for_worker = process_manager.clone(); match std::thread::Builder::new() .name(format!("litebox-broker-worker-{worker_id}")) .spawn_scoped(scope, move || { @@ -644,7 +601,7 @@ where &request_receiver, &response_sink, &worker_failure_coordinator, - worker_children.as_ref(), + process_manager_for_worker.as_ref(), ); }) { Ok(worker) => workers.push(worker), @@ -658,15 +615,15 @@ where read_requests( &mut request_source, request_sender, - control_sender, + process_start_sender, &failure_coordinator, - children.as_ref(), + process_manager.as_ref(), process_id, ); drop(cancellation); - let draining_launches = children + let draining_starts = process_manager .as_ref() - .map_or_else(Vec::new, |children| children.association_ending(process_id)); + .map(|process_manager| process_manager.association_ending(process_id)); for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -686,11 +643,11 @@ where { failure_coordinator.report_panic(IoError::other("broker readiness publisher panicked")); } - draining_launches + draining_starts }); - if let Some(children) = &children { - children.association_ended(process_id, draining_launches); + if let (Some(process_manager), Some(draining_starts)) = (&process_manager, draining_starts) { + process_manager.association_ended(process_id, draining_starts); } let result = match failure_coordinator.take_error() { @@ -718,9 +675,9 @@ where fn read_requests( request_source: &mut RequestSource, request_sender: SyncSender, - control_sender: SyncSender, + process_start_sender: SyncSender, failure_coordinator: &HostAssociationFailureCoordinator, - children: Option<&Arc>, + process_manager: Option<&Arc>, process_id: ProcessId, ) where RequestSource: HostRequestSource, @@ -733,8 +690,8 @@ fn read_requests( match request_source.recv_request() { Ok(HostReceive::Message(request)) => { if let BrokerOperation::AcknowledgeProcessStart(token) = &request.operation - && let Some(children) = children - && let Err(error) = children.admit_acknowledgement(process_id, *token) + && let Some(process_manager) = process_manager + && let Err(error) = process_manager.admit_acknowledgement(process_id, *token) { failure_coordinator.report(map_host_error(BrokerHostError::Broker(error))); break; @@ -744,7 +701,7 @@ fn read_requests( BrokerOperation::AcknowledgeProcessStart(_) | BrokerOperation::ReportProcessStartFailure(_) ) { - &control_sender + &process_start_sender } else { &request_sender }; @@ -816,7 +773,7 @@ fn run_worker( request_receiver: &Mutex>, response_sink: &ResponseSink, failure_coordinator: &HostAssociationFailureCoordinator, - children: Option<&Arc>, + process_manager: Option<&Arc>, ) where Memory: SharedMemory, ResponseSink: HostResponseSink, @@ -838,14 +795,14 @@ fn run_worker( association.execute_request_with( request, |process, operation, shared_buffers| { - children.and_then(|children| { - children.handle_operation(process, operation, shared_buffers) + process_manager.and_then(|process_manager| { + process_manager.handle_operation(process, operation, shared_buffers) }) }, |response| response_sink.send_response(response), |operation, result| { - if let Some(children) = children { - children.response_sent(process_id, operation, result); + if let Some(process_manager) = process_manager { + process_manager.response_sent(process_id, operation, result); } }, ) @@ -1088,6 +1045,10 @@ mod tests { response_sink, notifications, shutdown, + None, + true, + None, + None, )) .unwrap(); }); @@ -1294,7 +1255,7 @@ mod tests { failure_coordinator.report(IoError::new( ErrorKind::ConnectionAborted, - "lifecycle shutdown", + "process shutdown", )); failure_coordinator.report(IoError::new(ErrorKind::InvalidData, "protocol failure")); diff --git a/litebox_broker_userland/src/windows.rs b/litebox_broker_userland/src/windows.rs index 529297faa3..d05f499eb7 100644 --- a/litebox_broker_userland/src/windows.rs +++ b/litebox_broker_userland/src/windows.rs @@ -98,7 +98,7 @@ fn serve_control_stream( ) -> IoResult<()> { let control_channel = WindowsNamedPipeHostSetupChannel::from_host_guaranteed(control_stream, setup_deadline); - litebox_broker_userland::runtime::serve_association( + litebox_broker_userland::runtime::serve_in_process_runner_association( broker, control_channel, || WindowsSharedMemory::create(SHARED_BUFFER_POOL_SIZE), From 063297e9a6da80a4ef39af856e24da4143605f0a Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 15:58:03 -0700 Subject: [PATCH 12/21] Clarify runner process manager naming Name the module after RunnerProcessManager and represent each in-progress StartProcess operation as a transaction with explicit state. Clarify started-runner configuration and simplify the non-Linux exit-signal helper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_userland/src/runner.rs | 23 +- .../{process_start.rs => process_manager.rs} | 458 +++++++++++------- litebox_broker_userland/src/runtime.rs | 11 +- 3 files changed, 293 insertions(+), 199 deletions(-) rename litebox_broker_userland/src/runner/{process_start.rs => process_manager.rs} (85%) diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 38fbbf246e..34ca8f9808 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -17,13 +17,13 @@ use litebox_broker_core::BrokerCore; #[cfg(target_os = "linux")] mod linux; -mod process_start; +mod process_manager; #[cfg(all(windows, target_arch = "x86_64"))] mod windows; #[cfg(target_os = "linux")] use linux::PlatformRunnerEndpoint; -pub(crate) use process_start::{AssociationFailure, RunnerProcessManager, RunnerStartup}; +pub(crate) use process_manager::{AssociationFailure, RunnerProcessManager, RunnerStartup}; #[cfg(all(windows, target_arch = "x86_64"))] use windows::PlatformRunnerEndpoint; @@ -75,7 +75,7 @@ impl RunnerConfig { arguments } - fn for_process_start(&self) -> Self { + fn without_initial_arguments(&self) -> Self { Self { executable: self.executable.clone(), arguments: Vec::new(), @@ -92,7 +92,7 @@ pub struct RunnerInstance { runner: Arc>, shutdown: Arc, endpoint: PlatformRunnerEndpoint, - process_start_config: RunnerConfig, + started_runner_config: RunnerConfig, } struct RunnerShutdown { @@ -202,12 +202,12 @@ impl RunnerInstance { changed: Condvar::new(), termination_dispatched: AtomicBool::new(false), }); - let process_start_config = config.for_process_start(); + let started_runner_config = config.without_initial_arguments(); Ok(Self { runner, shutdown, endpoint, - process_start_config, + started_runner_config, }) } @@ -222,7 +222,7 @@ impl RunnerInstance { /// Panics if another runner owner poisoned the process mutex. pub fn run_to_completion(mut self, broker: &BrokerCore) -> IoResult { let process_manager = - RunnerProcessManager::new(self.process_start_config.clone(), broker.clone()); + RunnerProcessManager::new(self.started_runner_config.clone(), broker.clone()); let mut association_result = self.endpoint .serve(&self.runner, None, Arc::clone(&process_manager)); @@ -275,13 +275,8 @@ fn runner_exit_signal(status: ExitStatus) -> Option { status.signal() } -#[cfg(all(windows, target_arch = "x86_64"))] -const fn runner_exit_signal(_status: ExitStatus) -> Option { - None -} - -#[cfg(not(any(target_os = "linux", all(windows, target_arch = "x86_64"))))] -const fn runner_exit_signal(_status: ExitStatus) -> Option { +#[cfg(not(target_os = "linux"))] +fn runner_exit_signal(_status: ExitStatus) -> Option { None } diff --git a/litebox_broker_userland/src/runner/process_start.rs b/litebox_broker_userland/src/runner/process_manager.rs similarity index 85% rename from litebox_broker_userland/src/runner/process_start.rs rename to litebox_broker_userland/src/runner/process_manager.rs index 23dac8c1d2..7f4b32a5da 100644 --- a/litebox_broker_userland/src/runner/process_start.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -//! Coordination for broker-requested runner process starts. +//! Shared coordination for out-of-process runner instances. use std::io::{Error as IoError, Result as IoResult}; use std::process::ExitStatus; @@ -37,18 +37,18 @@ const _: () = const _: () = assert!(crate::runtime::PROCESS_START_CONTROL_QUEUE_CAPACITY >= MAX_PENDING_PROCESS_STARTS); -/// Shared coordination for runner process starts and active associations. +/// Shared ownership and coordination for runner processes and associations. pub(crate) struct RunnerProcessManager { broker: BrokerCore, - process_start_config: RunnerConfig, - state: Mutex, + started_runner_config: RunnerConfig, + state: Mutex, drained: Condvar, } /// Startup context for a runner whose broker process was created by its parent. pub(crate) struct RunnerStartup { process: Arc, - start: Arc, + transaction: Arc, data: ProcessStartupData, } @@ -95,8 +95,8 @@ impl TerminationProvenance { } } -struct RunnerProcessManagerInner { - starts: Vec<(ProcessStartToken, Arc)>, +struct RunnerProcessManagerState { + transactions: Vec<(ProcessStartToken, Arc)>, associations: Vec<(ProcessId, AssociationFailure)>, active_instances: usize, active_watchdogs: usize, @@ -104,14 +104,16 @@ struct RunnerProcessManagerInner { pub(crate) type AssociationFailure = Arc; -pub(crate) struct ProcessStartDrain { - starts: Vec>, +/// Transactions retained while an ending association drains in-flight work. +pub(crate) struct AssociationDrain { + transactions: Vec>, } -struct ProcessStart { +/// State for one `StartProcess` request from admission through finalization. +struct ProcessStartTransaction { parent_id: ProcessId, process: Arc, - state: Mutex, + state: Mutex, changed: Condvar, } @@ -133,7 +135,7 @@ enum StartResultPublication { Delivered, } -struct ProcessStartInner { +struct ProcessStartTransactionState { phase: ProcessStartPhase, publication: StartResultPublication, shutdown: Option>, @@ -214,7 +216,7 @@ impl RunnerInstance { startup: RunnerStartup, process_manager: Arc, ) -> RunnerCompletion { - let start = Arc::clone(&startup.start); + let start = Arc::clone(&startup.transaction); start.install_shutdown(Arc::clone(&self.shutdown)); let association_result = self .endpoint @@ -293,12 +295,12 @@ impl RunnerInstance { } impl RunnerProcessManager { - pub(super) fn new(process_start_config: RunnerConfig, broker: BrokerCore) -> Arc { + pub(super) fn new(started_runner_config: RunnerConfig, broker: BrokerCore) -> Arc { Arc::new(Self { broker, - process_start_config, - state: Mutex::new(RunnerProcessManagerInner { - starts: Vec::new(), + started_runner_config, + state: Mutex::new(RunnerProcessManagerState { + transactions: Vec::new(), associations: Vec::new(), active_instances: 0, active_watchdogs: 0, @@ -365,7 +367,7 @@ impl RunnerProcessManager { ) { match (operation, result) { (_, BrokerResult::ProcessStarted(started)) => { - let Some(start) = self.find_start(started.token) else { + let Some(start) = self.find_transaction(started.token) else { return; }; if start.parent_id == process_id { @@ -376,16 +378,16 @@ impl RunnerProcessManager { BrokerOperation::AcknowledgeProcessStart(token), BrokerResult::ProcessStartAcknowledged | BrokerResult::ProcessStartFailed(_), ) => { - let Some(start) = self.find_start(*token) else { + let Some(start) = self.find_transaction(*token) else { return; }; if start.parent_id != process_id { return; } if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { - self.remove_start(*token); + self.remove_transaction(*token); if let Some(abnormal) = finalization { - self.finish_process_start(&start, abnormal); + self.finish_transaction(&start, abnormal); } } } @@ -393,7 +395,7 @@ impl RunnerProcessManager { BrokerOperation::ReportProcessStartFailure(error), BrokerResult::ProcessStartFailed(reported), ) if error == reported => { - if let Some(start) = self.find_process_start(process_id) { + if let Some(start) = self.find_transaction_by_process_id(process_id) { start.start_failure_response_sent(*error); } } @@ -418,10 +420,10 @@ impl RunnerProcessManager { let inherited_objects = InheritedProcessObjects::new(&inherited_objects) .expect("child handle count must match the bounded inheritance request"); let process_id = process.id(); - let start = Arc::new(ProcessStart { + let start = Arc::new(ProcessStartTransaction { parent_id: parent.id(), process: Arc::clone(&process), - state: Mutex::new(ProcessStartInner { + state: Mutex::new(ProcessStartTransactionState { phase: ProcessStartPhase::Starting, publication: StartResultPublication::NotStarted, shutdown: None, @@ -448,17 +450,17 @@ impl RunnerProcessManager { .lock() .expect("runner process manager state mutex poisoned"); state - .starts + .transactions .try_reserve(1) .map_err(|_| ErrorCode::OutOfMemory)?; if parent.is_cancellation_requested() { return Err(ErrorCode::PeerClosed); } - if state.starts.len() >= MAX_PENDING_PROCESS_STARTS { + if state.transactions.len() >= MAX_PENDING_PROCESS_STARTS { return Err(ErrorCode::ResourceExhausted); } if state - .starts + .transactions .iter() .any(|(candidate, _)| *candidate == token) { @@ -468,7 +470,7 @@ impl RunnerProcessManager { .active_instances .checked_add(1) .ok_or(ErrorCode::ResourceExhausted)?; - state.starts.push((token, Arc::clone(&start))); + state.transactions.push((token, Arc::clone(&start))); state.active_instances = active_instances; return Ok(token); } @@ -481,8 +483,8 @@ impl RunnerProcessManager { }; let process_manager = Arc::clone(self); - let config = self.process_start_config.clone(); - let thread_start = Arc::clone(&start); + let config = self.started_runner_config.clone(); + let thread_transaction = Arc::clone(&start); let thread = std::thread::Builder::new() .name(format!("litebox-runner-{}", process_id.0)) .spawn(move || { @@ -491,7 +493,7 @@ impl RunnerProcessManager { instance.run_started_process_to_completion( RunnerStartup { process, - start: Arc::clone(&thread_start), + transaction: Arc::clone(&thread_transaction), data: ProcessStartupData { format, version, @@ -505,11 +507,11 @@ impl RunnerProcessManager { })); match outcome { Ok(Ok(result)) => { - process_manager.runner_finished(token, &thread_start, result, false); + process_manager.runner_finished(token, &thread_transaction, result, false); } Ok(Err(error)) => process_manager.runner_finished( token, - &thread_start, + &thread_transaction, RunnerCompletion { result: Err(error), runner_success: None, @@ -523,7 +525,7 @@ impl RunnerProcessManager { ), Err(_) => process_manager.runner_finished( token, - &thread_start, + &thread_transaction, RunnerCompletion { result: Err(IoError::other("runner process thread panicked")), runner_success: None, @@ -538,7 +540,7 @@ impl RunnerProcessManager { } }); if thread.is_err() { - self.remove_start(token); + self.remove_transaction(token); start.process.cleanup(true); self.finish_instance(); return Err(ErrorCode::OutOfMemory); @@ -573,7 +575,9 @@ impl RunnerProcessManager { parent_id: ProcessId, token: ProcessStartToken, ) -> Result<(), ErrorCode> { - let start = self.find_start(token).ok_or(ErrorCode::UnknownObject)?; + let start = self + .find_transaction(token) + .ok_or(ErrorCode::UnknownObject)?; if start.parent_id != parent_id { return Err(ErrorCode::UnknownObject); } @@ -614,7 +618,7 @@ impl RunnerProcessManager { parent_id: ProcessId, token: ProcessStartToken, ) -> Result { - let start = self.find_start(token).ok_or(ErrorCode::PeerClosed)?; + let start = self.find_transaction(token).ok_or(ErrorCode::PeerClosed)?; if start.parent_id != parent_id { return Err(ErrorCode::UnknownObject); } @@ -627,7 +631,7 @@ impl RunnerProcessManager { initial_thread_id: Option, ) -> Result<(), ErrorCode> { let start = self - .find_process_start(process_id) + .find_transaction_by_process_id(process_id) .ok_or(ErrorCode::PeerClosed)?; start.ready_and_wait(initial_thread_id) } @@ -638,7 +642,7 @@ impl RunnerProcessManager { error: ErrorCode, ) -> Result<(), ErrorCode> { let (token, start) = self - .find_process_start_entry(process_id) + .find_transaction_entry_by_process_id(process_id) .ok_or(ErrorCode::PeerClosed)?; start.report_start_failure(error)?; if self @@ -653,43 +657,45 @@ impl RunnerProcessManager { Ok(()) } - pub(crate) fn association_ending(&self, process_id: ProcessId) -> ProcessStartDrain { - let draining = self + pub(crate) fn association_ending(&self, process_id: ProcessId) -> AssociationDrain { + let draining_transactions = self .state .lock() .expect("runner process manager state mutex poisoned") - .starts + .transactions .iter() .filter(|(_, start)| start.parent_id == process_id) .map(|(_, start)| Arc::clone(start)) .collect::>(); - for start in &draining { + for start in &draining_transactions { start.abort(ErrorCode::PeerClosed, false, true); start.begin_receipt_drain(); } - let process_start = { + let transaction = { self.state .lock() .expect("runner process manager state mutex poisoned") - .starts + .transactions .iter() .find_map(|(_, start)| { (start.process.id() == process_id).then(|| Arc::clone(start)) }) }; - if let Some(start) = process_start { + if let Some(start) = transaction { start.association_closed(); } - ProcessStartDrain { starts: draining } + AssociationDrain { + transactions: draining_transactions, + } } - pub(crate) fn association_ended(&self, process_id: ProcessId, draining: ProcessStartDrain) { + pub(crate) fn association_ended(&self, process_id: ProcessId, draining: AssociationDrain) { self.unregister_association(process_id); - for start in draining.starts { - self.remove_start_by_process_id(start.process.id()); + for start in draining.transactions { + self.remove_transaction_by_process_id(start.process.id()); if let Some(abnormal) = start.finish_receipt_drain() { - self.finish_process_start(&start, abnormal); + self.finish_transaction(&start, abnormal); } } } @@ -725,7 +731,7 @@ impl RunnerProcessManager { process_id: ProcessId, failure: AssociationFailure, ) { - if let Some(start) = self.find_process_start(process_id) { + if let Some(start) = self.find_transaction_by_process_id(process_id) { start.install_association_failure(failure); } } @@ -758,7 +764,7 @@ impl RunnerProcessManager { fn arm_initial_receipt_deadline( self: &Arc, token: ProcessStartToken, - start: &Arc, + start: &Arc, deadline: Instant, ) -> Result<(), ()> { let start = Arc::clone(start); @@ -777,7 +783,7 @@ impl RunnerProcessManager { fn arm_internal_resolution_watchdog( self: &Arc, token: ProcessStartToken, - start: &Arc, + start: &Arc, ) -> Result<(), ()> { let start = Arc::clone(start); let parent_failure = self.find_association_failure(start.parent_id); @@ -795,7 +801,7 @@ impl RunnerProcessManager { fn arm_acknowledgement_publication_watchdog( self: &Arc, token: ProcessStartToken, - start: &Arc, + start: &Arc, ) -> Result<(), ()> { let start = Arc::clone(start); let parent_failure = self.find_association_failure(start.parent_id); @@ -813,7 +819,7 @@ impl RunnerProcessManager { fn arm_start_failure_publication_watchdog( self: &Arc, token: ProcessStartToken, - start: &Arc, + start: &Arc, ) -> Result<(), ()> { let start = Arc::clone(start); self.spawn_watchdog( @@ -857,7 +863,7 @@ impl RunnerProcessManager { fn apply_receipt_expiration( &self, token: ProcessStartToken, - start: &ProcessStart, + start: &ProcessStartTransaction, expiration: ReceiptExpiration, parent_failure: Option, ) { @@ -880,12 +886,12 @@ impl RunnerProcessManager { true }; if let Some(abnormal) = start.complete_timeout_callback() { - self.finish_process_start(start, abnormal); + self.finish_transaction(start, abnormal); } if fail_parent && !parent_failed { - self.remove_start(token); + self.remove_transaction(token); if let Some(abnormal) = start.finish_receipt_drain() { - self.finish_process_start(start, abnormal); + self.finish_transaction(start, abnormal); } } if let Some(deadline) = commit_supervision_deadline @@ -895,66 +901,69 @@ impl RunnerProcessManager { } } - fn find_start(&self, token: ProcessStartToken) -> Option> { + fn find_transaction(&self, token: ProcessStartToken) -> Option> { self.state .lock() .expect("runner process manager state mutex poisoned") - .starts + .transactions .iter() .find_map(|(candidate, start)| (*candidate == token).then(|| Arc::clone(start))) } - fn find_process_start(&self, process_id: ProcessId) -> Option> { - self.find_process_start_entry(process_id) + fn find_transaction_by_process_id( + &self, + process_id: ProcessId, + ) -> Option> { + self.find_transaction_entry_by_process_id(process_id) .map(|(_, start)| start) } - fn find_process_start_entry( + fn find_transaction_entry_by_process_id( &self, process_id: ProcessId, - ) -> Option<(ProcessStartToken, Arc)> { + ) -> Option<(ProcessStartToken, Arc)> { self.state .lock() .expect("runner process manager state mutex poisoned") - .starts + .transactions .iter() .find_map(|(token, start)| { (start.process.id() == process_id).then(|| (*token, Arc::clone(start))) }) } - fn remove_start(&self, token: ProcessStartToken) { + fn remove_transaction(&self, token: ProcessStartToken) { let mut state = self .state .lock() .expect("runner process manager state mutex poisoned"); if let Some(index) = state - .starts + .transactions .iter() .position(|(candidate, _)| *candidate == token) { - state.starts.swap_remove(index); + state.transactions.swap_remove(index); } } - fn remove_start_by_process_id(&self, process_id: ProcessId) { + fn remove_transaction_by_process_id(&self, process_id: ProcessId) { let mut state = self .state .lock() .expect("runner process manager state mutex poisoned"); if let Some(index) = state - .starts + .transactions .iter() .position(|(_, start)| start.process.id() == process_id) { - state.starts.swap_remove(index); + state.transactions.swap_remove(index); } } fn runner_finished( &self, token: ProcessStartToken, - start: &ProcessStart, + start: &ProcessStartTransaction, result: RunnerCompletion, thread_panicked: bool, ) { @@ -983,17 +992,17 @@ impl RunnerProcessManager { start.abort(ErrorCode::PeerClosed, false, false); } if !start.retains_published_receipt() { - self.remove_start(token); + self.remove_transaction(token); if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { debug_assert!(finalization.is_none()); } } if let Some(abnormal) = start.runner_finished(abnormal) { - self.finish_process_start(start, abnormal); + self.finish_transaction(start, abnormal); } } - fn finish_process_start(&self, start: &ProcessStart, abnormal: bool) { + fn finish_transaction(&self, start: &ProcessStartTransaction, abnormal: bool) { start.process.cleanup(!abnormal); self.finish_instance(); } @@ -1076,16 +1085,19 @@ const fn process_start_failure_is_expected(error: ErrorCode) -> bool { ) } -impl ProcessStart { +impl ProcessStartTransaction { fn wait_until_ready(&self) -> Result, ErrorCode> { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { match state.phase { ProcessStartPhase::Starting => { state = self .changed .wait(state) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); } ProcessStartPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), ProcessStartPhase::Committing @@ -1106,7 +1118,10 @@ impl ProcessStart { BrokerError::UnknownObject => ErrorCode::ProtocolState, error => ErrorCode::from(error), })?; - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if !matches!(state.phase, ProcessStartPhase::Starting) { return Err(match state.phase { ProcessStartPhase::Aborted(error) => error, @@ -1128,7 +1143,7 @@ impl ProcessStart { state = self .changed .wait(state) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); } ProcessStartPhase::Aborted(error) => return Err(error), ProcessStartPhase::Starting => unreachable!("ready state cannot regress"), @@ -1137,7 +1152,10 @@ impl ProcessStart { } fn begin_start_result_publication(&self) -> Result { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); match state.phase { ProcessStartPhase::Ready { .. } if state.publication == StartResultPublication::NotStarted => @@ -1154,7 +1172,10 @@ impl ProcessStart { if !process_start_failure_is_expected(error) { return Err(ErrorCode::ProtocolState); } - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); match state.phase { ProcessStartPhase::Starting => {} ProcessStartPhase::Aborted(error) => return Err(error), @@ -1172,7 +1193,10 @@ impl ProcessStart { fn start_failure_response_sent(&self, error: ErrorCode) { let (association_failure, shutdown) = { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); complete_deadline(&mut state.start_failure_publication_watchdog); let actions = if matches!(state.phase, ProcessStartPhase::Aborted(cause) if cause == error) && state.shutdown_request == ShutdownRequest::ExpectedStartFailurePending @@ -1197,7 +1221,10 @@ impl ProcessStart { } fn admit_acknowledgement(&self) -> Result<(), ErrorCode> { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if state.publication == StartResultPublication::NotStarted { return Err(ErrorCode::ProtocolState); } @@ -1225,7 +1252,10 @@ impl ProcessStart { } fn resolve_acknowledgement(&self) -> Result { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { match state.receipt { ReceiptState::AcknowledgementAdmitted => {} @@ -1245,14 +1275,17 @@ impl ProcessStart { state = self .changed .wait(state) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); } (ProcessStartPhase::Ready { .. }, StartResultPublication::Delivered) => { debug_assert!(matches!(state.resolution_watchdog, DeadlineState::Armed(_))); state.phase = ProcessStartPhase::Committing; drop(state); let commit_result = self.process.commit_start().map_err(ErrorCode::from); - state = self.state.lock().expect("process start mutex poisoned"); + state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); match commit_result { Ok(()) => { state.phase = ProcessStartPhase::Committed; @@ -1296,7 +1329,10 @@ impl ProcessStart { } fn mark_start_result_delivered(&self) { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if state.publication == StartResultPublication::Publishing { state.publication = StartResultPublication::Delivered; if state.receipt == ReceiptState::AcknowledgementAdmitted @@ -1316,7 +1352,10 @@ impl ProcessStart { fn install_shutdown(&self, shutdown: Arc) { let shutdown = { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.shutdown = Some(Arc::clone(&shutdown)); (state.shutdown_request != ShutdownRequest::None).then_some(shutdown) }; @@ -1327,7 +1366,10 @@ impl ProcessStart { fn install_association_failure(&self, failure: AssociationFailure) { let failure = { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.association_failure = Some(Arc::clone(&failure)); (state.shutdown_request != ShutdownRequest::None).then_some(failure) }; @@ -1338,7 +1380,10 @@ impl ProcessStart { fn abort(&self, error: ErrorCode, abnormal: bool, expected_shutdown: bool) { let (association_failure, shutdown) = { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.abnormal |= abnormal; match state.phase { ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { @@ -1397,7 +1442,10 @@ impl ProcessStart { } fn association_closed(&self) { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.association_failure = None; if matches!( state.phase, @@ -1409,7 +1457,10 @@ impl ProcessStart { } fn mark_shutdown_expected(&self) { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if state.shutdown_request == ShutdownRequest::None { state.shutdown_request = ShutdownRequest::Expected; } @@ -1418,14 +1469,14 @@ impl ProcessStart { fn mark_abnormal(&self) { self.state .lock() - .expect("process start mutex poisoned") + .expect("process start transaction mutex poisoned") .abnormal = true; } fn shutdown_request(&self) -> ShutdownRequest { self.state .lock() - .expect("process start mutex poisoned") + .expect("process start transaction mutex poisoned") .shutdown_request } @@ -1433,7 +1484,7 @@ impl ProcessStart { matches!( self.state .lock() - .expect("process start mutex poisoned") + .expect("process start transaction mutex poisoned") .phase, ProcessStartPhase::Committing | ProcessStartPhase::Committed @@ -1443,13 +1494,19 @@ impl ProcessStart { } fn retains_published_receipt(&self) -> bool { - let state = self.state.lock().expect("process start mutex poisoned"); + let state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.publication != StartResultPublication::NotStarted && state.receipt != ReceiptState::Resolved } fn resolve_receipt(&self) -> ReceiptResolution { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if matches!( state.receipt, ReceiptState::TimeoutPending | ReceiptState::Draining @@ -1461,11 +1518,14 @@ impl ProcessStart { complete_deadline(&mut state.acknowledgement_publication_watchdog); complete_deadline(&mut state.start_failure_publication_watchdog); self.changed.notify_all(); - ReceiptResolution::Resolved(self.complete_process_start_and_take_finalization(state)) + ReceiptResolution::Resolved(self.complete_and_take_finalization(state)) } fn begin_receipt_drain(&self) { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if state.receipt != ReceiptState::Resolved { state.receipt = ReceiptState::Draining; if !matches!(state.phase, ProcessStartPhase::Committing) { @@ -1478,17 +1538,23 @@ impl ProcessStart { } fn finish_receipt_drain(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.receipt = ReceiptState::Resolved; complete_deadline(&mut state.resolution_watchdog); complete_deadline(&mut state.acknowledgement_publication_watchdog); complete_deadline(&mut state.start_failure_publication_watchdog); self.changed.notify_all(); - self.complete_process_start_and_take_finalization(state) + self.complete_and_take_finalization(state) } fn expire_initial_receipt_deadline(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); let (error, abnormal) = match (state.receipt, state.publication) { (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { @@ -1496,11 +1562,14 @@ impl ProcessStart { } _ => return None, }; - expire_start(&mut state, error, abnormal, true, &self.changed) + expire_transaction(&mut state, error, abnormal, true, &self.changed) } fn wait_for_initial_receipt_deadline(&self, deadline: Instant) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { if !matches!( (state.receipt, state.publication), @@ -1519,7 +1588,7 @@ impl ProcessStart { let (next, _) = self .changed .wait_timeout(state, remaining) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); state = next; } let (error, abnormal) = match (state.receipt, state.publication) { @@ -1529,18 +1598,21 @@ impl ProcessStart { } _ => return None, }; - expire_start(&mut state, error, abnormal, true, &self.changed) + expire_transaction(&mut state, error, abnormal, true, &self.changed) } fn wait_for_internal_resolution_timeout(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { match state.resolution_watchdog { DeadlineState::Unarmed => { state = self .changed .wait(state) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); } DeadlineState::Armed(deadline) => { let remaining = deadline.saturating_duration_since(Instant::now()); @@ -1551,7 +1623,7 @@ impl ProcessStart { let (next, _) = self .changed .wait_timeout(state, remaining) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); state = next; } DeadlineState::Disarmed | DeadlineState::Fired => return None, @@ -1560,20 +1632,23 @@ impl ProcessStart { } fn wait_for_acknowledgement_publication_timeout(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { match state.acknowledgement_publication_watchdog { DeadlineState::Unarmed => { state = self .changed .wait(state) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); } DeadlineState::Armed(deadline) => { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { state.acknowledgement_publication_watchdog = DeadlineState::Fired; - return expire_start( + return expire_transaction( &mut state, ErrorCode::PeerClosed, false, @@ -1584,7 +1659,7 @@ impl ProcessStart { let (next, _) = self .changed .wait_timeout(state, remaining) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); state = next; } DeadlineState::Disarmed | DeadlineState::Fired => return None, @@ -1593,7 +1668,10 @@ impl ProcessStart { } fn wait_for_start_failure_publication_timeout(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); loop { match state.start_failure_publication_watchdog { DeadlineState::Armed(deadline) => { @@ -1605,7 +1683,7 @@ impl ProcessStart { let (next, _) = self .changed .wait_timeout(state, remaining) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); state = next; } DeadlineState::Unarmed | DeadlineState::Disarmed | DeadlineState::Fired => { @@ -1616,7 +1694,10 @@ impl ProcessStart { } fn fail_start_failure_publication_watchdog(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); if !matches!( state.start_failure_publication_watchdog, DeadlineState::Armed(_) @@ -1628,15 +1709,21 @@ impl ProcessStart { } fn fail_internal_resolution_watchdog(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); complete_deadline(&mut state.resolution_watchdog); complete_deadline(&mut state.acknowledgement_publication_watchdog); complete_deadline(&mut state.start_failure_publication_watchdog); - expire_start(&mut state, ErrorCode::Internal, true, true, &self.changed) + expire_transaction(&mut state, ErrorCode::Internal, true, true, &self.changed) } fn complete_timeout_callback(&self) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.active_control_callbacks = state .active_control_callbacks .checked_sub(1) @@ -1646,7 +1733,10 @@ impl ProcessStart { } fn wait_for_commit_resolution(&self, deadline: Instant) -> bool { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); while matches!(state.phase, ProcessStartPhase::Committing) { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { @@ -1655,7 +1745,7 @@ impl ProcessStart { let (next, wait_result) = self .changed .wait_timeout(state, remaining) - .expect("process start mutex poisoned"); + .expect("process start transaction mutex poisoned"); state = next; if wait_result.timed_out() && matches!(state.phase, ProcessStartPhase::Committing) { return false; @@ -1664,9 +1754,9 @@ impl ProcessStart { true } - fn complete_process_start_and_take_finalization( + fn complete_and_take_finalization( &self, - mut state: MutexGuard<'_, ProcessStartInner>, + mut state: MutexGuard<'_, ProcessStartTransactionState>, ) -> Option { if !matches!(state.phase, ProcessStartPhase::Committed) { return take_finalization(&mut state); @@ -1679,7 +1769,10 @@ impl ProcessStart { drop(state); let completion_failed = self.process.complete_start().is_err(); - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.active_control_callbacks = state .active_control_callbacks .checked_sub(1) @@ -1691,15 +1784,18 @@ impl ProcessStart { } fn runner_finished(&self, abnormal: bool) -> Option { - let mut state = self.state.lock().expect("process start mutex poisoned"); + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); state.abnormal |= abnormal; state.runner_finished = true; take_finalization(&mut state) } } -fn expire_start( - state: &mut ProcessStartInner, +fn expire_transaction( + state: &mut ProcessStartTransactionState, error: ErrorCode, abnormal: bool, fail_parent: bool, @@ -1754,7 +1850,7 @@ fn expire_start( } fn expire_start_failure_publication( - state: &mut ProcessStartInner, + state: &mut ProcessStartTransactionState, changed: &Condvar, ) -> Option { if !matches!(state.phase, ProcessStartPhase::Aborted(_)) @@ -1774,7 +1870,7 @@ fn expire_start_failure_publication( } fn expire_internal_resolution( - state: &mut ProcessStartInner, + state: &mut ProcessStartTransactionState, changed: &Condvar, ) -> Option { let committing_drain = state.receipt == ReceiptState::Draining @@ -1837,7 +1933,7 @@ fn complete_deadline(state: &mut DeadlineState) { } } -fn complete_acknowledgement_resolution(state: &mut ProcessStartInner) { +fn complete_acknowledgement_resolution(state: &mut ProcessStartTransactionState) { complete_deadline(&mut state.resolution_watchdog); if state.receipt == ReceiptState::AcknowledgementAdmitted { arm_deadline( @@ -1847,7 +1943,7 @@ fn complete_acknowledgement_resolution(state: &mut ProcessStartInner) { } } -fn take_finalization(state: &mut ProcessStartInner) -> Option { +fn take_finalization(state: &mut ProcessStartTransactionState) -> Option { if state.finalization_taken || state.active_control_callbacks != 0 || state.receipt != ReceiptState::Resolved @@ -1866,9 +1962,9 @@ fn take_finalization(state: &mut ProcessStartInner) -> Option { mod tests { use super::{ DeadlineState, PROCESS_START_RECEIPT_TIMEOUT, PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT, - ProcessStart, ProcessStartAcknowledgement, ProcessStartInner, ProcessStartPhase, - ReceiptResolution, ReceiptState, RunnerConfig, RunnerProcessManager, - RunnerProcessManagerInner, ShutdownRequest, StartResultPublication, + ProcessStartAcknowledgement, ProcessStartPhase, ProcessStartTransaction, + ProcessStartTransactionState, ReceiptResolution, ReceiptState, RunnerConfig, + RunnerProcessManager, RunnerProcessManagerState, ShutdownRequest, StartResultPublication, }; use litebox_broker_core::test_support::TestBrokerCoreBuilder; use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; @@ -1884,10 +1980,10 @@ mod tests { }; use std::time::{Duration, Instant}; - fn start_with_broker_in_phase( + fn transaction_with_broker_in_phase( publication: StartResultPublication, phase: ProcessStartPhase, - ) -> (BrokerCore, Arc) { + ) -> (BrokerCore, Arc) { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1903,10 +1999,10 @@ mod tests { } ( broker, - Arc::new(ProcessStart { + Arc::new(ProcessStartTransaction { parent_id: ProcessId(1), process, - state: Mutex::new(ProcessStartInner { + state: Mutex::new(ProcessStartTransactionState { phase, publication, shutdown: None, @@ -1926,8 +2022,10 @@ mod tests { ) } - fn start_with_broker(publication: StartResultPublication) -> (BrokerCore, Arc) { - start_with_broker_in_phase( + fn transaction_with_broker( + publication: StartResultPublication, + ) -> (BrokerCore, Arc) { + transaction_with_broker_in_phase( publication, ProcessStartPhase::Ready { initial_thread_id: None, @@ -1935,17 +2033,17 @@ mod tests { ) } - fn start(publication: StartResultPublication) -> Arc { - start_with_broker(publication).1 + fn transaction(publication: StartResultPublication) -> Arc { + transaction_with_broker(publication).1 } - fn starting_process(publication: StartResultPublication) -> Arc { - start_with_broker_in_phase(publication, ProcessStartPhase::Starting).1 + fn starting_transaction(publication: StartResultPublication) -> Arc { + transaction_with_broker_in_phase(publication, ProcessStartPhase::Starting).1 } #[test] fn publication_captures_an_absolute_receipt_deadline() { - let start = start(StartResultPublication::NotStarted); + let start = transaction(StartResultPublication::NotStarted); let before = Instant::now(); let deadline = start.begin_start_result_publication().unwrap(); @@ -1957,14 +2055,14 @@ mod tests { #[test] fn acknowledgement_ingress_claims_receipt_before_worker_resolution() { - let (broker, start) = start_with_broker(StartResultPublication::Delivered); + let (broker, start) = transaction_with_broker(StartResultPublication::Delivered); let token = ProcessStartToken(7); let parent_id = start.parent_id; let process_manager = Arc::new(RunnerProcessManager { broker, - process_start_config: RunnerConfig::new(PathBuf::new(), Vec::new()), - state: Mutex::new(RunnerProcessManagerInner { - starts: vec![(token, Arc::clone(&start))], + started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerState { + transactions: vec![(token, Arc::clone(&start))], associations: Vec::new(), active_instances: 1, active_watchdogs: 0, @@ -1999,7 +2097,7 @@ mod tests { #[test] fn reported_bootstrap_rejection_selects_normal_rollback() { - let start = starting_process(StartResultPublication::NotStarted); + let start = starting_transaction(StartResultPublication::NotStarted); start .report_start_failure(ErrorCode::UnsupportedOperation) @@ -2027,7 +2125,7 @@ mod tests { #[test] fn acknowledgement_waits_for_publication_bookkeeping() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.admit_acknowledgement().unwrap(); let waiting = Arc::clone(&start); let (sender, receiver) = mpsc::sync_channel(1); @@ -2065,7 +2163,7 @@ mod tests { #[test] fn acknowledgement_interrupted_by_drain_returns_peer_closed() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.admit_acknowledgement().unwrap(); let waiting = Arc::clone(&start); let worker = std::thread::spawn(move || waiting.resolve_acknowledgement()); @@ -2080,7 +2178,7 @@ mod tests { #[test] fn process_ready_interrupted_by_abort_returns_the_abort_cause() { - let ready = starting_process(StartResultPublication::NotStarted); + let ready = starting_transaction(StartResultPublication::NotStarted); ready.abort(ErrorCode::PeerClosed, false, true); assert!(matches!( ready.ready_and_wait(None), @@ -2092,7 +2190,7 @@ mod tests { #[test] fn failure_report_interrupted_by_abort_returns_the_abort_cause() { - let failed = starting_process(StartResultPublication::NotStarted); + let failed = starting_transaction(StartResultPublication::NotStarted); failed.abort(ErrorCode::PeerClosed, false, true); assert!(matches!( failed.report_start_failure(ErrorCode::UnsupportedOperation), @@ -2104,7 +2202,7 @@ mod tests { #[test] fn delivery_arms_the_resolution_watchdog_before_the_worker_resumes() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.admit_acknowledgement().unwrap(); assert!(matches!( @@ -2124,7 +2222,7 @@ mod tests { #[test] fn acknowledgement_publication_timeout_retains_receipt_for_drain() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); start.admit_acknowledgement().unwrap(); assert!(matches!( start.resolve_acknowledgement(), @@ -2157,7 +2255,7 @@ mod tests { #[test] fn start_failure_publication_timeout_terminates_the_runner() { - let start = starting_process(StartResultPublication::NotStarted); + let start = starting_transaction(StartResultPublication::NotStarted); start .report_start_failure(ErrorCode::UnsupportedOperation) .unwrap(); @@ -2190,7 +2288,7 @@ mod tests { #[test] fn precommit_resolution_timeout_returns_typed_internal_failure() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); start.admit_acknowledgement().unwrap(); start.state.lock().unwrap().resolution_watchdog = DeadlineState::Armed(Instant::now()); @@ -2231,7 +2329,7 @@ mod tests { #[test] fn process_start_abort_fails_an_installed_active_association() { - let start = start(StartResultPublication::NotStarted); + let start = transaction(StartResultPublication::NotStarted); let failed = Arc::new(AtomicBool::new(false)); let recorded = Arc::clone(&failed); start.install_association_failure(Arc::new(move || { @@ -2246,7 +2344,7 @@ mod tests { #[test] fn supervisor_wait_observes_commit_resolution() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); start.state.lock().unwrap().phase = ProcessStartPhase::Committing; let waiting = Arc::clone(&start); let worker = std::thread::spawn(move || { @@ -2263,7 +2361,7 @@ mod tests { #[test] fn receipt_drain_keeps_commit_supervision_armed() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); start.admit_acknowledgement().unwrap(); { let mut state = start.state.lock().unwrap(); @@ -2294,7 +2392,7 @@ mod tests { #[test] fn published_abort_returns_typed_start_failure() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.abort(ErrorCode::PeerClosed, false, false); start.mark_start_result_delivered(); start.admit_acknowledgement().unwrap(); @@ -2309,7 +2407,7 @@ mod tests { #[test] fn acknowledgement_before_publication_is_protocol_violation() { - let start = start(StartResultPublication::NotStarted); + let start = transaction(StartResultPublication::NotStarted); assert!(matches!( start.admit_acknowledgement(), @@ -2321,7 +2419,7 @@ mod tests { #[test] fn published_receipt_defers_process_finalization() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); start.abort(ErrorCode::PeerClosed, false, false); assert_eq!(start.runner_finished(false), None); @@ -2334,7 +2432,7 @@ mod tests { #[test] fn receipt_resolution_completes_process_start() { - let start = starting_process(StartResultPublication::Delivered); + let start = starting_transaction(StartResultPublication::Delivered); let thread_id = start.process.create_thread().unwrap(); start.process.mark_start_ready(Some(thread_id)).unwrap(); start.process.commit_start().unwrap(); @@ -2353,7 +2451,7 @@ mod tests { #[test] fn process_ready_waits_until_process_start_is_complete() { - let start = starting_process(StartResultPublication::Delivered); + let start = starting_transaction(StartResultPublication::Delivered); let thread_id = start.process.create_thread().unwrap(); let waiting = Arc::clone(&start); let (sender, receiver) = mpsc::sync_channel(1); @@ -2385,7 +2483,7 @@ mod tests { #[test] fn acknowledgement_admission_preserves_the_publication_deadline() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.admit_acknowledgement().unwrap(); let _expiration = start.expire_initial_receipt_deadline().unwrap(); @@ -2402,7 +2500,7 @@ mod tests { #[test] fn receipt_timeout_defers_finalization_until_association_drain() { - let start = start(StartResultPublication::Delivered); + let start = transaction(StartResultPublication::Delivered); assert_eq!(start.runner_finished(false), None); let _expiration = start.expire_initial_receipt_deadline().unwrap(); @@ -2418,7 +2516,7 @@ mod tests { #[test] fn unpublished_start_waits_for_receipt_drain_before_finalization() { - let start = start(StartResultPublication::NotStarted); + let start = transaction(StartResultPublication::NotStarted); start.begin_receipt_drain(); assert_eq!(start.runner_finished(false), None); @@ -2428,7 +2526,7 @@ mod tests { #[test] fn deferred_finalization_uses_the_latest_abnormal_disposition() { - let start = start(StartResultPublication::Publishing); + let start = transaction(StartResultPublication::Publishing); start.admit_acknowledgement().unwrap(); assert_eq!(start.runner_finished(false), None); @@ -2441,14 +2539,14 @@ mod tests { #[test] fn acknowledgement_send_does_not_steal_a_timed_out_receipt_from_drain() { - let (broker, start) = start_with_broker(StartResultPublication::Delivered); + let (broker, start) = transaction_with_broker(StartResultPublication::Delivered); let token = ProcessStartToken(7); let parent_id = start.parent_id; let process_manager = RunnerProcessManager { broker, - process_start_config: RunnerConfig::new(PathBuf::new(), Vec::new()), - state: Mutex::new(RunnerProcessManagerInner { - starts: vec![(token, Arc::clone(&start))], + started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerState { + transactions: vec![(token, Arc::clone(&start))], associations: Vec::new(), active_instances: 1, active_watchdogs: 0, @@ -2464,11 +2562,11 @@ mod tests { &BrokerResult::ProcessStartAcknowledged, ); - assert!(process_manager.find_start(token).is_some()); + assert!(process_manager.find_transaction(token).is_some()); assert_eq!(start.complete_timeout_callback(), None); let draining = process_manager.association_ending(parent_id); - assert_eq!(draining.starts.len(), 1); + assert_eq!(draining.transactions.len(), 1); process_manager.association_ended(parent_id, draining); - assert!(process_manager.find_start(token).is_none()); + assert!(process_manager.find_transaction(token).is_none()); } } diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 83eee81a3f..b9f922045e 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -524,7 +524,7 @@ where sync_channel(PROCESS_START_CONTROL_QUEUE_CAPACITY); let process_start_receiver = Arc::new(Mutex::new(process_start_receiver)); - let draining_starts = std::thread::scope(|scope| { + let association_drain = std::thread::scope(|scope| { let publisher_readiness = Arc::clone(&readiness); let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() @@ -621,7 +621,7 @@ where process_id, ); drop(cancellation); - let draining_starts = process_manager + let association_drain = process_manager .as_ref() .map(|process_manager| process_manager.association_ending(process_id)); for worker in workers { @@ -643,11 +643,12 @@ where { failure_coordinator.report_panic(IoError::other("broker readiness publisher panicked")); } - draining_starts + association_drain }); - if let (Some(process_manager), Some(draining_starts)) = (&process_manager, draining_starts) { - process_manager.association_ended(process_id, draining_starts); + if let (Some(process_manager), Some(association_drain)) = (&process_manager, association_drain) + { + process_manager.association_ended(process_id, association_drain); } let result = match failure_coordinator.take_error() { From 336c0fb8623cde0218031d00e19393d266dba5a6 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 16:16:51 -0700 Subject: [PATCH 13/21] Simplify runner process startup Make StartProcess block until the new runner reports ready and broker startup completes. Remove acknowledgement tokens, receipt state, watchdog workers, and the intermediate committed process state while preserving bounded startup, teardown, and runner finalization. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/process.rs | 62 +- litebox_broker_host/src/lib.rs | 1 - litebox_broker_local/src/process.rs | 30 +- litebox_broker_protocol/src/message.rs | 13 +- litebox_broker_protocol/src/process.rs | 9 +- litebox_broker_protocol/src/wire.rs | 34 +- litebox_broker_transport/src/pending_calls.rs | 2 +- .../src/unix_socket/local.rs | 13 +- .../src/local.rs | 3 +- .../src/named_pipe.rs | 8 +- litebox_broker_userland/src/runner.rs | 121 + .../src/runner/process_manager.rs | 2399 +++-------------- litebox_broker_userland/src/runtime.rs | 93 +- .../tests/userland_broker.rs | 44 +- 14 files changed, 599 insertions(+), 2233 deletions(-) diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index 7b8378b6c9..c8d5f9420b 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -128,7 +128,6 @@ pub struct BrokerProcess { pub(crate) enum ProcessState { Attaching, StartReady { initial_thread_id: Option }, - StartCommitted { initial_thread_id: Option }, Running, Exiting, } @@ -177,13 +176,10 @@ impl BrokerProcess { self.caller_credential } - /// Returns whether parent acknowledgement committed this process. + /// Returns whether this process completed broker startup. #[must_use] pub fn is_running(&self) -> bool { - matches!( - *self.state.lock(), - ProcessState::StartCommitted { .. } | ProcessState::Running - ) + matches!(*self.state.lock(), ProcessState::Running) } /// Records the initial thread supplied by `ProcessReady`. @@ -192,9 +188,9 @@ impl BrokerProcess { match *state { ProcessState::Attaching => {} ProcessState::Exiting => return Err(BrokerError::PeerClosed), - ProcessState::StartReady { .. } - | ProcessState::StartCommitted { .. } - | ProcessState::Running => return Err(BrokerError::Internal), + ProcessState::StartReady { .. } | ProcessState::Running => { + return Err(BrokerError::Internal); + } } if let Some(thread_id) = initial_thread_id && !self.threads.lock().contains_key(&thread_id) @@ -205,32 +201,15 @@ impl BrokerProcess { Ok(()) } - /// Commits this child after its start result reaches the parent. - pub fn commit_start(&self) -> Result<()> { - let mut state = self.state.lock(); - match *state { - ProcessState::StartReady { initial_thread_id } => { - *state = ProcessState::StartCommitted { initial_thread_id }; - Ok(()) - } - ProcessState::Attaching - | ProcessState::StartCommitted { .. } - | ProcessState::Running => Err(BrokerError::Internal), - ProcessState::Exiting => Err(BrokerError::PeerClosed), - } - } - - /// Completes startup after the acknowledgement response is published. + /// Completes startup after the runner reports ready. pub fn complete_start(&self) -> Result<()> { let mut state = self.state.lock(); match *state { - ProcessState::StartCommitted { .. } => { + ProcessState::StartReady { .. } => { *state = ProcessState::Running; Ok(()) } - ProcessState::Attaching | ProcessState::StartReady { .. } | ProcessState::Running => { - Err(BrokerError::Internal) - } + ProcessState::Attaching | ProcessState::Running => Err(BrokerError::Internal), ProcessState::Exiting => Err(BrokerError::PeerClosed), } } @@ -321,8 +300,6 @@ impl BrokerProcess { *state, ProcessState::StartReady { initial_thread_id: Some(pinned), - } | ProcessState::StartCommitted { - initial_thread_id: Some(pinned), } if pinned == thread_id ) { return Err(BrokerError::WouldBlock); @@ -984,7 +961,7 @@ mod tests { } #[test] - fn startup_states_defer_initial_thread_exit_until_publication_completes() { + fn startup_states_defer_initial_thread_exit_until_start_completes() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -998,17 +975,15 @@ mod tests { process.mark_start_ready(Some(thread)).unwrap(); assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); - process.commit_start().unwrap(); - assert!(process.is_running()); - assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); process.complete_start().unwrap(); + assert!(process.is_running()); assert_eq!(process.exit_thread(thread), Ok(())); process.cleanup(true); parent.cleanup(true); } #[test] - fn child_is_parented_and_requires_commit() { + fn child_is_parented_and_requires_start_completion() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1030,20 +1005,13 @@ mod tests { assert!(!child.is_running()); child.mark_start_ready(None).unwrap(); - child.commit_start().unwrap(); - assert!(child.is_running()); - assert!(matches!( - *child.state.lock(), - ProcessState::StartCommitted { - initial_thread_id: None - } - )); child.complete_start().unwrap(); + assert!(child.is_running()); assert_eq!(*child.state.lock(), ProcessState::Running); } #[test] - fn finished_uncommitted_child_releases_process_capacity() { + fn finished_attaching_child_releases_process_capacity() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1064,7 +1032,7 @@ mod tests { } #[test] - fn child_cannot_commit_after_teardown() { + fn child_cannot_complete_start_after_teardown() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -1079,7 +1047,7 @@ mod tests { child.cleanup(true); child.cleanup(true); - assert_eq!(child.commit_start(), Err(BrokerError::PeerClosed)); + assert_eq!(child.complete_start(), Err(BrokerError::PeerClosed)); let (replacement, _) = parent.create_child(&[]).unwrap(); replacement.cleanup(true); } diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index acd61382cf..1ef6e3bcb3 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -514,7 +514,6 @@ fn handle_request( handle_file_request(process, request, shared_buffers).map(BrokerResult::File) } BrokerOperation::StartProcess(_) - | BrokerOperation::AcknowledgeProcessStart(_) | BrokerOperation::ReportProcessReady(_) | BrokerOperation::ReportProcessStartFailure(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs index 3f66942e2f..9c54700c1f 100644 --- a/litebox_broker_local/src/process.rs +++ b/litebox_broker_local/src/process.rs @@ -6,7 +6,7 @@ use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartToken, ProcessStartupDescriptor, StartedProcess, + ProcessBootstrapVersion, ProcessStartupDescriptor, StartedProcess, }; use litebox_broker_protocol::shared_buffer::SharedBufferSequence; use litebox_broker_transport::channel::LocalCallChannel; @@ -16,9 +16,9 @@ use crate::{BrokerLocal, BrokerLocalError, Result}; impl BrokerLocal { /// Requests materialization of one child process. /// - /// The returned token must be acknowledged before the child may begin - /// guest execution. The caller must retain exclusive ownership of the - /// bootstrap sequence until this method returns. + /// This call blocks until the child reports ready or startup fails. The + /// caller must retain exclusive ownership of the bootstrap sequence until + /// this method returns. /// /// # Panics /// @@ -48,27 +48,7 @@ impl BrokerLocal { } } - /// Acknowledges a successful process-start result and releases the child. - /// - /// # Panics - /// - /// Panics if the broker returns a response for another operation. - pub fn acknowledge_process_start( - &self, - token: ProcessStartToken, - ) -> Result<(), Channel::Error> { - match self.request(BrokerOperation::AcknowledgeProcessStart(token))? { - BrokerResult::ProcessStartAcknowledged => Ok(()), - BrokerResult::ProcessStartFailed(error) | BrokerResult::Error(error) => { - Err(BrokerLocalError::Broker(error)) - } - response => { - panic!("broker returned unexpected process-start acknowledgement: {response:?}") - } - } - } - - /// Reports that this child process is ready and waits for parent acknowledgement. + /// Reports that this child process is ready and waits for broker startup completion. /// /// # Panics /// diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index 246a1af44b..fa42699124 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -17,7 +17,7 @@ use crate::pipe::{ CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; -use crate::process::{ProcessStartToken, ProcessStartupDescriptor, StartedProcess}; +use crate::process::{ProcessStartupDescriptor, StartedProcess}; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; use crate::socket::{ @@ -67,8 +67,6 @@ pub enum BrokerOperation { File(FileRequest), /// Start one child process from an opaque platform bootstrap. StartProcess(ProcessStartupDescriptor), - /// Commit a child process after its start result reaches the parent. - AcknowledgeProcessStart(ProcessStartToken), /// Report that this child process is ready to begin guest execution. ReportProcessReady(Option), /// Report that this child rejected its startup data before becoming ready. @@ -129,7 +127,6 @@ impl BrokerOperation { | Self::File( FileRequest::Seek(_) | FileRequest::Truncate(_) | FileRequest::HandleStatus(_), ) - | Self::AcknowledgeProcessStart(_) | Self::ReportProcessReady(_) | Self::ReportProcessStartFailure(_) => None, } @@ -249,13 +246,11 @@ pub enum BrokerResult { Stdio(StdioResponse), /// File response family. File(FileResponse), - /// A child was materialized and is ready for parent acknowledgement. + /// A child completed broker startup. ProcessStarted(StartedProcess), - /// Parent acknowledgement committed the child. - ProcessStartAcknowledged, - /// A child-start failure was reported or observed before commit. + /// A child-start failure was reported or observed before startup completed. ProcessStartFailed(ErrorCode), - /// Parent acknowledgement released the child. + /// The broker accepted the child's ready report. ProcessReady, /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index f97464d7e9..6342ea42b5 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -22,11 +22,6 @@ pub struct ProcessBootstrapFormat(pub u32); #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ProcessBootstrapVersion(pub u16); -/// Association-scoped token for acknowledging a child process start. -#[repr(transparent)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ProcessStartToken(pub u64); - /// Ordered broker-object handles inherited by a child. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct InheritedProcessObjects { @@ -90,11 +85,9 @@ pub struct ProcessStartupData { pub inherited_objects: InheritedProcessObjects, } -/// Reports a materialized child that is ready for parent acknowledgement. +/// Reports a child process that completed broker startup. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartedProcess { - /// Token that must be acknowledged before the child may begin guest execution. - pub token: ProcessStartToken, /// Broker-assigned child process ID. pub process_id: ProcessId, /// Broker-assigned initial thread ID when it differs from the process ID. diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 01aa2946d1..3b9b3691c3 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -24,7 +24,7 @@ use crate::message::{ }; use crate::process::{ InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartToken, ProcessStartupDescriptor, StartedProcess, + ProcessBootstrapVersion, ProcessStartupDescriptor, StartedProcess, }; use crate::readiness::ReadinessFlags; @@ -49,7 +49,7 @@ const REQUEST_TAG_FILE: u8 = 8; const REQUEST_TAG_CREATE_THREAD: u8 = 9; const REQUEST_TAG_EXIT_THREAD: u8 = 10; const REQUEST_TAG_START_PROCESS: u8 = 11; -const REQUEST_TAG_ACKNOWLEDGE_PROCESS_START: u8 = 12; +// Tag 12 is reserved for the removed process-start acknowledgement. const REQUEST_TAG_PROCESS_READY: u8 = 13; const REQUEST_TAG_REPORT_PROCESS_START_FAILURE: u8 = 14; @@ -66,7 +66,7 @@ const RESPONSE_TAG_FILE: u8 = 8; const RESPONSE_TAG_THREAD_CREATED: u8 = 9; const RESPONSE_TAG_THREAD_EXITED: u8 = 10; const RESPONSE_TAG_PROCESS_STARTED: u8 = 11; -const RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED: u8 = 12; +// Tag 12 is reserved for the removed process-start acknowledgement. const RESPONSE_TAG_PROCESS_READY: u8 = 13; const RESPONSE_TAG_PROCESS_START_FAILED: u8 = 14; @@ -129,7 +129,6 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { return Err(WireError::WrongMessagePhase); @@ -213,11 +212,6 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.shared_buffer_sequence(buffer); encode_inherited_objects(&mut encoder, inherited_objects); } - BrokerOperation::AcknowledgeProcessStart(token) => { - encoder.u8(REQUEST_TAG_ACKNOWLEDGE_PROCESS_START); - encoder.request_id(request_id); - encoder.u64(token.0); - } BrokerOperation::ReportProcessReady(initial_thread_id) => { encoder.u8(REQUEST_TAG_PROCESS_READY); encoder.request_id(request_id); @@ -249,7 +243,6 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_CREATE_THREAD | REQUEST_TAG_EXIT_THREAD | REQUEST_TAG_START_PROCESS - | REQUEST_TAG_ACKNOWLEDGE_PROCESS_START | REQUEST_TAG_PROCESS_READY | REQUEST_TAG_REPORT_PROCESS_START_FAILURE => {} _ => return Err(WireError::InvalidTag), @@ -272,9 +265,6 @@ pub fn decode_request(frame: &[u8]) -> Result { buffer: decoder.shared_buffer_sequence()?, inherited_objects: decode_inherited_objects(&mut decoder)?, }), - REQUEST_TAG_ACKNOWLEDGE_PROCESS_START => { - BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(decoder.u64()?)) - } REQUEST_TAG_PROCESS_READY => { BrokerOperation::ReportProcessReady(decode_optional_thread_id(&mut decoder)?) } @@ -366,7 +356,6 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { return Err(WireError::WrongMessagePhase); @@ -439,20 +428,14 @@ pub fn encode_response(response: BrokerResponse) -> Vec { fs::encode_fs_response(&mut encoder, response); } BrokerResult::ProcessStarted(StartedProcess { - token, process_id, initial_thread_id, }) => { encoder.u8(RESPONSE_TAG_PROCESS_STARTED); encoder.request_id(request_id); - encoder.u64(token.0); encoder.process_id(process_id); encode_optional_thread_id(&mut encoder, initial_thread_id); } - BrokerResult::ProcessStartAcknowledged => { - encoder.u8(RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED); - encoder.request_id(request_id); - } BrokerResult::ProcessStartFailed(error) => { encoder.u8(RESPONSE_TAG_PROCESS_START_FAILED); encoder.request_id(request_id); @@ -491,7 +474,6 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_THREAD_CREATED | RESPONSE_TAG_THREAD_EXITED | RESPONSE_TAG_PROCESS_STARTED - | RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED | RESPONSE_TAG_PROCESS_START_FAILED | RESPONSE_TAG_PROCESS_READY => {} _ => return Err(WireError::InvalidTag), @@ -510,11 +492,9 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_STDIO => BrokerResult::Stdio(stdio::decode_stdio_response(&mut decoder)?), RESPONSE_TAG_FILE => BrokerResult::File(fs::decode_fs_response(&mut decoder)?), RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(StartedProcess { - token: ProcessStartToken(decoder.u64()?), process_id: decoder.process_id()?, initial_thread_id: decode_optional_thread_id(&mut decoder)?, }), - RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED => BrokerResult::ProcessStartAcknowledged, RESPONSE_TAG_PROCESS_START_FAILED => { BrokerResult::ProcessStartFailed(decode_error_code(&mut decoder)?) } @@ -655,7 +635,7 @@ mod tests { }; use crate::process::{ InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, - ProcessStartToken, ProcessStartupDescriptor, StartedProcess, + ProcessStartupDescriptor, StartedProcess, }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ @@ -715,7 +695,6 @@ mod tests { RESPONSE_TAG_THREAD_CREATED, RESPONSE_TAG_THREAD_EXITED, RESPONSE_TAG_PROCESS_STARTED, - RESPONSE_TAG_PROCESS_START_ACKNOWLEDGED, RESPONSE_TAG_PROCESS_READY, RESPONSE_TAG_PROCESS_START_FAILED, ], @@ -732,7 +711,6 @@ mod tests { REQUEST_TAG_CREATE_THREAD, REQUEST_TAG_EXIT_THREAD, REQUEST_TAG_START_PROCESS, - REQUEST_TAG_ACKNOWLEDGE_PROCESS_START, REQUEST_TAG_PROCESS_READY, REQUEST_TAG_REPORT_PROCESS_START_FAILURE, ] @@ -1015,7 +993,6 @@ mod tests { ]) .unwrap(), }), - BrokerOperation::AcknowledgeProcessStart(ProcessStartToken(u64::MAX)), BrokerOperation::ReportProcessReady(None), BrokerOperation::ReportProcessReady(Some(thread_id(19))), BrokerOperation::ReportProcessStartFailure(ErrorCode::UnsupportedOperation), @@ -1363,16 +1340,13 @@ mod tests { BrokerResult::File(FileResponse::Rmdir), BrokerResult::File(FileResponse::Failed(FileError::Io)), BrokerResult::ProcessStarted(StartedProcess { - token: ProcessStartToken(u64::MAX), process_id: process_id(u32::MAX), initial_thread_id: None, }), BrokerResult::ProcessStarted(StartedProcess { - token: ProcessStartToken(7), process_id: process_id(9), initial_thread_id: Some(thread_id(11)), }), - BrokerResult::ProcessStartAcknowledged, BrokerResult::ProcessStartFailed(ErrorCode::PeerClosed), BrokerResult::ProcessReady, BrokerResult::Error(ErrorCode::PolicyDenied), diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs index 90b5c1f69c..2047d8fcfe 100644 --- a/litebox_broker_transport/src/pending_calls.rs +++ b/litebox_broker_transport/src/pending_calls.rs @@ -13,7 +13,7 @@ use litebox_broker_protocol::message::BrokerResponse; /// Maximum number of active calls waiting for broker responses. pub const MAX_PENDING_CALLS: usize = 64; -/// Pending-call capacity unavailable to ordinary operations. +/// Pending-call capacity reserved for control requests that must report failure. pub const RESERVED_PENDING_CALL_CAPACITY: usize = 8; /// Maximum active ordinary calls after preserving reserved capacity. pub const MAX_ORDINARY_PENDING_CALLS: usize = MAX_PENDING_CALLS - RESERVED_PENDING_CALL_CAPACITY; diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index d85ae82e3e..6bec67ff45 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -323,8 +323,7 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { let request_id = request.request_id; let pending_call = if matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) - | BrokerOperation::ReportProcessStartFailure(_) + BrokerOperation::ReportProcessStartFailure(_) ) { association .pending_calls @@ -775,7 +774,7 @@ mod control_ring_tests { } #[test] - fn pending_capacity_preserves_process_start_calls() { + fn pending_capacity_preserves_process_start_failure_reports() { use litebox_broker_transport::pending_calls::{ MAX_ORDINARY_PENDING_CALLS, RESERVED_PENDING_CALL_CAPACITY, }; @@ -800,8 +799,8 @@ mod control_ring_tests { reserved_start.wait(); reserved_channel.call(BrokerRequest { request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), - operation: BrokerOperation::AcknowledgeProcessStart( - litebox_broker_protocol::process::ProcessStartToken(index as u64), + operation: BrokerOperation::ReportProcessStartFailure( + litebox_broker_protocol::error::ErrorCode::UnsupportedOperation, ), }) }) @@ -814,14 +813,14 @@ mod control_ring_tests { } assert!(published.iter().any(|request| matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) + BrokerOperation::ReportProcessStartFailure(_) ))); assert_eq!( published .iter() .filter(|request| matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) + BrokerOperation::ReportProcessStartFailure(_) )) .count(), RESERVED_PENDING_CALL_CAPACITY diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 71712358fe..42dddaf266 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -232,8 +232,7 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { let request_id = request.request_id; let pending_call = if matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) - | BrokerOperation::ReportProcessStartFailure(_) + BrokerOperation::ReportProcessStartFailure(_) ) { association .pending_calls diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index bca8816a4a..67d9b96ebc 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -507,14 +507,14 @@ mod tests { } assert!(published.iter().any(|request| matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) + BrokerOperation::ReportProcessStartFailure(_) ))); assert_eq!( published .iter() .filter(|request| matches!( &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) + BrokerOperation::ReportProcessStartFailure(_) )) .count(), RESERVED_PENDING_CALL_CAPACITY @@ -575,8 +575,8 @@ mod tests { reserved_start.wait(); reserved_calls.call(BrokerRequest { request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), - operation: BrokerOperation::AcknowledgeProcessStart( - litebox_broker_protocol::process::ProcessStartToken(index as u64), + operation: BrokerOperation::ReportProcessStartFailure( + litebox_broker_protocol::error::ErrorCode::UnsupportedOperation, ), }) }) diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index 34ca8f9808..d777a4edeb 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -15,6 +15,8 @@ use std::time::{Duration, Instant}; use litebox_broker_core::BrokerCore; +use crate::runtime::AssociationFailureCause; + #[cfg(target_os = "linux")] mod linux; mod process_manager; @@ -110,6 +112,43 @@ enum RunnerShutdownState { Retired, } +struct RunnerCompletion { + result: IoResult, + runner_success: Option, + runner_signal: Option, + runner_exit_code: Option, + termination_provenance: TerminationProvenance, + association_panicked: bool, + shutdown_observation_failed: bool, +} + +#[derive(Clone, Copy, Default)] +struct TerminationProvenance(u8); + +impl TerminationProvenance { + const BROKER_TERMINATION: u8 = 1; + const REPORTED_START_FAILURE: u8 = 2; + + const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { + let mut value = 0; + if broker_termination { + value |= Self::BROKER_TERMINATION; + } + if reported_start_failure { + value |= Self::REPORTED_START_FAILURE; + } + Self(value) + } + + const fn broker_termination(self) -> bool { + self.0 & Self::BROKER_TERMINATION != 0 + } + + const fn reported_start_failure(self) -> bool { + self.0 & Self::REPORTED_START_FAILURE != 0 + } +} + impl RunnerShutdown { fn shutdown(&self) { let mut state = self.state.lock().expect("runner shutdown mutex poisoned"); @@ -257,6 +296,88 @@ impl RunnerInstance { association_result.result?; Ok(runner_status) } + + fn run_started_process_to_completion( + mut self, + startup: RunnerStartup, + process_manager: Arc, + ) -> RunnerCompletion { + let transaction = startup.transaction(); + transaction.install_shutdown(Arc::clone(&self.shutdown)); + let association_result = self + .endpoint + .serve(&self.runner, Some(startup), process_manager); + self.endpoint.close(); + let shutdown_request = transaction.shutdown_request(); + let shutdown_was_expected = shutdown_request.was_expected(); + if association_result.abnormal { + transaction.mark_abnormal(); + } + let runner_exited = if !shutdown_was_expected + && matches!( + association_result.failure_cause, + AssociationFailureCause::None | AssociationFailureCause::PeerClosed + ) + && !association_result.panicked + { + self.shutdown + .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) + } else { + self.shutdown.has_exited() + }; + let shutdown_observation_failed = match runner_exited { + Ok(true) => { + if association_result.failure_cause == AssociationFailureCause::Other { + transaction.mark_abnormal(); + } + false + } + Ok(false) => { + if !shutdown_was_expected + || association_result.failure_cause == AssociationFailureCause::Other + { + transaction.mark_abnormal(); + } + transaction.mark_shutdown_expected(); + self.shutdown.shutdown(); + false + } + Err(_) => { + transaction.mark_abnormal(); + self.shutdown.shutdown(); + true + } + }; + self.shutdown.retire(); + let runner_status = wait_for_runner_exit(&self.runner); + if runner_status.is_err() { + transaction.mark_abnormal(); + } + let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); + let runner_signal = runner_status + .as_ref() + .ok() + .copied() + .and_then(runner_exit_signal); + let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); + let termination_provenance = TerminationProvenance::new( + self.shutdown.termination_was_dispatched(), + shutdown_request.expected_start_failure_was_reported(), + ); + let result = runner_status.and_then(|status| { + association_result.result?; + Ok(status) + }); + RunnerCompletion { + result, + runner_success, + runner_signal, + runner_exit_code, + termination_provenance, + association_panicked: association_result.panicked, + shutdown_observation_failed, + } + } } impl Drop for RunnerInstance { diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index 7f4b32a5da..b57cf8c06d 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -4,8 +4,7 @@ //! Shared coordination for out-of-process runner instances. use std::io::{Error as IoError, Result as IoResult}; -use std::process::ExitStatus; -use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; @@ -14,28 +13,19 @@ use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartToken, ProcessStartupData, StartedProcess, + ProcessBootstrapVersion, ProcessStartupData, StartedProcess, }; use litebox_broker_protocol::{ProcessId, ThreadId}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use super::{ - PROCESS_EXIT_OBSERVATION_TIMEOUT, RunnerConfig, RunnerInstance, RunnerShutdown, - runner_exit_code_is_crash, runner_exit_code_is_expected_shutdown, runner_exit_signal, - runner_signal_is_abnormal, wait_for_runner_exit, + RunnerCompletion, RunnerConfig, RunnerInstance, RunnerShutdown, TerminationProvenance, + runner_exit_code_is_crash, runner_exit_code_is_expected_shutdown, runner_signal_is_abnormal, }; -use crate::runtime::AssociationFailureCause; -const PROCESS_START_RECEIPT_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT: Duration = Duration::from_secs(5); -const PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); -const MAX_PENDING_PROCESS_STARTS: usize = crate::WORKER_COUNT - 1; +const PROCESS_START_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_PENDING_PROCESS_STARTS: usize = crate::WORKER_COUNT; const _: () = assert!(MAX_PENDING_PROCESS_STARTS > 0); -const _: () = - assert!(crate::runtime::PROCESS_START_CONTROL_WORKER_COUNT > MAX_PENDING_PROCESS_STARTS); -const _: () = - assert!(crate::runtime::PROCESS_START_CONTROL_QUEUE_CAPACITY >= MAX_PENDING_PROCESS_STARTS); /// Shared ownership and coordination for runner processes and associations. pub(crate) struct RunnerProcessManager { @@ -53,64 +43,25 @@ pub(crate) struct RunnerStartup { } impl RunnerStartup { - pub(crate) fn into_process_and_data(self) -> (Arc, ProcessStartupData) { - (self.process, self.data) - } -} - -struct RunnerCompletion { - result: IoResult, - runner_success: Option, - runner_signal: Option, - runner_exit_code: Option, - termination_provenance: TerminationProvenance, - association_panicked: bool, - shutdown_observation_failed: bool, -} - -#[derive(Clone, Copy, Default)] -struct TerminationProvenance(u8); - -impl TerminationProvenance { - const BROKER_TERMINATION: u8 = 1; - const REPORTED_START_FAILURE: u8 = 2; - - const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { - let mut value = 0; - if broker_termination { - value |= Self::BROKER_TERMINATION; - } - if reported_start_failure { - value |= Self::REPORTED_START_FAILURE; - } - Self(value) - } - - const fn broker_termination(self) -> bool { - self.0 & Self::BROKER_TERMINATION != 0 + pub(super) fn transaction(&self) -> Arc { + Arc::clone(&self.transaction) } - const fn reported_start_failure(self) -> bool { - self.0 & Self::REPORTED_START_FAILURE != 0 + pub(crate) fn into_process_and_data(self) -> (Arc, ProcessStartupData) { + (self.process, self.data) } } struct RunnerProcessManagerState { - transactions: Vec<(ProcessStartToken, Arc)>, + transactions: Vec>, associations: Vec<(ProcessId, AssociationFailure)>, active_instances: usize, - active_watchdogs: usize, } pub(crate) type AssociationFailure = Arc; -/// Transactions retained while an ending association drains in-flight work. -pub(crate) struct AssociationDrain { - transactions: Vec>, -} - -/// State for one `StartProcess` request from admission through finalization. -struct ProcessStartTransaction { +/// State for one blocking `StartProcess` request. +pub(super) struct ProcessStartTransaction { parent_id: ProcessId, process: Arc, state: Mutex, @@ -121,176 +72,44 @@ struct ProcessStartTransaction { enum ProcessStartPhase { Starting, Ready { initial_thread_id: Option }, - Committing, - Committed, - CompletingStart, - StartComplete, - Aborted(ErrorCode), -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum StartResultPublication { - NotStarted, - Publishing, - Delivered, + Completing { cancellation: Option }, + Running, + Failed(ErrorCode), } struct ProcessStartTransactionState { phase: ProcessStartPhase, - publication: StartResultPublication, shutdown: Option>, association_failure: Option, shutdown_request: ShutdownRequest, abnormal: bool, - receipt: ReceiptState, - resolution_watchdog: DeadlineState, - acknowledgement_publication_watchdog: DeadlineState, - start_failure_publication_watchdog: DeadlineState, - active_control_callbacks: usize, - runner_finished: bool, - finalization_taken: bool, + start_completed: bool, + finalization: FinalizationState, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FinalizationState { + Active, + ProtocolFinished, + RunnerFinished, + Finalized, } #[derive(Clone, Copy, PartialEq, Eq)] -enum ShutdownRequest { +pub(super) enum ShutdownRequest { None, Expected, - ExpectedStartFailurePending, ExpectedStartFailure, Unexpected, } impl ShutdownRequest { - const fn was_expected(self) -> bool { - matches!( - self, - Self::Expected | Self::ExpectedStartFailurePending | Self::ExpectedStartFailure - ) - } - - const fn expected_start_failure_was_reported(self) -> bool { - matches!( - self, - Self::ExpectedStartFailurePending | Self::ExpectedStartFailure - ) + pub(super) const fn was_expected(self) -> bool { + matches!(self, Self::Expected | Self::ExpectedStartFailure) } -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum ReceiptState { - AwaitingAcknowledgement, - AcknowledgementAdmitted, - TimeoutPending, - Draining, - Resolved, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum DeadlineState { - Unarmed, - Armed(Instant), - Disarmed, - Fired, -} - -enum ProcessStartAcknowledgement { - Acknowledged, - Failed(ErrorCode), -} -struct ReceiptExpiration { - shutdown: Option>, - association_failure: Option, - fail_parent: bool, - commit_supervision_deadline: Option, -} - -enum ReceiptResolution { - Resolved(Option), - DeferredToDrain, -} - -impl RunnerInstance { - fn run_started_process_to_completion( - mut self, - startup: RunnerStartup, - process_manager: Arc, - ) -> RunnerCompletion { - let start = Arc::clone(&startup.transaction); - start.install_shutdown(Arc::clone(&self.shutdown)); - let association_result = self - .endpoint - .serve(&self.runner, Some(startup), process_manager); - self.endpoint.close(); - let shutdown_request = start.shutdown_request(); - let shutdown_was_expected = shutdown_request.was_expected(); - if association_result.abnormal { - start.mark_abnormal(); - } - let runner_exited = if !shutdown_was_expected - && matches!( - association_result.failure_cause, - AssociationFailureCause::None | AssociationFailureCause::PeerClosed - ) - && !association_result.panicked - { - self.shutdown - .wait_for_exit(PROCESS_EXIT_OBSERVATION_TIMEOUT) - } else { - self.shutdown.has_exited() - }; - let shutdown_observation_failed = match runner_exited { - Ok(true) => { - if association_result.failure_cause == AssociationFailureCause::Other { - start.mark_abnormal(); - } - false - } - Ok(false) => { - if !shutdown_was_expected - || association_result.failure_cause == AssociationFailureCause::Other - { - start.mark_abnormal(); - } - start.mark_shutdown_expected(); - self.shutdown.shutdown(); - false - } - Err(_) => { - start.mark_abnormal(); - self.shutdown.shutdown(); - true - } - }; - self.shutdown.retire(); - let runner_status = wait_for_runner_exit(&self.runner); - if runner_status.is_err() { - start.mark_abnormal(); - } - let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); - let runner_signal = runner_status - .as_ref() - .ok() - .copied() - .and_then(runner_exit_signal); - let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); - let termination_provenance = TerminationProvenance::new( - self.shutdown.termination_was_dispatched(), - shutdown_request.expected_start_failure_was_reported(), - ); - let result = runner_status.and_then(|status| { - association_result.result?; - Ok(status) - }); - RunnerCompletion { - result, - runner_success, - runner_signal, - runner_exit_code, - termination_provenance, - association_panicked: association_result.panicked, - shutdown_observation_failed, - } + pub(super) const fn expected_start_failure_was_reported(self) -> bool { + matches!(self, Self::ExpectedStartFailure) } } @@ -303,7 +122,6 @@ impl RunnerProcessManager { transactions: Vec::new(), associations: Vec::new(), active_instances: 0, - active_watchdogs: 0, }), drained: Condvar::new(), }) @@ -334,17 +152,6 @@ impl RunnerProcessManager { }) .map(BrokerResult::ProcessStarted), ), - BrokerOperation::AcknowledgeProcessStart(token) => { - Some(match self.resolve_acknowledgement(process.id(), *token) { - Ok(ProcessStartAcknowledgement::Acknowledged) => { - Ok(BrokerResult::ProcessStartAcknowledged) - } - Ok(ProcessStartAcknowledgement::Failed(error)) => { - Ok(BrokerResult::ProcessStartFailed(error)) - } - Err(error) => Err(RequestFailure::Abort(error)), - }) - } BrokerOperation::ReportProcessReady(initial_thread_id) => Some( self.process_ready(process.id(), *initial_thread_id) .map(|()| BrokerResult::ProcessReady) @@ -359,50 +166,6 @@ impl RunnerProcessManager { } } - pub(crate) fn response_sent( - &self, - process_id: ProcessId, - operation: &BrokerOperation, - result: &BrokerResult, - ) { - match (operation, result) { - (_, BrokerResult::ProcessStarted(started)) => { - let Some(start) = self.find_transaction(started.token) else { - return; - }; - if start.parent_id == process_id { - start.mark_start_result_delivered(); - } - } - ( - BrokerOperation::AcknowledgeProcessStart(token), - BrokerResult::ProcessStartAcknowledged | BrokerResult::ProcessStartFailed(_), - ) => { - let Some(start) = self.find_transaction(*token) else { - return; - }; - if start.parent_id != process_id { - return; - } - if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { - self.remove_transaction(*token); - if let Some(abnormal) = finalization { - self.finish_transaction(&start, abnormal); - } - } - } - ( - BrokerOperation::ReportProcessStartFailure(error), - BrokerResult::ProcessStartFailed(reported), - ) if error == reported => { - if let Some(start) = self.find_transaction_by_process_id(process_id) { - start.start_failure_response_sent(*error); - } - } - _ => {} - } - } - fn start_process( self: &Arc, parent: &BrokerProcess, @@ -420,71 +183,48 @@ impl RunnerProcessManager { let inherited_objects = InheritedProcessObjects::new(&inherited_objects) .expect("child handle count must match the bounded inheritance request"); let process_id = process.id(); - let start = Arc::new(ProcessStartTransaction { + let transaction = Arc::new(ProcessStartTransaction { parent_id: parent.id(), process: Arc::clone(&process), state: Mutex::new(ProcessStartTransactionState { phase: ProcessStartPhase::Starting, - publication: StartResultPublication::NotStarted, shutdown: None, association_failure: None, shutdown_request: ShutdownRequest::None, abnormal: false, - receipt: ReceiptState::AwaitingAcknowledgement, - resolution_watchdog: DeadlineState::Unarmed, - acknowledgement_publication_watchdog: DeadlineState::Unarmed, - start_failure_publication_watchdog: DeadlineState::Unarmed, - active_control_callbacks: 0, - runner_finished: false, - finalization_taken: false, + start_completed: false, + finalization: FinalizationState::Active, }), changed: Condvar::new(), }); - let token = match (|| { - loop { - let mut token = [0; 8]; - getrandom::fill(&mut token).map_err(|_| ErrorCode::Internal)?; - let token = ProcessStartToken(u64::from_ne_bytes(token)); - let mut state = self - .state - .lock() - .expect("runner process manager state mutex poisoned"); - state - .transactions - .try_reserve(1) - .map_err(|_| ErrorCode::OutOfMemory)?; - if parent.is_cancellation_requested() { - return Err(ErrorCode::PeerClosed); - } - if state.transactions.len() >= MAX_PENDING_PROCESS_STARTS { - return Err(ErrorCode::ResourceExhausted); - } - if state - .transactions - .iter() - .any(|(candidate, _)| *candidate == token) - { - continue; - } - let active_instances = state - .active_instances - .checked_add(1) - .ok_or(ErrorCode::ResourceExhausted)?; - state.transactions.push((token, Arc::clone(&start))); + + let registration = { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + if state.transactions.try_reserve(1).is_err() { + Err(ErrorCode::OutOfMemory) + } else if parent.is_cancellation_requested() { + Err(ErrorCode::PeerClosed) + } else if state.transactions.len() >= MAX_PENDING_PROCESS_STARTS { + Err(ErrorCode::ResourceExhausted) + } else if let Some(active_instances) = state.active_instances.checked_add(1) { state.active_instances = active_instances; - return Ok(token); - } - })() { - Ok(token) => token, - Err(error) => { - process.cleanup(true); - return Err(error); + state.transactions.push(Arc::clone(&transaction)); + Ok(()) + } else { + Err(ErrorCode::ResourceExhausted) } }; + if let Err(error) = registration { + process.cleanup(true); + return Err(error); + } let process_manager = Arc::clone(self); let config = self.started_runner_config.clone(); - let thread_transaction = Arc::clone(&start); + let thread_transaction = Arc::clone(&transaction); let thread = std::thread::Builder::new() .name(format!("litebox-runner-{}", process_id.0)) .spawn(move || { @@ -507,10 +247,9 @@ impl RunnerProcessManager { })); match outcome { Ok(Ok(result)) => { - process_manager.runner_finished(token, &thread_transaction, result, false); + process_manager.runner_finished(&thread_transaction, result, false); } Ok(Err(error)) => process_manager.runner_finished( - token, &thread_transaction, RunnerCompletion { result: Err(error), @@ -524,7 +263,6 @@ impl RunnerProcessManager { false, ), Err(_) => process_manager.runner_finished( - token, &thread_transaction, RunnerCompletion { result: Err(IoError::other("runner process thread panicked")), @@ -540,89 +278,32 @@ impl RunnerProcessManager { } }); if thread.is_err() { - self.remove_transaction(token); - start.process.cleanup(true); + self.remove_transaction(process_id); + transaction.process.cleanup(true); self.finish_instance(); return Err(ErrorCode::OutOfMemory); } drop(thread); - let initial_thread_id = start.wait_until_ready()?; - let receipt_deadline = start.begin_start_result_publication()?; - if self - .arm_initial_receipt_deadline(token, &start, receipt_deadline) - .is_err() - { - if let Some(expiration) = start.expire_initial_receipt_deadline() { - self.apply_receipt_expiration( - token, - &start, - expiration, - self.find_association_failure(start.parent_id), - ); - } - return Err(ErrorCode::OutOfMemory); - } - Ok(StartedProcess { - token, - process_id, - initial_thread_id, - }) - } - - pub(crate) fn admit_acknowledgement( - self: &Arc, - parent_id: ProcessId, - token: ProcessStartToken, - ) -> Result<(), ErrorCode> { - let start = self - .find_transaction(token) - .ok_or(ErrorCode::UnknownObject)?; - if start.parent_id != parent_id { - return Err(ErrorCode::UnknownObject); - } - start.admit_acknowledgement()?; - if self - .arm_internal_resolution_watchdog(token, &start) - .is_err() - { - if let Some(expiration) = start.fail_internal_resolution_watchdog() { - self.apply_receipt_expiration( - token, - &start, - expiration, - self.find_association_failure(start.parent_id), - ); - } - return Err(ErrorCode::Internal); - } - if self - .arm_acknowledgement_publication_watchdog(token, &start) - .is_err() - { - if let Some(expiration) = start.fail_internal_resolution_watchdog() { - self.apply_receipt_expiration( - token, - &start, - expiration, - self.find_association_failure(start.parent_id), - ); - } - return Err(ErrorCode::Internal); - } - Ok(()) - } - - fn resolve_acknowledgement( - &self, - parent_id: ProcessId, - token: ProcessStartToken, - ) -> Result { - let start = self.find_transaction(token).ok_or(ErrorCode::PeerClosed)?; - if start.parent_id != parent_id { - return Err(ErrorCode::UnknownObject); + let deadline = Instant::now() + PROCESS_START_TIMEOUT; + let result = transaction + .wait_until_ready(deadline) + .and_then(|initial_thread_id| { + if parent.is_cancellation_requested() { + transaction.abort(ErrorCode::PeerClosed, false, true); + return Err(ErrorCode::PeerClosed); + } + transaction.complete_start()?; + Ok(StartedProcess { + process_id, + initial_thread_id, + }) + }); + self.remove_transaction(process_id); + if let Some(abnormal) = transaction.protocol_finished() { + self.finish_transaction(&transaction, abnormal); } - start.resolve_acknowledgement() + result } fn process_ready( @@ -630,74 +311,50 @@ impl RunnerProcessManager { process_id: ProcessId, initial_thread_id: Option, ) -> Result<(), ErrorCode> { - let start = self - .find_transaction_by_process_id(process_id) - .ok_or(ErrorCode::PeerClosed)?; - start.ready_and_wait(initial_thread_id) + self.find_transaction(process_id) + .ok_or(ErrorCode::PeerClosed)? + .ready_and_wait(initial_thread_id) } fn report_process_start_failure( - self: &Arc, + &self, process_id: ProcessId, error: ErrorCode, ) -> Result<(), ErrorCode> { - let (token, start) = self - .find_transaction_entry_by_process_id(process_id) - .ok_or(ErrorCode::PeerClosed)?; - start.report_start_failure(error)?; - if self - .arm_start_failure_publication_watchdog(token, &start) - .is_err() - { - if let Some(expiration) = start.fail_start_failure_publication_watchdog() { - self.apply_receipt_expiration(token, &start, expiration, None); - } - return Err(ErrorCode::Internal); - } - Ok(()) + self.find_transaction(process_id) + .ok_or(ErrorCode::PeerClosed)? + .report_start_failure(error) } - pub(crate) fn association_ending(&self, process_id: ProcessId) -> AssociationDrain { - let draining_transactions = self - .state - .lock() - .expect("runner process manager state mutex poisoned") - .transactions - .iter() - .filter(|(_, start)| start.parent_id == process_id) - .map(|(_, start)| Arc::clone(start)) - .collect::>(); - for start in &draining_transactions { - start.abort(ErrorCode::PeerClosed, false, true); - start.begin_receipt_drain(); - } - - let transaction = { - self.state + pub(crate) fn association_ending(&self, process_id: ProcessId) { + let (children, transaction) = { + let state = self + .state .lock() - .expect("runner process manager state mutex poisoned") + .expect("runner process manager state mutex poisoned"); + let children = state .transactions .iter() - .find_map(|(_, start)| { - (start.process.id() == process_id).then(|| Arc::clone(start)) - }) + .filter(|transaction| transaction.parent_id == process_id) + .cloned() + .collect::>(); + let transaction = state + .transactions + .iter() + .find(|transaction| transaction.process.id() == process_id) + .cloned(); + (children, transaction) }; - if let Some(start) = transaction { - start.association_closed(); + for child in children { + child.abort(ErrorCode::PeerClosed, false, true); } - AssociationDrain { - transactions: draining_transactions, + if let Some(transaction) = transaction { + transaction.association_closed(); } } - pub(crate) fn association_ended(&self, process_id: ProcessId, draining: AssociationDrain) { + pub(crate) fn association_ended(&self, process_id: ProcessId) { self.unregister_association(process_id); - for start in draining.transactions { - self.remove_transaction_by_process_id(start.process.id()); - if let Some(abnormal) = start.finish_receipt_drain() { - self.finish_transaction(&start, abnormal); - } - } } pub(crate) fn register_association( @@ -705,37 +362,37 @@ impl RunnerProcessManager { process_id: ProcessId, failure: AssociationFailure, ) -> IoResult<()> { - let mut state = self - .state - .lock() - .expect("runner process manager state mutex poisoned"); - state - .associations - .try_reserve(1) - .map_err(|_| IoError::other("failed to reserve broker association registration"))?; - if state - .associations - .iter() - .any(|(candidate, _)| *candidate == process_id) - { - return Err(IoError::other( - "a broker process already has a live association", - )); + let transaction = { + let mut state = self + .state + .lock() + .expect("runner process manager state mutex poisoned"); + state + .associations + .try_reserve(1) + .map_err(|_| IoError::other("failed to reserve broker association registration"))?; + if state + .associations + .iter() + .any(|(candidate, _)| *candidate == process_id) + { + return Err(IoError::other( + "a broker process already has a live association", + )); + } + state.associations.push((process_id, failure.clone())); + state + .transactions + .iter() + .find(|transaction| transaction.process.id() == process_id) + .cloned() + }; + if let Some(transaction) = transaction { + transaction.install_association_failure(failure); } - state.associations.push((process_id, failure)); Ok(()) } - pub(crate) fn install_process_association_failure( - &self, - process_id: ProcessId, - failure: AssociationFailure, - ) { - if let Some(start) = self.find_transaction_by_process_id(process_id) { - start.install_association_failure(failure); - } - } - fn unregister_association(&self, process_id: ProcessId) { let mut state = self .state @@ -750,203 +407,17 @@ impl RunnerProcessManager { } } - fn find_association_failure(&self, process_id: ProcessId) -> Option { - self.state - .lock() - .expect("runner process manager state mutex poisoned") - .associations - .iter() - .find_map(|(candidate, failure)| { - (*candidate == process_id).then(|| Arc::clone(failure)) - }) - } - - fn arm_initial_receipt_deadline( - self: &Arc, - token: ProcessStartToken, - start: &Arc, - deadline: Instant, - ) -> Result<(), ()> { - let start = Arc::clone(start); - let parent_failure = self.find_association_failure(start.parent_id); - self.spawn_watchdog( - format!("litebox-start-receipt-{}", start.process.id().0), - move |process_manager| { - let Some(expiration) = start.wait_for_initial_receipt_deadline(deadline) else { - return; - }; - process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); - }, - ) - } - - fn arm_internal_resolution_watchdog( - self: &Arc, - token: ProcessStartToken, - start: &Arc, - ) -> Result<(), ()> { - let start = Arc::clone(start); - let parent_failure = self.find_association_failure(start.parent_id); - self.spawn_watchdog( - format!("litebox-start-resolution-{}", start.process.id().0), - move |process_manager| { - let Some(expiration) = start.wait_for_internal_resolution_timeout() else { - return; - }; - process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); - }, - ) - } - - fn arm_acknowledgement_publication_watchdog( - self: &Arc, - token: ProcessStartToken, - start: &Arc, - ) -> Result<(), ()> { - let start = Arc::clone(start); - let parent_failure = self.find_association_failure(start.parent_id); - self.spawn_watchdog( - format!("litebox-start-ack-publication-{}", start.process.id().0), - move |process_manager| { - let Some(expiration) = start.wait_for_acknowledgement_publication_timeout() else { - return; - }; - process_manager.apply_receipt_expiration(token, &start, expiration, parent_failure); - }, - ) - } - - fn arm_start_failure_publication_watchdog( - self: &Arc, - token: ProcessStartToken, - start: &Arc, - ) -> Result<(), ()> { - let start = Arc::clone(start); - self.spawn_watchdog( - format!("litebox-start-failure-publication-{}", start.process.id().0), - move |process_manager| { - let Some(expiration) = start.wait_for_start_failure_publication_timeout() else { - return; - }; - process_manager.apply_receipt_expiration(token, &start, expiration, None); - }, - ) - } - - fn spawn_watchdog( - self: &Arc, - name: String, - watchdog: impl FnOnce(&Arc) + Send + 'static, - ) -> Result<(), ()> { - { - let mut state = self - .state - .lock() - .expect("runner process manager state mutex poisoned"); - state.active_watchdogs = state.active_watchdogs.checked_add(1).ok_or(())?; - } - let process_manager = Arc::clone(self); - if let Ok(thread) = std::thread::Builder::new().name(name).spawn(move || { - let _completion = WatchdogCompletion { - process_manager: Arc::clone(&process_manager), - }; - watchdog(&process_manager); - }) { - drop(thread); - Ok(()) - } else { - self.finish_watchdog(); - Err(()) - } - } - - fn apply_receipt_expiration( - &self, - token: ProcessStartToken, - start: &ProcessStartTransaction, - expiration: ReceiptExpiration, - parent_failure: Option, - ) { - let fail_parent = expiration.fail_parent; - let commit_supervision_deadline = expiration.commit_supervision_deadline; - if let Some(association_failure) = expiration.association_failure { - association_failure(); - } - if let Some(shutdown) = expiration.shutdown { - shutdown.shutdown(); - } - let parent_failed = if fail_parent { - if let Some(parent_failure) = parent_failure { - parent_failure(); - true - } else { - false - } - } else { - true - }; - if let Some(abnormal) = start.complete_timeout_callback() { - self.finish_transaction(start, abnormal); - } - if fail_parent && !parent_failed { - self.remove_transaction(token); - if let Some(abnormal) = start.finish_receipt_drain() { - self.finish_transaction(start, abnormal); - } - } - if let Some(deadline) = commit_supervision_deadline - && !start.wait_for_commit_resolution(deadline) - { - std::process::abort(); - } - } - - fn find_transaction(&self, token: ProcessStartToken) -> Option> { - self.state - .lock() - .expect("runner process manager state mutex poisoned") - .transactions - .iter() - .find_map(|(candidate, start)| (*candidate == token).then(|| Arc::clone(start))) - } - - fn find_transaction_by_process_id( - &self, - process_id: ProcessId, - ) -> Option> { - self.find_transaction_entry_by_process_id(process_id) - .map(|(_, start)| start) - } - - fn find_transaction_entry_by_process_id( - &self, - process_id: ProcessId, - ) -> Option<(ProcessStartToken, Arc)> { + fn find_transaction(&self, process_id: ProcessId) -> Option> { self.state .lock() .expect("runner process manager state mutex poisoned") .transactions .iter() - .find_map(|(token, start)| { - (start.process.id() == process_id).then(|| (*token, Arc::clone(start))) - }) - } - - fn remove_transaction(&self, token: ProcessStartToken) { - let mut state = self - .state - .lock() - .expect("runner process manager state mutex poisoned"); - if let Some(index) = state - .transactions - .iter() - .position(|(candidate, _)| *candidate == token) - { - state.transactions.swap_remove(index); - } + .find(|transaction| transaction.process.id() == process_id) + .cloned() } - fn remove_transaction_by_process_id(&self, process_id: ProcessId) { + fn remove_transaction(&self, process_id: ProcessId) { let mut state = self .state .lock() @@ -954,7 +425,7 @@ impl RunnerProcessManager { if let Some(index) = state .transactions .iter() - .position(|(_, start)| start.process.id() == process_id) + .position(|transaction| transaction.process.id() == process_id) { state.transactions.swap_remove(index); } @@ -962,14 +433,13 @@ impl RunnerProcessManager { fn runner_finished( &self, - token: ProcessStartToken, - start: &ProcessStartTransaction, + transaction: &ProcessStartTransaction, result: RunnerCompletion, thread_panicked: bool, ) { let unexpected_runner_failure = result.runner_success == Some(false) && result.runner_signal.is_none() - && !start.commit_was_claimed() + && !transaction.start_completed() && !result.termination_provenance.reported_start_failure() && !runner_exit_code_is_expected_shutdown( result.runner_exit_code, @@ -986,24 +456,17 @@ impl RunnerProcessManager { || runner_exit_code_is_crash(result.runner_exit_code) || unexpected_runner_failure; if result.result.is_err() || result.runner_success != Some(true) { - start.abort(ErrorCode::PeerClosed, abnormal, false); - } else { - // Any clean host exit before commit still aborts creation. - start.abort(ErrorCode::PeerClosed, false, false); - } - if !start.retains_published_receipt() { - self.remove_transaction(token); - if let ReceiptResolution::Resolved(finalization) = start.resolve_receipt() { - debug_assert!(finalization.is_none()); - } + transaction.abort(ErrorCode::PeerClosed, abnormal, false); + } else if !transaction.start_completed() { + transaction.abort(ErrorCode::PeerClosed, false, false); } - if let Some(abnormal) = start.runner_finished(abnormal) { - self.finish_transaction(start, abnormal); + if let Some(abnormal) = transaction.runner_finished(abnormal) { + self.finish_transaction(transaction, abnormal); } } - fn finish_transaction(&self, start: &ProcessStartTransaction, abnormal: bool) { - start.process.cleanup(!abnormal); + fn finish_transaction(&self, transaction: &ProcessStartTransaction, abnormal: bool) { + transaction.process.cleanup(!abnormal); self.finish_instance(); } @@ -1016,48 +479,24 @@ impl RunnerProcessManager { .active_instances .checked_sub(1) .expect("runner instance count must remain balanced"); - if state.active_instances == 0 && state.active_watchdogs == 0 { + if state.active_instances == 0 { self.drained.notify_all(); } } - fn finish_watchdog(&self) { + pub(super) fn wait_for_drain(&self) { let mut state = self .state .lock() .expect("runner process manager state mutex poisoned"); - state.active_watchdogs = state - .active_watchdogs - .checked_sub(1) - .expect("runner watchdog count must remain balanced"); - if state.active_instances == 0 && state.active_watchdogs == 0 { - self.drained.notify_all(); + while state.active_instances != 0 { + state = self + .drained + .wait(state) + .expect("runner process manager state mutex poisoned"); } } - - pub(super) fn wait_for_drain(&self) { - let mut state = self - .state - .lock() - .expect("runner process manager state mutex poisoned"); - while state.active_instances != 0 || state.active_watchdogs != 0 { - state = self - .drained - .wait(state) - .expect("runner process manager state mutex poisoned"); - } - } -} - -struct WatchdogCompletion { - process_manager: Arc, -} - -impl Drop for WatchdogCompletion { - fn drop(&mut self) { - self.process_manager.finish_watchdog(); - } -} +} const fn process_extension_error(error: ErrorCode) -> RequestFailure { match error { @@ -1086,7 +525,7 @@ const fn process_start_failure_is_expected(error: ErrorCode) -> bool { } impl ProcessStartTransaction { - fn wait_until_ready(&self) -> Result, ErrorCode> { + fn wait_until_ready(&self, deadline: Instant) -> Result, ErrorCode> { let mut state = self .state .lock() @@ -1094,80 +533,67 @@ impl ProcessStartTransaction { loop { match state.phase { ProcessStartPhase::Starting => { - state = self + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + drop(state); + self.abort(ErrorCode::Internal, true, true); + return Err(ErrorCode::Internal); + } + let (next, wait_result) = self .changed - .wait(state) + .wait_timeout(state, remaining) .expect("process start transaction mutex poisoned"); + state = next; + if wait_result.timed_out() && matches!(state.phase, ProcessStartPhase::Starting) + { + drop(state); + self.abort(ErrorCode::Internal, true, true); + return Err(ErrorCode::Internal); + } } ProcessStartPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), - ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete => { + ProcessStartPhase::Completing { .. } | ProcessStartPhase::Running => { return Err(ErrorCode::ProtocolState); } - ProcessStartPhase::Aborted(error) => return Err(error), + ProcessStartPhase::Failed(error) => return Err(error), } } } fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { - self.process - .mark_start_ready(initial_thread_id) - .map_err(|error| match error { - BrokerError::UnknownObject => ErrorCode::ProtocolState, - error => ErrorCode::from(error), - })?; let mut state = self .state .lock() .expect("process start transaction mutex poisoned"); if !matches!(state.phase, ProcessStartPhase::Starting) { return Err(match state.phase { - ProcessStartPhase::Aborted(error) => error, + ProcessStartPhase::Failed(error) => error, _ => ErrorCode::ProtocolState, }); } + self.process + .mark_start_ready(initial_thread_id) + .map_err(|error| match error { + BrokerError::UnknownObject => ErrorCode::ProtocolState, + error => ErrorCode::from(error), + })?; state.phase = ProcessStartPhase::Ready { initial_thread_id }; self.changed.notify_all(); loop { match state.phase { - ProcessStartPhase::StartComplete if state.active_control_callbacks == 0 => { - return Ok(()); - } - ProcessStartPhase::Ready { .. } - | ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete => { + ProcessStartPhase::Ready { .. } | ProcessStartPhase::Completing { .. } => { state = self .changed .wait(state) .expect("process start transaction mutex poisoned"); } - ProcessStartPhase::Aborted(error) => return Err(error), + ProcessStartPhase::Running => return Ok(()), + ProcessStartPhase::Failed(error) => return Err(error), ProcessStartPhase::Starting => unreachable!("ready state cannot regress"), } } } - fn begin_start_result_publication(&self) -> Result { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - match state.phase { - ProcessStartPhase::Ready { .. } - if state.publication == StartResultPublication::NotStarted => - { - state.publication = StartResultPublication::Publishing; - Ok(Instant::now() + PROCESS_START_RECEIPT_TIMEOUT) - } - ProcessStartPhase::Aborted(error) => Err(error), - _ => Err(ErrorCode::ProtocolState), - } - } - fn report_start_failure(&self, error: ErrorCode) -> Result<(), ErrorCode> { if !process_start_failure_is_expected(error) { return Err(ErrorCode::ProtocolState); @@ -1178,179 +604,75 @@ impl ProcessStartTransaction { .expect("process start transaction mutex poisoned"); match state.phase { ProcessStartPhase::Starting => {} - ProcessStartPhase::Aborted(error) => return Err(error), + ProcessStartPhase::Failed(error) => return Err(error), _ => return Err(ErrorCode::ProtocolState), } - state.phase = ProcessStartPhase::Aborted(error); - state.shutdown_request = ShutdownRequest::ExpectedStartFailurePending; - arm_deadline( - &mut state.start_failure_publication_watchdog, - PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, - ); + state.phase = ProcessStartPhase::Failed(error); + state.shutdown_request = ShutdownRequest::ExpectedStartFailure; self.changed.notify_all(); Ok(()) } - fn start_failure_response_sent(&self, error: ErrorCode) { - let (association_failure, shutdown) = { + fn complete_start(&self) -> Result<(), ErrorCode> { + { let mut state = self .state .lock() .expect("process start transaction mutex poisoned"); - complete_deadline(&mut state.start_failure_publication_watchdog); - let actions = if matches!(state.phase, ProcessStartPhase::Aborted(cause) if cause == error) - && state.shutdown_request == ShutdownRequest::ExpectedStartFailurePending - { - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } else { - (None, None) - }; - self.changed.notify_all(); - actions - }; - if let Some(association_failure) = association_failure { - association_failure(); - } - if let Some(shutdown) = shutdown { - shutdown.shutdown(); - } - } - - fn admit_acknowledgement(&self) -> Result<(), ErrorCode> { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - if state.publication == StartResultPublication::NotStarted { - return Err(ErrorCode::ProtocolState); - } - match state.receipt { - ReceiptState::AwaitingAcknowledgement => {} - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { - return Err(ErrorCode::PeerClosed); + match state.phase { + ProcessStartPhase::Ready { .. } => { + state.phase = ProcessStartPhase::Completing { cancellation: None }; + } + ProcessStartPhase::Failed(error) => return Err(error), + _ => return Err(ErrorCode::ProtocolState), } - ReceiptState::AcknowledgementAdmitted => return Err(ErrorCode::UnknownObject), - } - state.receipt = ReceiptState::AcknowledgementAdmitted; - if state.publication == StartResultPublication::Delivered - && matches!( - state.phase, - ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) - ) - { - arm_deadline( - &mut state.resolution_watchdog, - PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, - ); - self.changed.notify_all(); } - Ok(()) - } - fn resolve_acknowledgement(&self) -> Result { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - loop { - match state.receipt { - ReceiptState::AcknowledgementAdmitted => {} - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved => { - return Err(ErrorCode::PeerClosed); - } - ReceiptState::AwaitingAcknowledgement => return Err(ErrorCode::ProtocolState), - } - if state.publication == StartResultPublication::NotStarted { - return Err(ErrorCode::ProtocolState); - } - match (state.phase, state.publication) { - ( - ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_), - StartResultPublication::Publishing, - ) => { - state = self - .changed - .wait(state) - .expect("process start transaction mutex poisoned"); - } - (ProcessStartPhase::Ready { .. }, StartResultPublication::Delivered) => { - debug_assert!(matches!(state.resolution_watchdog, DeadlineState::Armed(_))); - state.phase = ProcessStartPhase::Committing; - drop(state); - let commit_result = self.process.commit_start().map_err(ErrorCode::from); - state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - match commit_result { - Ok(()) => { - state.phase = ProcessStartPhase::Committed; - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Acknowledged); - } - Err(error) => { - state.phase = ProcessStartPhase::Aborted(error); - state.abnormal |= error == ErrorCode::Internal; - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Failed(error)); - } + let completion = self.process.complete_start().map_err(ErrorCode::from); + let (result, association_failure, shutdown) = { + let mut state = self + .state + .lock() + .expect("process start transaction mutex poisoned"); + let ProcessStartPhase::Completing { cancellation } = state.phase else { + unreachable!("process completion phase cannot change independently"); + }; + match completion { + Ok(()) => { + state.start_completed = true; + if let Some(error) = cancellation { + state.phase = ProcessStartPhase::Failed(error); + (Err(error), None, None) + } else { + state.phase = ProcessStartPhase::Running; + (Ok(()), None, None) } } - (ProcessStartPhase::Aborted(error), StartResultPublication::Delivered) => { - debug_assert!(matches!( - state.resolution_watchdog, - DeadlineState::Armed(_) | DeadlineState::Fired - )); - complete_acknowledgement_resolution(&mut state); - self.changed.notify_all(); - return Ok(ProcessStartAcknowledgement::Failed(error)); - } - (ProcessStartPhase::Starting, _) => return Err(ErrorCode::ProtocolState), - ( - ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete, - _, - ) => { - return Err(ErrorCode::ProtocolState); - } - (_, StartResultPublication::NotStarted) => { - unreachable!("publication state was checked before phase dispatch") + Err(error) => { + state.phase = ProcessStartPhase::Failed(error); + state.abnormal |= error == ErrorCode::Internal; + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; + } + ( + Err(error), + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) } } + }; + self.changed.notify_all(); + if let Some(association_failure) = association_failure { + association_failure(); } - } - - fn mark_start_result_delivered(&self) { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - if state.publication == StartResultPublication::Publishing { - state.publication = StartResultPublication::Delivered; - if state.receipt == ReceiptState::AcknowledgementAdmitted - && matches!( - state.phase, - ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) - ) - { - arm_deadline( - &mut state.resolution_watchdog, - PROCESS_START_INTERNAL_RESOLUTION_TIMEOUT, - ); - } - self.changed.notify_all(); + if let Some(shutdown) = shutdown { + shutdown.shutdown(); } + result } - fn install_shutdown(&self, shutdown: Arc) { + pub(super) fn install_shutdown(&self, shutdown: Arc) { let shutdown = { let mut state = self .state @@ -1385,52 +707,36 @@ impl ProcessStartTransaction { .lock() .expect("process start transaction mutex poisoned"); state.abnormal |= abnormal; - match state.phase { + let should_terminate = match &mut state.phase { ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { - state.phase = ProcessStartPhase::Aborted(error); - let newly_requested = state.shutdown_request == ShutdownRequest::None; - if newly_requested { - state.shutdown_request = if expected_shutdown { - ShutdownRequest::Expected - } else { - ShutdownRequest::Unexpected - }; - } - self.changed.notify_all(); - ( - newly_requested - .then(|| state.association_failure.as_ref().map(Arc::clone)) - .flatten(), - state.shutdown.clone(), - ) + state.phase = ProcessStartPhase::Failed(error); + true } - ProcessStartPhase::Aborted(_) => match state.shutdown_request { - ShutdownRequest::None => { - state.shutdown_request = if expected_shutdown { - ShutdownRequest::Expected - } else { - ShutdownRequest::Unexpected - }; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } - ShutdownRequest::ExpectedStartFailurePending => { - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) + ProcessStartPhase::Completing { cancellation } => { + if cancellation.is_none() { + *cancellation = Some(error); + true + } else { + false } + } + ProcessStartPhase::Failed(_) | ProcessStartPhase::Running => false, + }; + if should_terminate && state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = if expected_shutdown { ShutdownRequest::Expected - | ShutdownRequest::ExpectedStartFailure - | ShutdownRequest::Unexpected => (None, None), - }, - ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete => (None, None), + } else { + ShutdownRequest::Unexpected + }; + } + self.changed.notify_all(); + if should_terminate { + ( + state.association_failure.as_ref().map(Arc::clone), + state.shutdown.clone(), + ) + } else { + (None, None) } }; if let Some(association_failure) = association_failure { @@ -1447,16 +753,21 @@ impl ProcessStartTransaction { .lock() .expect("process start transaction mutex poisoned"); state.association_failure = None; - if matches!( - state.phase, - ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } - ) { - state.phase = ProcessStartPhase::Aborted(ErrorCode::PeerClosed); - self.changed.notify_all(); + match &mut state.phase { + ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { + state.phase = ProcessStartPhase::Failed(ErrorCode::PeerClosed); + } + ProcessStartPhase::Completing { cancellation } => { + if cancellation.is_none() { + *cancellation = Some(ErrorCode::PeerClosed); + } + } + ProcessStartPhase::Failed(_) | ProcessStartPhase::Running => {} } + self.changed.notify_all(); } - fn mark_shutdown_expected(&self) { + pub(super) fn mark_shutdown_expected(&self) { let mut state = self .state .lock() @@ -1466,1107 +777,237 @@ impl ProcessStartTransaction { } } - fn mark_abnormal(&self) { + pub(super) fn mark_abnormal(&self) { self.state .lock() .expect("process start transaction mutex poisoned") .abnormal = true; } - fn shutdown_request(&self) -> ShutdownRequest { + pub(super) fn shutdown_request(&self) -> ShutdownRequest { self.state .lock() .expect("process start transaction mutex poisoned") .shutdown_request } - fn commit_was_claimed(&self) -> bool { - matches!( - self.state - .lock() - .expect("process start transaction mutex poisoned") - .phase, - ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete - ) - } - - fn retains_published_receipt(&self) -> bool { - let state = self - .state + fn start_completed(&self) -> bool { + self.state .lock() - .expect("process start transaction mutex poisoned"); - state.publication != StartResultPublication::NotStarted - && state.receipt != ReceiptState::Resolved + .expect("process start transaction mutex poisoned") + .start_completed } - fn resolve_receipt(&self) -> ReceiptResolution { + fn protocol_finished(&self) -> Option { let mut state = self .state .lock() .expect("process start transaction mutex poisoned"); - if matches!( - state.receipt, - ReceiptState::TimeoutPending | ReceiptState::Draining - ) { - return ReceiptResolution::DeferredToDrain; + match state.finalization { + FinalizationState::Active => { + state.finalization = FinalizationState::ProtocolFinished; + None + } + FinalizationState::RunnerFinished => { + state.finalization = FinalizationState::Finalized; + Some(state.abnormal) + } + FinalizationState::ProtocolFinished | FinalizationState::Finalized => None, } - state.receipt = ReceiptState::Resolved; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); - ReceiptResolution::Resolved(self.complete_and_take_finalization(state)) } - fn begin_receipt_drain(&self) { + fn runner_finished(&self, abnormal: bool) -> Option { let mut state = self .state .lock() .expect("process start transaction mutex poisoned"); - if state.receipt != ReceiptState::Resolved { - state.receipt = ReceiptState::Draining; - if !matches!(state.phase, ProcessStartPhase::Committing) { - complete_deadline(&mut state.resolution_watchdog); + state.abnormal |= abnormal; + match state.finalization { + FinalizationState::Active => { + state.finalization = FinalizationState::RunnerFinished; + None } - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); + FinalizationState::ProtocolFinished => { + state.finalization = FinalizationState::Finalized; + Some(state.abnormal) + } + FinalizationState::RunnerFinished | FinalizationState::Finalized => None, } } +} - fn finish_receipt_drain(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - state.receipt = ReceiptState::Resolved; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - self.changed.notify_all(); - self.complete_and_take_finalization(state) - } +#[cfg(test)] +mod tests { + use super::{ + FinalizationState, PROCESS_START_TIMEOUT, ProcessStartPhase, ProcessStartTransaction, + ProcessStartTransactionState, RunnerConfig, RunnerProcessManager, + RunnerProcessManagerState, ShutdownRequest, + }; + use litebox_broker_core::test_support::TestBrokerCoreBuilder; + use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; + use litebox_broker_protocol::error::ErrorCode; + use std::path::PathBuf; + use std::sync::{Arc, Condvar, Mutex, mpsc}; + use std::time::{Duration, Instant}; - fn expire_initial_receipt_deadline(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - let (error, abnormal) = match (state.receipt, state.publication) { - (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), - (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { - (ErrorCode::Internal, true) - } - _ => return None, - }; - expire_transaction(&mut state, error, abnormal, true, &self.changed) + fn starting_transaction() -> (BrokerCore, Arc) { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let parent = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let parent_id = parent.id(); + let (process, _) = parent.create_child(&[]).unwrap(); + parent.cleanup(true); + ( + broker, + Arc::new(ProcessStartTransaction { + parent_id, + process, + state: Mutex::new(ProcessStartTransactionState { + phase: ProcessStartPhase::Starting, + shutdown: None, + association_failure: None, + shutdown_request: ShutdownRequest::None, + abnormal: false, + start_completed: false, + finalization: FinalizationState::Active, + }), + changed: Condvar::new(), + }), + ) } - fn wait_for_initial_receipt_deadline(&self, deadline: Instant) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - loop { - if !matches!( - (state.receipt, state.publication), - (ReceiptState::AwaitingAcknowledgement, _) - | ( - ReceiptState::AcknowledgementAdmitted, - StartResultPublication::Publishing - ) - ) { - return None; - } - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - break; - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("process start transaction mutex poisoned"); - state = next; - } - let (error, abnormal) = match (state.receipt, state.publication) { - (ReceiptState::AwaitingAcknowledgement, _) => (ErrorCode::PeerClosed, false), - (ReceiptState::AcknowledgementAdmitted, StartResultPublication::Publishing) => { - (ErrorCode::Internal, true) - } - _ => return None, - }; - expire_transaction(&mut state, error, abnormal, true, &self.changed) - } + #[test] + fn ready_report_waits_for_broker_start_completion() { + let (_broker, transaction) = starting_transaction(); + let ready = Arc::clone(&transaction); + let waiter = std::thread::spawn(move || ready.ready_and_wait(None)); - fn wait_for_internal_resolution_timeout(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - loop { - match state.resolution_watchdog { - DeadlineState::Unarmed => { - state = self - .changed - .wait(state) - .expect("process start transaction mutex poisoned"); - } - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.resolution_watchdog = DeadlineState::Fired; - return expire_internal_resolution(&mut state, &self.changed); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("process start transaction mutex poisoned"); - state = next; - } - DeadlineState::Disarmed | DeadlineState::Fired => return None, - } - } - } - - fn wait_for_acknowledgement_publication_timeout(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - loop { - match state.acknowledgement_publication_watchdog { - DeadlineState::Unarmed => { - state = self - .changed - .wait(state) - .expect("process start transaction mutex poisoned"); - } - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.acknowledgement_publication_watchdog = DeadlineState::Fired; - return expire_transaction( - &mut state, - ErrorCode::PeerClosed, - false, - true, - &self.changed, - ); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("process start transaction mutex poisoned"); - state = next; - } - DeadlineState::Disarmed | DeadlineState::Fired => return None, - } - } - } - - fn wait_for_start_failure_publication_timeout(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - loop { - match state.start_failure_publication_watchdog { - DeadlineState::Armed(deadline) => { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - state.start_failure_publication_watchdog = DeadlineState::Fired; - return expire_start_failure_publication(&mut state, &self.changed); - } - let (next, _) = self - .changed - .wait_timeout(state, remaining) - .expect("process start transaction mutex poisoned"); - state = next; - } - DeadlineState::Unarmed | DeadlineState::Disarmed | DeadlineState::Fired => { - return None; - } - } - } - } - - fn fail_start_failure_publication_watchdog(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - if !matches!( - state.start_failure_publication_watchdog, - DeadlineState::Armed(_) - ) { - return None; - } - state.start_failure_publication_watchdog = DeadlineState::Fired; - expire_start_failure_publication(&mut state, &self.changed) - } - - fn fail_internal_resolution_watchdog(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - expire_transaction(&mut state, ErrorCode::Internal, true, true, &self.changed) - } - - fn complete_timeout_callback(&self) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - state.active_control_callbacks = state - .active_control_callbacks - .checked_sub(1) - .expect("process-start timeout callback count must remain balanced"); - self.changed.notify_all(); - take_finalization(&mut state) - } - - fn wait_for_commit_resolution(&self, deadline: Instant) -> bool { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - while matches!(state.phase, ProcessStartPhase::Committing) { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return false; - } - let (next, wait_result) = self - .changed - .wait_timeout(state, remaining) - .expect("process start transaction mutex poisoned"); - state = next; - if wait_result.timed_out() && matches!(state.phase, ProcessStartPhase::Committing) { - return false; - } - } - true - } - - fn complete_and_take_finalization( - &self, - mut state: MutexGuard<'_, ProcessStartTransactionState>, - ) -> Option { - if !matches!(state.phase, ProcessStartPhase::Committed) { - return take_finalization(&mut state); - } - state.phase = ProcessStartPhase::CompletingStart; - state.active_control_callbacks = state - .active_control_callbacks - .checked_add(1) - .expect("process-start control callback count must remain bounded"); - drop(state); - - let completion_failed = self.process.complete_start().is_err(); - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - state.active_control_callbacks = state - .active_control_callbacks - .checked_sub(1) - .expect("process-start control callback count must remain balanced"); - state.abnormal |= completion_failed; - state.phase = ProcessStartPhase::StartComplete; - self.changed.notify_all(); - take_finalization(&mut state) - } - - fn runner_finished(&self, abnormal: bool) -> Option { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - state.abnormal |= abnormal; - state.runner_finished = true; - take_finalization(&mut state) - } -} - -fn expire_transaction( - state: &mut ProcessStartTransactionState, - error: ErrorCode, - abnormal: bool, - fail_parent: bool, - changed: &Condvar, -) -> Option { - if matches!( - state.receipt, - ReceiptState::TimeoutPending | ReceiptState::Draining | ReceiptState::Resolved - ) { - return None; - } - state.active_control_callbacks += 1; - state.receipt = ReceiptState::TimeoutPending; - state.abnormal |= abnormal; - complete_deadline(&mut state.resolution_watchdog); - complete_deadline(&mut state.acknowledgement_publication_watchdog); - complete_deadline(&mut state.start_failure_publication_watchdog); - let (association_failure, shutdown) = match state.phase { - ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { - state.phase = ProcessStartPhase::Aborted(error); - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } - ProcessStartPhase::Aborted(_) => { - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) - } else { - (None, None) - } - } - ProcessStartPhase::Committing - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete => (None, None), - }; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown, - association_failure, - fail_parent, - commit_supervision_deadline: None, - }) -} - -fn expire_start_failure_publication( - state: &mut ProcessStartTransactionState, - changed: &Condvar, -) -> Option { - if !matches!(state.phase, ProcessStartPhase::Aborted(_)) - || state.shutdown_request != ShutdownRequest::ExpectedStartFailurePending - { - return None; - } - state.active_control_callbacks += 1; - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown: state.shutdown.clone(), - association_failure: state.association_failure.as_ref().map(Arc::clone), - fail_parent: false, - commit_supervision_deadline: None, - }) -} - -fn expire_internal_resolution( - state: &mut ProcessStartTransactionState, - changed: &Condvar, -) -> Option { - let committing_drain = state.receipt == ReceiptState::Draining - && matches!(state.phase, ProcessStartPhase::Committing); - if state.receipt != ReceiptState::AcknowledgementAdmitted && !committing_drain { - return None; - } - state.active_control_callbacks += 1; - state.abnormal = true; - let (association_failure, shutdown, fail_parent, commit_supervision_deadline) = - match state.phase { - ProcessStartPhase::Ready { .. } | ProcessStartPhase::Aborted(_) => { - state.phase = ProcessStartPhase::Aborted(ErrorCode::Internal); - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - ( - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - false, - None, - ) - } - ProcessStartPhase::Committing => { - state.receipt = ReceiptState::TimeoutPending; - complete_deadline(&mut state.acknowledgement_publication_watchdog); - ( - None, - None, - true, - Some(Instant::now() + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT), - ) - } - ProcessStartPhase::Starting - | ProcessStartPhase::Committed - | ProcessStartPhase::CompletingStart - | ProcessStartPhase::StartComplete => { - state.active_control_callbacks -= 1; - return None; - } - }; - changed.notify_all(); - Some(ReceiptExpiration { - shutdown, - association_failure, - fail_parent, - commit_supervision_deadline, - }) -} - -fn arm_deadline(state: &mut DeadlineState, timeout: Duration) { - if *state == DeadlineState::Unarmed { - *state = DeadlineState::Armed(Instant::now() + timeout); - } -} - -fn complete_deadline(state: &mut DeadlineState) { - if matches!(state, DeadlineState::Unarmed | DeadlineState::Armed(_)) { - *state = DeadlineState::Disarmed; - } -} - -fn complete_acknowledgement_resolution(state: &mut ProcessStartTransactionState) { - complete_deadline(&mut state.resolution_watchdog); - if state.receipt == ReceiptState::AcknowledgementAdmitted { - arm_deadline( - &mut state.acknowledgement_publication_watchdog, - PROCESS_START_ACKNOWLEDGEMENT_PUBLICATION_TIMEOUT, - ); - } -} - -fn take_finalization(state: &mut ProcessStartTransactionState) -> Option { - if state.finalization_taken - || state.active_control_callbacks != 0 - || state.receipt != ReceiptState::Resolved - { - return None; - } - if !state.runner_finished { - return None; - } - state.runner_finished = false; - state.finalization_taken = true; - Some(state.abnormal) -} - -#[cfg(test)] -mod tests { - use super::{ - DeadlineState, PROCESS_START_RECEIPT_TIMEOUT, PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT, - ProcessStartAcknowledgement, ProcessStartPhase, ProcessStartTransaction, - ProcessStartTransactionState, ReceiptResolution, ReceiptState, RunnerConfig, - RunnerProcessManager, RunnerProcessManagerState, ShutdownRequest, StartResultPublication, - }; - use litebox_broker_core::test_support::TestBrokerCoreBuilder; - use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; - use litebox_broker_protocol::ProcessId; - use litebox_broker_protocol::error::ErrorCode; - use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; - use litebox_broker_protocol::process::ProcessStartToken; - use std::path::PathBuf; - use std::sync::{ - Arc, Condvar, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc, - }; - use std::time::{Duration, Instant}; - - fn transaction_with_broker_in_phase( - publication: StartResultPublication, - phase: ProcessStartPhase, - ) -> (BrokerCore, Arc) { - let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( - ObjectRights::all(), - )) - .build() - .unwrap(); - let parent = broker - .create_process(CallerCredential::Unauthenticated) - .unwrap(); - let (process, _) = parent.create_child(&[]).unwrap(); - parent.cleanup(true); - if let ProcessStartPhase::Ready { initial_thread_id } = phase { - process.mark_start_ready(initial_thread_id).unwrap(); - } - ( - broker, - Arc::new(ProcessStartTransaction { - parent_id: ProcessId(1), - process, - state: Mutex::new(ProcessStartTransactionState { - phase, - publication, - shutdown: None, - association_failure: None, - shutdown_request: ShutdownRequest::None, - abnormal: false, - receipt: ReceiptState::AwaitingAcknowledgement, - resolution_watchdog: DeadlineState::Unarmed, - acknowledgement_publication_watchdog: DeadlineState::Unarmed, - start_failure_publication_watchdog: DeadlineState::Unarmed, - active_control_callbacks: 0, - runner_finished: false, - finalization_taken: false, - }), - changed: Condvar::new(), - }), - ) - } - - fn transaction_with_broker( - publication: StartResultPublication, - ) -> (BrokerCore, Arc) { - transaction_with_broker_in_phase( - publication, - ProcessStartPhase::Ready { - initial_thread_id: None, - }, - ) - } - - fn transaction(publication: StartResultPublication) -> Arc { - transaction_with_broker(publication).1 - } - - fn starting_transaction(publication: StartResultPublication) -> Arc { - transaction_with_broker_in_phase(publication, ProcessStartPhase::Starting).1 - } - - #[test] - fn publication_captures_an_absolute_receipt_deadline() { - let start = transaction(StartResultPublication::NotStarted); - let before = Instant::now(); - - let deadline = start.begin_start_result_publication().unwrap(); - let after = Instant::now(); - - assert!(deadline >= before + PROCESS_START_RECEIPT_TIMEOUT); - assert!(deadline <= after + PROCESS_START_RECEIPT_TIMEOUT); - } - - #[test] - fn acknowledgement_ingress_claims_receipt_before_worker_resolution() { - let (broker, start) = transaction_with_broker(StartResultPublication::Delivered); - let token = ProcessStartToken(7); - let parent_id = start.parent_id; - let process_manager = Arc::new(RunnerProcessManager { - broker, - started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), - state: Mutex::new(RunnerProcessManagerState { - transactions: vec![(token, Arc::clone(&start))], - associations: Vec::new(), - active_instances: 1, - active_watchdogs: 0, - }), - drained: Condvar::new(), - }); - - process_manager - .admit_acknowledgement(parent_id, token) - .unwrap(); - - assert_eq!(process_manager.state.lock().unwrap().active_watchdogs, 2); - assert!(matches!( - start.state.lock().unwrap().receipt, - ReceiptState::AcknowledgementAdmitted - )); - assert!(start.expire_initial_receipt_deadline().is_none()); - assert!(matches!( - process_manager.resolve_acknowledgement(parent_id, token), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - process_manager.response_sent( - parent_id, - &BrokerOperation::AcknowledgeProcessStart(token), - &BrokerResult::ProcessStartAcknowledged, - ); - start.process.cleanup(true); - process_manager.finish_instance(); - process_manager.wait_for_drain(); - assert_eq!(process_manager.state.lock().unwrap().active_watchdogs, 0); + assert_eq!( + transaction + .wait_until_ready(Instant::now() + Duration::from_secs(1)) + .unwrap(), + None + ); + transaction.complete_start().unwrap(); + + waiter.join().unwrap().unwrap(); + assert!(transaction.process.is_running()); + transaction.process.cleanup(true); } #[test] - fn reported_bootstrap_rejection_selects_normal_rollback() { - let start = starting_transaction(StartResultPublication::NotStarted); + fn reported_start_failure_wakes_the_parent_request() { + let (_broker, transaction) = starting_transaction(); - start + transaction .report_start_failure(ErrorCode::UnsupportedOperation) .unwrap(); - assert!(matches!( - start.wait_until_ready(), + assert_eq!( + transaction.wait_until_ready(Instant::now() + Duration::from_secs(1)), Err(ErrorCode::UnsupportedOperation) - )); - assert!( - start - .shutdown_request() - .expected_start_failure_was_reported() ); - assert!(!start.state.lock().unwrap().abnormal); - start.start_failure_response_sent(ErrorCode::UnsupportedOperation); - assert!(start.shutdown_request().was_expected()); - assert_eq!(start.runner_finished(false), None); - assert!(matches!( - start.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - start.process.cleanup(true); + assert!(transaction.shutdown_request().was_expected()); + transaction.process.cleanup(true); } #[test] - fn acknowledgement_waits_for_publication_bookkeeping() { - let start = transaction(StartResultPublication::Publishing); - start.admit_acknowledgement().unwrap(); - let waiting = Arc::clone(&start); - let (sender, receiver) = mpsc::sync_channel(1); - let worker = - std::thread::spawn(move || sender.send(waiting.resolve_acknowledgement()).unwrap()); - - assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); - assert!(matches!( - start.state.lock().unwrap().resolution_watchdog, - DeadlineState::Unarmed - )); - start.mark_start_result_delivered(); - assert!(matches!( - receiver.recv_timeout(Duration::from_secs(1)).unwrap(), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - worker.join().unwrap(); - assert!(start.process.is_running()); - assert!(matches!( - start.state.lock().unwrap().resolution_watchdog, - DeadlineState::Disarmed - )); - assert!(matches!( - start - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog, - DeadlineState::Armed(_) - )); + fn startup_timeout_aborts_the_transaction() { + let (_broker, transaction) = starting_transaction(); + let deadline = Instant::now() + Duration::from_millis(1); - start.resolve_receipt(); - start.process.cleanup(true); - } - - #[test] - fn acknowledgement_interrupted_by_drain_returns_peer_closed() { - let start = transaction(StartResultPublication::Publishing); - start.admit_acknowledgement().unwrap(); - let waiting = Arc::clone(&start); - let worker = std::thread::spawn(move || waiting.resolve_acknowledgement()); - - std::thread::sleep(Duration::from_millis(20)); - start.begin_receipt_drain(); - - assert!(matches!(worker.join().unwrap(), Err(ErrorCode::PeerClosed))); - start.finish_receipt_drain(); - start.process.cleanup(true); - } - - #[test] - fn process_ready_interrupted_by_abort_returns_the_abort_cause() { - let ready = starting_transaction(StartResultPublication::NotStarted); - ready.abort(ErrorCode::PeerClosed, false, true); - assert!(matches!( - ready.ready_and_wait(None), - Err(ErrorCode::PeerClosed) - )); - ready.resolve_receipt(); - ready.process.cleanup(true); - } - - #[test] - fn failure_report_interrupted_by_abort_returns_the_abort_cause() { - let failed = starting_transaction(StartResultPublication::NotStarted); - failed.abort(ErrorCode::PeerClosed, false, true); - assert!(matches!( - failed.report_start_failure(ErrorCode::UnsupportedOperation), - Err(ErrorCode::PeerClosed) - )); - failed.resolve_receipt(); - failed.process.cleanup(true); - } - - #[test] - fn delivery_arms_the_resolution_watchdog_before_the_worker_resumes() { - let start = transaction(StartResultPublication::Publishing); - start.admit_acknowledgement().unwrap(); - - assert!(matches!( - start.state.lock().unwrap().resolution_watchdog, - DeadlineState::Unarmed - )); - start.mark_start_result_delivered(); - assert!(matches!( - start.state.lock().unwrap().resolution_watchdog, - DeadlineState::Armed(_) - )); - - start.begin_receipt_drain(); - start.finish_receipt_drain(); - start.process.cleanup(true); - } - - #[test] - fn acknowledgement_publication_timeout_retains_receipt_for_drain() { - let start = transaction(StartResultPublication::Delivered); - start.admit_acknowledgement().unwrap(); - assert!(matches!( - start.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Acknowledged) - )); - start - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = start - .wait_for_acknowledgement_publication_timeout() - .unwrap(); - assert!(expiration.fail_parent); - let state = start.state.lock().unwrap(); - assert!(matches!(state.receipt, ReceiptState::TimeoutPending)); + assert_eq!( + transaction.wait_until_ready(deadline), + Err(ErrorCode::Internal) + ); assert!(matches!( - state.acknowledgement_publication_watchdog, - DeadlineState::Fired + transaction.state.lock().unwrap().phase, + ProcessStartPhase::Failed(ErrorCode::Internal) )); - assert!(!state.abnormal); - drop(state); - - assert_eq!(start.complete_timeout_callback(), None); - start.begin_receipt_drain(); - start.finish_receipt_drain(); - start.process.cleanup(true); + transaction.process.cleanup(true); } #[test] - fn start_failure_publication_timeout_terminates_the_runner() { - let start = starting_transaction(StartResultPublication::NotStarted); - start - .report_start_failure(ErrorCode::UnsupportedOperation) + fn parent_abort_releases_a_ready_child() { + let (_broker, transaction) = starting_transaction(); + let ready = Arc::clone(&transaction); + let waiter = std::thread::spawn(move || ready.ready_and_wait(None)); + transaction + .wait_until_ready(Instant::now() + Duration::from_secs(1)) .unwrap(); - start - .state - .lock() - .unwrap() - .start_failure_publication_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = start.wait_for_start_failure_publication_timeout().unwrap(); - - assert!(!expiration.fail_parent); - assert!(start.shutdown_request().was_expected()); - assert!(matches!( - start - .state - .lock() - .unwrap() - .start_failure_publication_watchdog, - DeadlineState::Fired - )); - assert_eq!(start.complete_timeout_callback(), None); - assert_eq!(start.runner_finished(false), None); - assert!(matches!( - start.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - start.process.cleanup(true); - } - - #[test] - fn precommit_resolution_timeout_returns_typed_internal_failure() { - let start = transaction(StartResultPublication::Delivered); - start.admit_acknowledgement().unwrap(); - start.state.lock().unwrap().resolution_watchdog = DeadlineState::Armed(Instant::now()); - - let expiration = start.wait_for_internal_resolution_timeout().unwrap(); - assert!(!expiration.fail_parent); - let state = start.state.lock().unwrap(); - assert!(matches!( - state.phase, - ProcessStartPhase::Aborted(ErrorCode::Internal) - )); - assert!(matches!( - state.receipt, - ReceiptState::AcknowledgementAdmitted - )); - assert!(matches!( - state.acknowledgement_publication_watchdog, - DeadlineState::Unarmed - )); - assert!(state.abnormal); - drop(state); - assert_eq!(start.complete_timeout_callback(), None); - - assert!(matches!( - start.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Failed(ErrorCode::Internal)) - )); - assert!(matches!( - start - .state - .lock() - .unwrap() - .acknowledgement_publication_watchdog, - DeadlineState::Armed(_) - )); - start.resolve_receipt(); - start.process.cleanup(false); - } - - #[test] - fn process_start_abort_fails_an_installed_active_association() { - let start = transaction(StartResultPublication::NotStarted); - let failed = Arc::new(AtomicBool::new(false)); - let recorded = Arc::clone(&failed); - start.install_association_failure(Arc::new(move || { - recorded.store(true, Ordering::Release); - })); - - start.abort(ErrorCode::PeerClosed, false, true); - - assert!(failed.load(Ordering::Acquire)); - start.process.cleanup(true); - } - - #[test] - fn supervisor_wait_observes_commit_resolution() { - let start = transaction(StartResultPublication::Delivered); - start.state.lock().unwrap().phase = ProcessStartPhase::Committing; - let waiting = Arc::clone(&start); - let worker = std::thread::spawn(move || { - waiting.wait_for_commit_resolution(Instant::now() + Duration::from_secs(1)) - }); - - std::thread::sleep(Duration::from_millis(20)); - start.state.lock().unwrap().phase = ProcessStartPhase::Committed; - start.changed.notify_all(); - - assert!(worker.join().unwrap()); - start.process.cleanup(true); - } - - #[test] - fn receipt_drain_keeps_commit_supervision_armed() { - let start = transaction(StartResultPublication::Delivered); - start.admit_acknowledgement().unwrap(); - { - let mut state = start.state.lock().unwrap(); - state.phase = ProcessStartPhase::Committing; - state.resolution_watchdog = DeadlineState::Armed(Instant::now()); - } - start.begin_receipt_drain(); - assert!(matches!( - start.state.lock().unwrap().resolution_watchdog, - DeadlineState::Armed(_) - )); - let before = Instant::now(); - let expiration = start.wait_for_internal_resolution_timeout().unwrap(); - let after = Instant::now(); - let deadline = expiration.commit_supervision_deadline.unwrap(); - assert!(deadline >= before + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); - assert!(deadline <= after + PROCESS_START_SUPERVISOR_SHUTDOWN_TIMEOUT); - { - let mut state = start.state.lock().unwrap(); - state.phase = ProcessStartPhase::Aborted(ErrorCode::Internal); - } - start.changed.notify_all(); - assert_eq!(start.complete_timeout_callback(), None); - start.finish_receipt_drain(); - start.process.cleanup(false); - } + transaction.abort(ErrorCode::PeerClosed, false, true); - #[test] - fn published_abort_returns_typed_start_failure() { - let start = transaction(StartResultPublication::Publishing); - start.abort(ErrorCode::PeerClosed, false, false); - start.mark_start_result_delivered(); - start.admit_acknowledgement().unwrap(); - - assert!(matches!( - start.resolve_acknowledgement(), - Ok(ProcessStartAcknowledgement::Failed(ErrorCode::PeerClosed)) - )); - start.resolve_receipt(); - start.process.cleanup(true); + assert_eq!(waiter.join().unwrap(), Err(ErrorCode::PeerClosed)); + transaction.process.cleanup(true); } #[test] - fn acknowledgement_before_publication_is_protocol_violation() { - let start = transaction(StartResultPublication::NotStarted); + fn finalization_waits_for_protocol_and_runner_completion() { + let (_broker, transaction) = starting_transaction(); - assert!(matches!( - start.admit_acknowledgement(), - Err(ErrorCode::ProtocolState) - )); - start.resolve_receipt(); - start.process.cleanup(true); + assert_eq!(transaction.protocol_finished(), None); + assert_eq!(transaction.runner_finished(false), Some(false)); + assert_eq!(transaction.runner_finished(false), None); + transaction.process.cleanup(true); } #[test] - fn published_receipt_defers_process_finalization() { - let start = transaction(StartResultPublication::Delivered); - start.abort(ErrorCode::PeerClosed, false, false); - - assert_eq!(start.runner_finished(false), None); - assert!(matches!( - start.resolve_receipt(), - ReceiptResolution::Resolved(Some(false)) - )); - start.process.cleanup(true); - } - - #[test] - fn receipt_resolution_completes_process_start() { - let start = starting_transaction(StartResultPublication::Delivered); - let thread_id = start.process.create_thread().unwrap(); - start.process.mark_start_ready(Some(thread_id)).unwrap(); - start.process.commit_start().unwrap(); - start.state.lock().unwrap().phase = ProcessStartPhase::Committed; - - start.resolve_receipt(); - start.resolve_receipt(); - - let state = start.state.lock().unwrap(); - assert!(matches!(state.phase, ProcessStartPhase::StartComplete)); - assert!(!state.abnormal); - drop(state); - assert_eq!(start.process.exit_thread(thread_id), Ok(())); - start.process.cleanup(true); - } - - #[test] - fn process_ready_waits_until_process_start_is_complete() { - let start = starting_transaction(StartResultPublication::Delivered); - let thread_id = start.process.create_thread().unwrap(); - let waiting = Arc::clone(&start); - let (sender, receiver) = mpsc::sync_channel(1); - let worker = std::thread::spawn(move || { - sender - .send(waiting.ready_and_wait(Some(thread_id))) - .unwrap(); - }); + fn ending_parent_association_aborts_pending_children() { + let (broker, transaction) = starting_transaction(); + let parent_id = transaction.parent_id; + let manager = RunnerProcessManager { + broker, + started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerState { + transactions: vec![Arc::clone(&transaction)], + associations: Vec::new(), + active_instances: 1, + }), + drained: Condvar::new(), + }; - let mut state = start.state.lock().unwrap(); - while !matches!(state.phase, ProcessStartPhase::Ready { .. }) { - state = start.changed.wait(state).unwrap(); - } - start.process.commit_start().unwrap(); - state.phase = ProcessStartPhase::Committed; - start.changed.notify_all(); - drop(state); - assert!(receiver.recv_timeout(Duration::from_millis(20)).is_err()); + manager.association_ending(parent_id); - start.resolve_receipt(); assert_eq!( - receiver.recv_timeout(Duration::from_secs(1)).unwrap(), - Ok(()) + transaction.wait_until_ready(Instant::now() + PROCESS_START_TIMEOUT), + Err(ErrorCode::PeerClosed) ); - worker.join().unwrap(); - assert_eq!(start.process.exit_thread(thread_id), Ok(())); - start.process.cleanup(true); + transaction.process.cleanup(true); } #[test] - fn acknowledgement_admission_preserves_the_publication_deadline() { - let start = transaction(StartResultPublication::Publishing); - start.admit_acknowledgement().unwrap(); - - let _expiration = start.expire_initial_receipt_deadline().unwrap(); - let state = start.state.lock().unwrap(); - assert!(matches!( - state.phase, - ProcessStartPhase::Aborted(ErrorCode::Internal) - )); - assert!(state.abnormal); - drop(state); - assert_eq!(start.complete_timeout_callback(), None); - start.process.cleanup(true); - } - - #[test] - fn receipt_timeout_defers_finalization_until_association_drain() { - let start = transaction(StartResultPublication::Delivered); - - assert_eq!(start.runner_finished(false), None); - let _expiration = start.expire_initial_receipt_deadline().unwrap(); - assert!(matches!( - start.resolve_receipt(), - ReceiptResolution::DeferredToDrain - )); - assert_eq!(start.complete_timeout_callback(), None); - start.begin_receipt_drain(); - assert_eq!(start.finish_receipt_drain(), Some(false)); - start.process.cleanup(true); - } - - #[test] - fn unpublished_start_waits_for_receipt_drain_before_finalization() { - let start = transaction(StartResultPublication::NotStarted); - - start.begin_receipt_drain(); - assert_eq!(start.runner_finished(false), None); - assert_eq!(start.finish_receipt_drain(), Some(false)); - start.process.cleanup(true); - } - - #[test] - fn deferred_finalization_uses_the_latest_abnormal_disposition() { - let start = transaction(StartResultPublication::Publishing); - start.admit_acknowledgement().unwrap(); - - assert_eq!(start.runner_finished(false), None); - let _expiration = start.expire_initial_receipt_deadline().unwrap(); - assert_eq!(start.complete_timeout_callback(), None); - start.begin_receipt_drain(); - assert_eq!(start.finish_receipt_drain(), Some(true)); - start.process.cleanup(false); - } - - #[test] - fn acknowledgement_send_does_not_steal_a_timed_out_receipt_from_drain() { - let (broker, start) = transaction_with_broker(StartResultPublication::Delivered); - let token = ProcessStartToken(7); - let parent_id = start.parent_id; - let process_manager = RunnerProcessManager { + fn association_registration_installs_pending_failure_callback() { + let (broker, transaction) = starting_transaction(); + let process_id = transaction.process.id(); + let manager = RunnerProcessManager { broker, started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), state: Mutex::new(RunnerProcessManagerState { - transactions: vec![(token, Arc::clone(&start))], + transactions: vec![Arc::clone(&transaction)], associations: Vec::new(), active_instances: 1, - active_watchdogs: 0, }), drained: Condvar::new(), }; + let (failed, failure) = mpsc::sync_channel(1); - assert_eq!(start.runner_finished(false), None); - let _expiration = start.expire_initial_receipt_deadline().unwrap(); - process_manager.response_sent( - parent_id, - &BrokerOperation::AcknowledgeProcessStart(token), - &BrokerResult::ProcessStartAcknowledged, - ); + manager + .register_association( + process_id, + Arc::new(move || { + let _ = failed.send(()); + }), + ) + .unwrap(); + transaction.abort(ErrorCode::Internal, true, true); - assert!(process_manager.find_transaction(token).is_some()); - assert_eq!(start.complete_timeout_callback(), None); - let draining = process_manager.association_ending(parent_id); - assert_eq!(draining.transactions.len(), 1); - process_manager.association_ended(parent_id, draining); - assert!(process_manager.find_transaction(token).is_none()); + failure.recv_timeout(Duration::from_secs(1)).unwrap(); + transaction.process.cleanup(true); } } diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index b9f922045e..e7a25dd918 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -29,9 +29,8 @@ use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{ BrokerHostAssociation, BrokerHostError, ConnectionTermination, setup_connection, }; -use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::{BrokerOperation, BrokerRequest}; +use litebox_broker_protocol::message::BrokerRequest; use litebox_broker_protocol::shared_buffer::SHARED_BUFFER_LAYOUT; use litebox_broker_transport::channel::{ HostAssociationShutdown, HostNotificationChannel, HostReceive, HostRequestSource, @@ -44,8 +43,6 @@ use crate::readiness::ReadinessPublisherRuntime; use crate::runner::{RunnerProcessManager, RunnerStartup}; const REQUEST_QUEUE_CAPACITY: usize = 64; -pub(crate) const PROCESS_START_CONTROL_WORKER_COUNT: usize = crate::WORKER_COUNT; -pub(crate) const PROCESS_START_CONTROL_QUEUE_CAPACITY: usize = crate::WORKER_COUNT; const REQUEST_QUEUE_RETRY_DELAY: Duration = Duration::from_millis(1); const REQUEST_QUEUE_STALL_TIMEOUT: Duration = Duration::from_secs(5); @@ -509,22 +506,16 @@ where "process association terminated by process-start control", )); }); - if let Err(error) = process_manager - .register_association(association.process_id(), Arc::clone(&association_failure)) + if let Err(error) = + process_manager.register_association(association.process_id(), association_failure) { failure_coordinator.report(error); - } else { - process_manager - .install_process_association_failure(association.process_id(), association_failure); } } let (request_sender, request_receiver) = sync_channel(REQUEST_QUEUE_CAPACITY); let request_receiver = Arc::new(Mutex::new(request_receiver)); - let (process_start_sender, process_start_receiver) = - sync_channel(PROCESS_START_CONTROL_QUEUE_CAPACITY); - let process_start_receiver = Arc::new(Mutex::new(process_start_receiver)); - let association_drain = std::thread::scope(|scope| { + std::thread::scope(|scope| { let publisher_readiness = Arc::clone(&readiness); let publisher_failure_coordinator = Arc::clone(&failure_coordinator); let publisher = std::thread::Builder::new() @@ -561,32 +552,7 @@ where association: &association, }; - let mut workers = - Vec::with_capacity(crate::WORKER_COUNT + PROCESS_START_CONTROL_WORKER_COUNT); - for worker_id in 0..PROCESS_START_CONTROL_WORKER_COUNT { - let association = Arc::clone(&association); - let process_start_receiver = Arc::clone(&process_start_receiver); - let response_sink = response_sink.clone(); - let worker_failure_coordinator = Arc::clone(&failure_coordinator); - let process_manager_for_worker = process_manager.clone(); - match std::thread::Builder::new() - .name(format!("litebox-broker-process-start-worker-{worker_id}")) - .spawn_scoped(scope, move || { - run_worker( - &association, - &process_start_receiver, - &response_sink, - &worker_failure_coordinator, - process_manager_for_worker.as_ref(), - ); - }) { - Ok(worker) => workers.push(worker), - Err(error) => { - failure_coordinator.report(error); - break; - } - } - } + let mut workers = Vec::with_capacity(crate::WORKER_COUNT); for worker_id in 0..crate::WORKER_COUNT { let association = Arc::clone(&association); let request_receiver = Arc::clone(&request_receiver); @@ -612,18 +578,11 @@ where } } - read_requests( - &mut request_source, - request_sender, - process_start_sender, - &failure_coordinator, - process_manager.as_ref(), - process_id, - ); + read_requests(&mut request_source, request_sender, &failure_coordinator); drop(cancellation); - let association_drain = process_manager - .as_ref() - .map(|process_manager| process_manager.association_ending(process_id)); + if let Some(process_manager) = &process_manager { + process_manager.association_ending(process_id); + } for worker in workers { if worker.join().is_err() { failure_coordinator.report_panic(IoError::other("broker request worker panicked")); @@ -643,12 +602,10 @@ where { failure_coordinator.report_panic(IoError::other("broker readiness publisher panicked")); } - association_drain }); - if let (Some(process_manager), Some(association_drain)) = (&process_manager, association_drain) - { - process_manager.association_ended(process_id, association_drain); + if let Some(process_manager) = &process_manager { + process_manager.association_ended(process_id); } let result = match failure_coordinator.take_error() { @@ -676,10 +633,7 @@ where fn read_requests( request_source: &mut RequestSource, request_sender: SyncSender, - process_start_sender: SyncSender, failure_coordinator: &HostAssociationFailureCoordinator, - process_manager: Option<&Arc>, - process_id: ProcessId, ) where RequestSource: HostRequestSource, Shutdown: HostAssociationShutdown, @@ -690,24 +644,8 @@ fn read_requests( } match request_source.recv_request() { Ok(HostReceive::Message(request)) => { - if let BrokerOperation::AcknowledgeProcessStart(token) = &request.operation - && let Some(process_manager) = process_manager - && let Err(error) = process_manager.admit_acknowledgement(process_id, *token) - { - failure_coordinator.report(map_host_error(BrokerHostError::Broker(error))); - break; - } - let sender = if matches!( - &request.operation, - BrokerOperation::AcknowledgeProcessStart(_) - | BrokerOperation::ReportProcessStartFailure(_) - ) { - &process_start_sender - } else { - &request_sender - }; if !enqueue_request( - sender, + &request_sender, request, failure_coordinator, REQUEST_QUEUE_STALL_TIMEOUT, @@ -792,7 +730,6 @@ fn run_worker( continue; } match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let process_id = association.process_id(); association.execute_request_with( request, |process, operation, shared_buffers| { @@ -801,11 +738,7 @@ fn run_worker( }) }, |response| response_sink.send_response(response), - |operation, result| { - if let Some(process_manager) = process_manager { - process_manager.response_sent(process_id, operation, result); - } - }, + |_, _| {}, ) })) { Ok(Ok(()) | Err(BrokerHostError::AssociationFailed)) => {} diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 70ae7867ed..46e69a538f 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -27,7 +27,6 @@ use litebox_broker_transport_linux_userland::unix_socket::{ const RUNNER_ARGUMENT: &str = "broker-userland-test-runner"; const NETWORK_RUNNER_ARGUMENT: &str = "broker-userland-network-test-runner"; const CHILD_START_RUNNER_ARGUMENT: &str = "broker-userland-child-start-runner"; -const CHILD_CANCEL_RUNNER_ARGUMENT: &str = "broker-userland-child-cancel-runner"; const TEST_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x7465_7374); const FAILING_BOOTSTRAP_FORMAT: ProcessBootstrapFormat = ProcessBootstrapFormat(0x6661_696c); const BROKER_PROCESS_TIMEOUT: Duration = Duration::from_secs(30); @@ -74,19 +73,6 @@ fn run_parent_test() { assert!(child_result.ends_with("\nstarted\nfinished\n")); std::fs::remove_file(child_marker).unwrap(); - let cancelled_marker = unique_child_marker_path(); - let mut cancel_command = Command::new(env!("CARGO_BIN_EXE_litebox-broker-userland")); - cancel_command - .arg("--runner") - .arg(&test_executable) - .arg(CHILD_CANCEL_RUNNER_ARGUMENT) - .arg(&cancelled_marker); - wait_for_broker(cancel_command); - let cancelled_result = std::fs::read_to_string(&cancelled_marker).unwrap(); - assert!(cancelled_result.starts_with("ready:")); - assert_eq!(cancelled_result.lines().count(), 1); - std::fs::remove_file(cancelled_marker).unwrap(); - let gateway = std::net::Ipv4Addr::new(10, 0, 2, 1); let tcp_listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); let tcp_port = tcp_listener.local_addr().unwrap().port(); @@ -278,32 +264,10 @@ fn run_fake_runner(args: &[OsString]) { ) .unwrap(); assert_eq!(started.initial_thread_id, None); - let ready = format!("ready:{}\n", started.process_id.0); - wait_for_marker(marker, &ready); - std::thread::sleep(Duration::from_millis(100)); - assert_eq!(std::fs::read_to_string(marker).unwrap(), ready); - local.acknowledge_process_start(started.token).unwrap(); - return; - } - if args.get(3).and_then(|argument| argument.to_str()) == Some(CHILD_CANCEL_RUNNER_ARGUMENT) { - assert_eq!(args.len(), 5, "unexpected runner arguments: {args:?}"); - let marker = Path::new(&args[4]); - let bootstrap = marker.as_os_str().as_encoded_bytes(); - let inherited_event = local.create_event_with_count(1).unwrap(); - let started = local - .request_process_start( - TEST_BOOTSTRAP_FORMAT, - ProcessBootstrapVersion(1), - SharedBufferSequence::new( - &[SharedBufferSlotIndex(0)], - bootstrap.len().try_into().unwrap(), - ) - .unwrap(), - bootstrap, - InheritedProcessObjects::new(&[inherited_event]).unwrap(), - ) - .unwrap(); - wait_for_marker(marker, &format!("ready:{}\n", started.process_id.0)); + wait_for_marker( + marker, + &format!("ready:{}\nstarted\nfinished\n", started.process_id.0), + ); return; } assert_eq!( From cd9bdfc6805e88ff0f17697127b081b5fd722f7b Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 18:35:03 -0700 Subject: [PATCH 14/21] Simplify runner startup negotiation Allocate process and initial thread identities before launch, commit startup when the broker association activates, and remove the separate readiness protocol. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox/src/thread.rs | 9 + litebox_broker_core/src/lib.rs | 16 + litebox_broker_core/src/process.rs | 61 +-- litebox_broker_host/src/lib.rs | 57 ++- litebox_broker_host/src/test_support.rs | 5 +- litebox_broker_local/src/lib.rs | 54 ++- litebox_broker_local/src/process.rs | 39 +- litebox_broker_local/src/test_support.rs | 5 +- litebox_broker_local_userland/src/linux.rs | 1 + litebox_broker_protocol/src/message.rs | 16 +- litebox_broker_protocol/src/process.rs | 6 +- litebox_broker_protocol/src/wire.rs | 93 +---- litebox_broker_transport/src/pending_calls.rs | 44 +- .../src/unix_socket/host.rs | 1 + .../src/unix_socket/local.rs | 97 +---- .../src/local.rs | 19 +- .../src/named_pipe.rs | 124 +----- litebox_broker_userland/src/runner.rs | 64 +-- .../src/runner/process_manager.rs | 383 +++++------------- litebox_broker_userland/src/runtime.rs | 25 +- .../tests/userland_broker.rs | 23 +- .../src/lib.rs | 15 +- litebox_runner_linux_userland/src/lib.rs | 9 +- .../src/lib.rs | 10 +- .../src/lib.rs | 18 +- litebox_runner_windows_userland/src/lib.rs | 7 +- litebox_shim_linux/src/lib.rs | 25 +- 27 files changed, 383 insertions(+), 843 deletions(-) diff --git a/litebox/src/thread.rs b/litebox/src/thread.rs index 4610415599..d886959d13 100644 --- a/litebox/src/thread.rs +++ b/litebox/src/thread.rs @@ -68,6 +68,15 @@ impl Thread { } impl LiteBox { + /// Adopts a broker thread allocated during process negotiation. + /// + /// The caller must pass the initial thread ID from the negotiated broker + /// association exactly once. + pub fn adopt_thread(&self, id: ThreadId) -> Result { + let broker = self.broker_control().ok_or(CreateError::Unavailable)?; + Ok(Thread { id, broker }) + } + /// Creates a thread belonging to this LiteBox process. /// /// # Panics diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index d9d3f7ebb2..6d1d106226 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -314,6 +314,22 @@ impl BrokerCore { self.create_process_with_parent(None, caller_credential, process::ProcessState::Running) } + /// Allocates one authenticated broker process awaiting association activation. + /// + /// The deployment must call [`BrokerProcess::complete_start`] after the + /// process association becomes active. + /// + /// # Panics + /// + /// Panics if the shared ID allocator violates its range or uniqueness + /// invariants. + pub fn create_attaching_process( + &self, + caller_credential: CallerCredential, + ) -> Result> { + self.create_process_with_parent(None, caller_credential, process::ProcessState::Attaching) + } + fn create_process_with_parent( &self, parent_id: Option, diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index c8d5f9420b..cafad327c2 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -127,7 +127,6 @@ pub struct BrokerProcess { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ProcessState { Attaching, - StartReady { initial_thread_id: Option }, Running, Exiting, } @@ -182,34 +181,15 @@ impl BrokerProcess { matches!(*self.state.lock(), ProcessState::Running) } - /// Records the initial thread supplied by `ProcessReady`. - pub fn mark_start_ready(&self, initial_thread_id: Option) -> Result<()> { - let mut state = self.state.lock(); - match *state { - ProcessState::Attaching => {} - ProcessState::Exiting => return Err(BrokerError::PeerClosed), - ProcessState::StartReady { .. } | ProcessState::Running => { - return Err(BrokerError::Internal); - } - } - if let Some(thread_id) = initial_thread_id - && !self.threads.lock().contains_key(&thread_id) - { - return Err(BrokerError::UnknownObject); - } - *state = ProcessState::StartReady { initial_thread_id }; - Ok(()) - } - - /// Completes startup after the runner reports ready. + /// Completes startup after the process association becomes active. pub fn complete_start(&self) -> Result<()> { let mut state = self.state.lock(); match *state { - ProcessState::StartReady { .. } => { + ProcessState::Attaching => { *state = ProcessState::Running; Ok(()) } - ProcessState::Attaching | ProcessState::Running => Err(BrokerError::Internal), + ProcessState::Running => Err(BrokerError::Internal), ProcessState::Exiting => Err(BrokerError::PeerClosed), } } @@ -295,20 +275,10 @@ impl BrokerProcess { /// Records broker thread exit after its local task teardown completes. pub fn exit_thread(&self, thread_id: ThreadId) -> Result<()> { - let state = self.state.lock(); - if matches!( - *state, - ProcessState::StartReady { - initial_thread_id: Some(pinned), - } if pinned == thread_id - ) { - return Err(BrokerError::WouldBlock); - } let mut threads = self.threads.lock(); let thread = threads .remove(&thread_id) .ok_or(BrokerError::UnknownObject)?; - drop(state); drop(threads); self.core .active_thread_count @@ -960,28 +930,6 @@ mod tests { assert!(!first.owns_thread(thread)); } - #[test] - fn startup_states_defer_initial_thread_exit_until_start_completes() { - let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( - ObjectRights::all(), - )) - .build() - .unwrap(); - let parent = broker - .create_process(CallerCredential::Unauthenticated) - .unwrap(); - let (process, _) = parent.create_child(&[]).unwrap(); - let thread = process.create_thread().unwrap(); - - process.mark_start_ready(Some(thread)).unwrap(); - assert_eq!(process.exit_thread(thread), Err(BrokerError::WouldBlock)); - process.complete_start().unwrap(); - assert!(process.is_running()); - assert_eq!(process.exit_thread(thread), Ok(())); - process.cleanup(true); - parent.cleanup(true); - } - #[test] fn child_is_parented_and_requires_start_completion() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( @@ -994,9 +942,11 @@ mod tests { .unwrap(); let source_handle = crate::event::create(&parent, 1).unwrap(); let (child, inherited_objects) = parent.create_child(&[source_handle]).unwrap(); + let initial_thread_id = child.create_thread().unwrap(); let inherited_handle = inherited_objects[0]; assert_eq!(child.parent_id(), Some(parent.id())); + assert!(child.owns_thread(initial_thread_id)); assert_ne!(inherited_handle, source_handle); assert_eq!( child.check_readiness(inherited_handle).unwrap(), @@ -1004,7 +954,6 @@ mod tests { ); assert!(!child.is_running()); - child.mark_start_ready(None).unwrap(); child.complete_start().unwrap(); assert!(child.is_running()); assert_eq!(*child.state.lock(), ProcessState::Running); diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 1ef6e3bcb3..d23b8a25fd 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -61,7 +61,7 @@ use litebox_broker_protocol::stdio::{ IsTerminalStdioRequest, IsTerminalStdioResponse, MAX_STDIO_TRANSFER_SIZE, ReadStdioRequest, ReadStdioResponse, WriteStdioRequest, WriteStdioResponse, }; -use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId}; +use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId, ThreadId}; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use spin::mutex::SpinMutex; @@ -117,6 +117,11 @@ impl<'a, Memory: SharedMemory> BrokerHostAssociation<'a, Memory> { self.process.id() } + /// Marks the process running after deployment-specific association activation. + pub fn activate_process(&self) -> litebox_broker_core::Result<()> { + self.process.complete_start() + } + /// Requests cancellation of provider operations after the peer disconnects. pub fn request_cancellation(&self) { self.process.request_cancellation(); @@ -221,7 +226,7 @@ impl<'a, Memory: SharedMemory> BrokerHostAssociation<'a, Memory> { #[allow(clippy::too_many_arguments)] pub fn setup_connection<'a, SetupChannel, Memory, ChannelError>( core: &BrokerCore, - process: Option>, + process: Option<(Arc, ThreadId)>, startup: Option, setup_channel: &mut SetupChannel, shared_buffers: &'a SharedBufferPool, @@ -312,8 +317,12 @@ where } let finish_on_setup_error = process.is_none(); - let process = match process.take() { - Some(process) if process.caller_credential() == caller_credential => process, + let (process, initial_thread_id) = match process.take() { + Some((process, initial_thread_id)) + if process.caller_credential() == caller_credential => + { + (process, initial_thread_id) + } Some(_) => { let error = ErrorCode::PolicyDenied; setup_channel @@ -321,8 +330,25 @@ where .map_err(BrokerHostError::Channel)?; return Ok(Err(ConnectionTermination::Rejected(error))); } - None => match core.create_process(caller_credential) { - Ok(process) => process, + None => match core.create_attaching_process(caller_credential) { + Ok(process) => match process.create_thread() { + Ok(initial_thread_id) => (process, initial_thread_id), + Err( + error @ (litebox_broker_core::BrokerError::ResourceExhausted + | litebox_broker_core::BrokerError::OutOfMemory), + ) => { + process.cleanup(true); + let error = ErrorCode::from(error); + setup_channel + .send_handshake_response(&BrokerHandshakeResponse::Error(error)) + .map_err(BrokerHostError::Channel)?; + return Ok(Err(ConnectionTermination::Rejected(error))); + } + Err(error) => { + process.cleanup(true); + return Err(BrokerHostError::from(error)); + } + }, Err(litebox_broker_core::BrokerError::ResourceExhausted) => { let error = ErrorCode::ResourceExhausted; setup_channel @@ -336,6 +362,7 @@ where let response = BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: process.id(), + initial_thread_id, startup, }; let process_retained = retain_process(&process); @@ -513,9 +540,7 @@ fn handle_request( BrokerOperation::File(request) => { handle_file_request(process, request, shared_buffers).map(BrokerResult::File) } - BrokerOperation::StartProcess(_) - | BrokerOperation::ReportProcessReady(_) - | BrokerOperation::ReportProcessStartFailure(_) => { + BrokerOperation::StartProcess(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) } } @@ -1573,7 +1598,7 @@ mod tests { .into_inner() .unwrap() .expect("deployment owner must retain the negotiated process"); - assert!(process.is_running()); + assert!(!process.is_running()); process.cleanup(true); } @@ -1886,6 +1911,7 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: root_process_id(1), + initial_thread_id: ThreadId(2), startup: None, } ); @@ -1921,7 +1947,8 @@ mod tests { }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: root_process_id(2), + process_id: root_process_id(3), + initial_thread_id: ThreadId(4), startup: None, } ] @@ -1960,7 +1987,7 @@ mod tests { .create_process(CallerCredential::Unauthenticated) .unwrap() .id(), - root_process_id(3) + root_process_id(5) ); } @@ -1995,7 +2022,8 @@ mod tests { channel.handshake_responses, [BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: root_process_id(4), + process_id: root_process_id(6), + initial_thread_id: ThreadId(7), startup: None, }] ); @@ -2807,6 +2835,9 @@ mod tests { Ok(association) => association, Err(termination) => return Ok(termination), }; + association + .activate_process() + .expect("test broker process must activate once"); let result = (|| { loop { let request = match control_channel diff --git a/litebox_broker_host/src/test_support.rs b/litebox_broker_host/src/test_support.rs index e3df5b9c6a..64dd5a4c1a 100644 --- a/litebox_broker_host/src/test_support.rs +++ b/litebox_broker_host/src/test_support.rs @@ -65,6 +65,9 @@ impl InProcessBrokerSetup { .association .take() .expect("the in-process local endpoint must negotiate before activation"); + association + .activate_process() + .expect("the in-process broker process must activate once"); InProcessBrokerChannel { association: Some(association), panicked: AtomicBool::new(false), @@ -326,7 +329,7 @@ mod tests { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) - .with_limits(BrokerCoreLimits::DEFAULT.with_thread_quotas(1, 1)) + .with_limits(BrokerCoreLimits::DEFAULT.with_thread_quotas(2, 2)) .build() .unwrap(); diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 4f48ff491f..70f5db339f 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -60,6 +60,7 @@ pub use error::{BrokerLocalError, Result}; pub struct BrokerLocal { channel: Channel, process_id: ProcessId, + initial_thread_id: ThreadId, shared_buffers: SharedBufferPool>, next_request_id: AtomicU64, } @@ -70,12 +71,18 @@ pub struct BrokerNotifications { } impl BrokerLocal { - fn new(channel: Channel, process_id: ProcessId, shared_memory: Arc) -> Self { + fn new( + channel: Channel, + process_id: ProcessId, + initial_thread_id: ThreadId, + shared_memory: Arc, + ) -> Self { let shared_buffers = SharedBufferPool::new(shared_memory, SHARED_BUFFER_LAYOUT) .expect("broker association shared memory has an invalid size"); Self { channel, process_id, + initial_thread_id, shared_buffers, next_request_id: AtomicU64::new(0), } @@ -121,6 +128,7 @@ impl BrokerLocal { response @ BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + initial_thread_id, startup, } => { assert_eq!( @@ -129,7 +137,7 @@ impl BrokerLocal { ); let (channel, shared_memory, activated) = activate(setup).map_err(BrokerLocalError::Channel)?; - let local = Self::new(channel, process_id, shared_memory); + let local = Self::new(channel, process_id, initial_thread_id, shared_memory); let startup = match startup { Some(startup) => { let mut payload = Vec::new(); @@ -155,7 +163,8 @@ impl BrokerLocal { BrokerHandshakeResponse::Error(error) => match error { ErrorCode::UnsupportedVersion | ErrorCode::PolicyDenied - | ErrorCode::ResourceExhausted => Err(BrokerLocalError::Broker(error)), + | ErrorCode::ResourceExhausted + | ErrorCode::OutOfMemory => Err(BrokerLocalError::Broker(error)), ErrorCode::MalformedRequest | ErrorCode::ProtocolState | ErrorCode::UnsupportedOperation @@ -171,6 +180,12 @@ impl BrokerLocal { self.process_id } + /// Returns the broker-assigned initial thread ID. + #[must_use] + pub const fn initial_thread_id(&self) -> ThreadId { + self.initial_thread_id + } + /// Creates a broker thread belonging to this process. /// /// # Panics @@ -373,6 +388,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), startup: None, }), None, @@ -396,6 +412,7 @@ mod tests { assert_eq!(setup_calls.get(), 1); assert!(startup.is_none()); assert_eq!(local.process_id(), test_process_id()); + assert_eq!(local.initial_thread_id(), ThreadId(2)); } #[test] @@ -637,6 +654,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id: test_process_id(), + initial_thread_id: ThreadId(2), startup: None, }), None, @@ -689,21 +707,21 @@ mod tests { } #[test] - fn negotiate_returns_process_id_exhaustion() { - let channel = FakeControlChannel::new( - Some(BrokerHandshakeResponse::Error(ErrorCode::ResourceExhausted)), - None, - ); - let setup_called = Cell::new(false); + fn negotiate_returns_resource_allocation_errors() { + for error in [ErrorCode::ResourceExhausted, ErrorCode::OutOfMemory] { + let channel = + FakeControlChannel::new(Some(BrokerHandshakeResponse::Error(error)), None); + let setup_called = Cell::new(false); - assert!(matches!( - BrokerLocal::negotiate(channel, |channel| { - setup_called.set(true); - Ok((channel, noop_shared_memory(), ())) - }), - Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)) - )); - assert!(!setup_called.get()); + assert!(matches!( + BrokerLocal::negotiate(channel, |channel| { + setup_called.set(true); + Ok((channel, noop_shared_memory(), ())) + }), + Err(BrokerLocalError::Broker(reported)) if reported == error + )); + assert!(!setup_called.get()); + } } #[test] @@ -730,6 +748,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), startup: None, }), None, @@ -754,6 +773,7 @@ mod tests { Some(BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: test_process_id(), + initial_thread_id: ThreadId(2), startup: None, }), None, diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs index 9c54700c1f..ef5ee2637c 100644 --- a/litebox_broker_local/src/process.rs +++ b/litebox_broker_local/src/process.rs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -use litebox_broker_protocol::ThreadId; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ @@ -16,9 +15,9 @@ use crate::{BrokerLocal, BrokerLocalError, Result}; impl BrokerLocal { /// Requests materialization of one child process. /// - /// This call blocks until the child reports ready or startup fails. The - /// caller must retain exclusive ownership of the bootstrap sequence until - /// this method returns. + /// This call blocks until the child's broker association is active or + /// launch fails. The caller must retain exclusive ownership of the + /// bootstrap sequence until this method returns. /// /// # Panics /// @@ -47,36 +46,4 @@ impl BrokerLocal { response => panic!("broker returned unexpected process-start response: {response:?}"), } } - - /// Reports that this child process is ready and waits for broker startup completion. - /// - /// # Panics - /// - /// Panics if the broker returns a response for another operation. - pub fn process_ready(&self, initial_thread_id: Option) -> Result<(), Channel::Error> { - match self.request(BrokerOperation::ReportProcessReady(initial_thread_id))? { - BrokerResult::ProcessReady => Ok(()), - BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), - response => panic!("broker returned unexpected process-ready response: {response:?}"), - } - } - - /// Reports that this child rejected its startup data before becoming ready. - /// - /// # Panics - /// - /// Panics if the broker returns a response for another operation or echoes - /// a different failure. - pub fn report_process_start_failure(&self, error: ErrorCode) -> Result<(), Channel::Error> { - match self.request(BrokerOperation::ReportProcessStartFailure(error))? { - BrokerResult::ProcessStartFailed(reported) if reported == error => Ok(()), - BrokerResult::ProcessStartFailed(reported) => { - panic!("broker reported a different process-start failure: {reported}") - } - BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), - response => { - panic!("broker returned unexpected process-start failure response: {response:?}") - } - } - } } diff --git a/litebox_broker_local/src/test_support.rs b/litebox_broker_local/src/test_support.rs index 6712bf5dbf..3d1b6891a0 100644 --- a/litebox_broker_local/src/test_support.rs +++ b/litebox_broker_local/src/test_support.rs @@ -5,13 +5,14 @@ use alloc::sync::Arc; -use litebox_broker_protocol::ProcessId; +use litebox_broker_protocol::{ProcessId, ThreadId}; use litebox_broker_transport::channel::LocalCallChannel; use litebox_broker_transport::shared_memory::SharedMemory; use crate::BrokerLocal; const TEST_PROCESS_ID: ProcessId = ProcessId(1); +const TEST_THREAD_ID: ThreadId = ThreadId(2); /// Constructs an active broker-local association without exercising protocol negotiation. /// @@ -46,5 +47,5 @@ pub fn test_broker_local_with_process_id( where Channel: LocalCallChannel, { - BrokerLocal::new(channel, process_id, shared_memory) + BrokerLocal::new(channel, process_id, TEST_THREAD_ID, shared_memory) } diff --git a/litebox_broker_local_userland/src/linux.rs b/litebox_broker_local_userland/src/linux.rs index a816212b1b..b29ebc87c9 100644 --- a/litebox_broker_local_userland/src/linux.rs +++ b/litebox_broker_local_userland/src/linux.rs @@ -268,6 +268,7 @@ mod tests { &litebox_broker_protocol::message::BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }, ) diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index fa42699124..663193aeee 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -67,10 +67,6 @@ pub enum BrokerOperation { File(FileRequest), /// Start one child process from an opaque platform bootstrap. StartProcess(ProcessStartupDescriptor), - /// Report that this child process is ready to begin guest execution. - ReportProcessReady(Option), - /// Report that this child rejected its startup data before becoming ready. - ReportProcessStartFailure(ErrorCode), } impl BrokerOperation { @@ -126,9 +122,7 @@ impl BrokerOperation { | Self::Stdio(StdioRequest::IsTerminal(_)) | Self::File( FileRequest::Seek(_) | FileRequest::Truncate(_) | FileRequest::HandleStatus(_), - ) - | Self::ReportProcessReady(_) - | Self::ReportProcessStartFailure(_) => None, + ) => None, } } } @@ -154,6 +148,8 @@ pub enum BrokerHandshakeResponse { broker_protocol_version: ProtocolVersion, /// Assigned process ID. process_id: ProcessId, + /// Assigned initial thread ID. + initial_thread_id: ThreadId, /// Child startup data, absent for the initial process. startup: Option, }, @@ -246,12 +242,8 @@ pub enum BrokerResult { Stdio(StdioResponse), /// File response family. File(FileResponse), - /// A child completed broker startup. + /// A child established its broker association. ProcessStarted(StartedProcess), - /// A child-start failure was reported or observed before startup completed. - ProcessStartFailed(ErrorCode), - /// The broker accepted the child's ready report. - ProcessReady, /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), } diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index 6342ea42b5..ea7e48fb15 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -85,11 +85,11 @@ pub struct ProcessStartupData { pub inherited_objects: InheritedProcessObjects, } -/// Reports a child process that completed broker startup. +/// Identifies a child process whose broker association was established. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct StartedProcess { /// Broker-assigned child process ID. pub process_id: ProcessId, - /// Broker-assigned initial thread ID when it differs from the process ID. - pub initial_thread_id: Option, + /// Broker-assigned initial thread ID. + pub initial_thread_id: ThreadId, } diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 3b9b3691c3..597a8ca3a3 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -50,8 +50,7 @@ const REQUEST_TAG_CREATE_THREAD: u8 = 9; const REQUEST_TAG_EXIT_THREAD: u8 = 10; const REQUEST_TAG_START_PROCESS: u8 = 11; // Tag 12 is reserved for the removed process-start acknowledgement. -const REQUEST_TAG_PROCESS_READY: u8 = 13; -const REQUEST_TAG_REPORT_PROCESS_START_FAILURE: u8 = 14; +// Tags 13 and 14 are reserved for the removed process-ready protocol. // Paired request and successful-response tags intentionally share values. const RESPONSE_TAG_NEGOTIATED: u8 = 0; @@ -67,8 +66,7 @@ const RESPONSE_TAG_THREAD_CREATED: u8 = 9; const RESPONSE_TAG_THREAD_EXITED: u8 = 10; const RESPONSE_TAG_PROCESS_STARTED: u8 = 11; // Tag 12 is reserved for the removed process-start acknowledgement. -const RESPONSE_TAG_PROCESS_READY: u8 = 13; -const RESPONSE_TAG_PROCESS_START_FAILED: u8 = 14; +// Tags 13 and 14 are reserved for the removed process-ready protocol. // Reserve the top of the tag space for responses without paired requests. const RESPONSE_TAG_ERROR: u8 = 253; @@ -128,9 +126,7 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_START_PROCESS => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -212,16 +208,6 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.shared_buffer_sequence(buffer); encode_inherited_objects(&mut encoder, inherited_objects); } - BrokerOperation::ReportProcessReady(initial_thread_id) => { - encoder.u8(REQUEST_TAG_PROCESS_READY); - encoder.request_id(request_id); - encode_optional_thread_id(&mut encoder, initial_thread_id); - } - BrokerOperation::ReportProcessStartFailure(error) => { - encoder.u8(REQUEST_TAG_REPORT_PROCESS_START_FAILURE); - encoder.request_id(request_id); - encode_error_code(&mut encoder, error); - } } encoder.finish() } @@ -242,9 +228,7 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_FILE | REQUEST_TAG_CREATE_THREAD | REQUEST_TAG_EXIT_THREAD - | REQUEST_TAG_START_PROCESS - | REQUEST_TAG_PROCESS_READY - | REQUEST_TAG_REPORT_PROCESS_START_FAILURE => {} + | REQUEST_TAG_START_PROCESS => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -265,12 +249,6 @@ pub fn decode_request(frame: &[u8]) -> Result { buffer: decoder.shared_buffer_sequence()?, inherited_objects: decode_inherited_objects(&mut decoder)?, }), - REQUEST_TAG_PROCESS_READY => { - BrokerOperation::ReportProcessReady(decode_optional_thread_id(&mut decoder)?) - } - REQUEST_TAG_REPORT_PROCESS_START_FAILURE => { - BrokerOperation::ReportProcessStartFailure(decode_error_code(&mut decoder)?) - } _ => unreachable!("active request tag was validated"), }; decoder.finish()?; @@ -290,11 +268,13 @@ pub fn encode_handshake_response(response: BrokerHandshakeResponse) -> Vec { BrokerHandshakeResponse::Negotiated { broker_protocol_version, process_id, + initial_thread_id, startup, } => { encoder.u8(RESPONSE_TAG_NEGOTIATED); encoder.protocol_version(broker_protocol_version); encoder.process_id(process_id); + encoder.thread_id(initial_thread_id); match startup { Some(ProcessStartupDescriptor { format, @@ -333,6 +313,7 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result BrokerHandshakeResponse::Negotiated { broker_protocol_version: decoder.protocol_version()?, process_id: decoder.process_id()?, + initial_thread_id: decoder.thread_id()?, startup: match decoder.u8()? { 0 => None, 1 => Some(ProcessStartupDescriptor { @@ -355,9 +336,7 @@ pub fn decode_handshake_response(frame: &[u8]) -> Result { + | RESPONSE_TAG_PROCESS_STARTED => { return Err(WireError::WrongMessagePhase); } RESPONSE_TAG_VERSION_MISMATCH => BrokerHandshakeResponse::VersionMismatch { @@ -434,16 +413,7 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.u8(RESPONSE_TAG_PROCESS_STARTED); encoder.request_id(request_id); encoder.process_id(process_id); - encode_optional_thread_id(&mut encoder, initial_thread_id); - } - BrokerResult::ProcessStartFailed(error) => { - encoder.u8(RESPONSE_TAG_PROCESS_START_FAILED); - encoder.request_id(request_id); - encode_error_code(&mut encoder, error); - } - BrokerResult::ProcessReady => { - encoder.u8(RESPONSE_TAG_PROCESS_READY); - encoder.request_id(request_id); + encoder.thread_id(initial_thread_id); } BrokerResult::Error(error) => { encoder.u8(RESPONSE_TAG_ERROR); @@ -473,9 +443,7 @@ pub fn decode_response(frame: &[u8]) -> Result { | RESPONSE_TAG_FILE | RESPONSE_TAG_THREAD_CREATED | RESPONSE_TAG_THREAD_EXITED - | RESPONSE_TAG_PROCESS_STARTED - | RESPONSE_TAG_PROCESS_START_FAILED - | RESPONSE_TAG_PROCESS_READY => {} + | RESPONSE_TAG_PROCESS_STARTED => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -493,35 +461,14 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_FILE => BrokerResult::File(fs::decode_fs_response(&mut decoder)?), RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(StartedProcess { process_id: decoder.process_id()?, - initial_thread_id: decode_optional_thread_id(&mut decoder)?, + initial_thread_id: decoder.thread_id()?, }), - RESPONSE_TAG_PROCESS_START_FAILED => { - BrokerResult::ProcessStartFailed(decode_error_code(&mut decoder)?) - } - RESPONSE_TAG_PROCESS_READY => BrokerResult::ProcessReady, _ => unreachable!("active response tag was validated"), }; decoder.finish()?; Ok(BrokerResponse { request_id, result }) } -fn encode_optional_thread_id(encoder: &mut Encoder, thread_id: Option) { - encoder.u8(u8::from(thread_id.is_some())); - if let Some(thread_id) = thread_id { - encoder.thread_id(thread_id); - } -} - -fn decode_optional_thread_id( - decoder: &mut Decoder<'_>, -) -> Result, WireError> { - match decoder.u8()? { - 0 => Ok(None), - 1 => Ok(Some(decoder.thread_id()?)), - _ => Err(WireError::InvalidTag), - } -} - fn encode_inherited_objects(encoder: &mut Encoder, objects: InheritedProcessObjects) { encoder.u8(u8::try_from(objects.as_slice().len()) .expect("bounded inherited-object count must fit in u8")); @@ -695,8 +642,6 @@ mod tests { RESPONSE_TAG_THREAD_CREATED, RESPONSE_TAG_THREAD_EXITED, RESPONSE_TAG_PROCESS_STARTED, - RESPONSE_TAG_PROCESS_READY, - RESPONSE_TAG_PROCESS_START_FAILED, ], [ REQUEST_TAG_NEGOTIATE, @@ -711,8 +656,6 @@ mod tests { REQUEST_TAG_CREATE_THREAD, REQUEST_TAG_EXIT_THREAD, REQUEST_TAG_START_PROCESS, - REQUEST_TAG_PROCESS_READY, - REQUEST_TAG_REPORT_PROCESS_START_FAILURE, ] ); assert_eq!( @@ -993,9 +936,6 @@ mod tests { ]) .unwrap(), }), - BrokerOperation::ReportProcessReady(None), - BrokerOperation::ReportProcessReady(Some(thread_id(19))), - BrokerOperation::ReportProcessStartFailure(ErrorCode::UnsupportedOperation), ]; let mut maximum_encoded_size = 0; @@ -1169,11 +1109,13 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + initial_thread_id: thread_id(2), startup: None, }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(7), + initial_thread_id: thread_id(8), startup: Some(ProcessStartupDescriptor { format: ProcessBootstrapFormat(0x7465_7374), version: ProcessBootstrapVersion(1), @@ -1341,14 +1283,12 @@ mod tests { BrokerResult::File(FileResponse::Failed(FileError::Io)), BrokerResult::ProcessStarted(StartedProcess { process_id: process_id(u32::MAX), - initial_thread_id: None, + initial_thread_id: thread_id(u32::MAX - 1), }), BrokerResult::ProcessStarted(StartedProcess { process_id: process_id(9), - initial_thread_id: Some(thread_id(11)), + initial_thread_id: thread_id(11), }), - BrokerResult::ProcessStartFailed(ErrorCode::PeerClosed), - BrokerResult::ProcessReady, BrokerResult::Error(ErrorCode::PolicyDenied), BrokerResult::Error(ErrorCode::WouldBlock), BrokerResult::Error(ErrorCode::PeerClosed), @@ -1854,6 +1794,7 @@ mod tests { let negotiated = BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + initial_thread_id: thread_id(2), startup: None, }; let mut invalid_startup = encode_handshake_response(negotiated.clone()); @@ -1881,11 +1822,13 @@ mod tests { BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(1), + initial_thread_id: thread_id(2), startup: None, }, BrokerHandshakeResponse::Negotiated { broker_protocol_version: ProtocolVersion(1), process_id: process_id(2), + initial_thread_id: thread_id(3), startup: Some(ProcessStartupDescriptor { format: ProcessBootstrapFormat(3), version: ProcessBootstrapVersion(4), diff --git a/litebox_broker_transport/src/pending_calls.rs b/litebox_broker_transport/src/pending_calls.rs index 2047d8fcfe..c831404a9f 100644 --- a/litebox_broker_transport/src/pending_calls.rs +++ b/litebox_broker_transport/src/pending_calls.rs @@ -13,10 +13,6 @@ use litebox_broker_protocol::message::BrokerResponse; /// Maximum number of active calls waiting for broker responses. pub const MAX_PENDING_CALLS: usize = 64; -/// Pending-call capacity reserved for control requests that must report failure. -pub const RESERVED_PENDING_CALL_CAPACITY: usize = 8; -/// Maximum active ordinary calls after preserving reserved capacity. -pub const MAX_ORDINARY_PENDING_CALLS: usize = MAX_PENDING_CALLS - RESERVED_PENDING_CALL_CAPACITY; /// A mutex usable by [`PendingCalls`]. pub trait PendingCallsMutex { @@ -87,21 +83,18 @@ pub struct PendingCalls { struct PendingCallsInner { calls: BTreeMap>>, - ordinary_calls: usize, failure: Option>, } /// Completion state for one request awaiting a broker response. pub struct PendingCall { - counts_against_ordinary_limit: bool, result: Sync::Mutex>>>, result_ready: Sync::Condvar>>>, } impl PendingCall { - fn new(counts_against_ordinary_limit: bool) -> Self { + fn new() -> Self { Self { - counts_against_ordinary_limit, result: Sync::mutex(None), result_ready: Sync::condvar(), } @@ -132,7 +125,6 @@ impl PendingCalls { Self { state: Sync::mutex(PendingCallsInner { calls: BTreeMap::new(), - ordinary_calls: 0, failure: None, }), capacity_available: Sync::condvar(), @@ -144,29 +136,9 @@ impl PendingCalls { &self, request_id: RequestId, ) -> Result>, PendingCallsError> { - self.register_inner(request_id, true) - } - - /// Registers work using capacity ordinary calls cannot consume. - pub fn register_with_reserved_capacity( - &self, - request_id: RequestId, - ) -> Result>, PendingCallsError> { - self.register_inner(request_id, false) - } - - fn register_inner( - &self, - request_id: RequestId, - counts_against_ordinary_limit: bool, - ) -> Result>, PendingCallsError> { - let pending_call = Arc::new(PendingCall::new(counts_against_ordinary_limit)); + let pending_call = Arc::new(PendingCall::new()); let mut state = self.state.lock(); - while (state.calls.len() >= MAX_PENDING_CALLS - || (counts_against_ordinary_limit - && state.ordinary_calls >= MAX_ORDINARY_PENDING_CALLS)) - && state.failure.is_none() - { + while state.calls.len() >= MAX_PENDING_CALLS && state.failure.is_none() { state = self.capacity_available.wait(state); } if let Some(error) = state.failure.as_ref() { @@ -175,9 +147,6 @@ impl PendingCalls { match state.calls.entry(request_id) { Entry::Vacant(entry) => { entry.insert(Arc::clone(&pending_call)); - if counts_against_ordinary_limit { - state.ordinary_calls += 1; - } } Entry::Occupied(_) => return Err(PendingCallsError::DuplicateRequestId), } @@ -199,12 +168,6 @@ impl PendingCalls { let Some(pending_call) = state.calls.remove(&response.request_id) else { return Err(PendingCallsError::UnknownResponseId); }; - if pending_call.counts_against_ordinary_limit { - state.ordinary_calls = state - .ordinary_calls - .checked_sub(1) - .expect("ordinary pending-call count must remain balanced"); - } self.capacity_available.notify_all(); pending_call }; @@ -223,7 +186,6 @@ impl PendingCalls { } state.failure = Some(Arc::clone(&error)); let pending_calls = core::mem::take(&mut state.calls); - state.ordinary_calls = 0; self.capacity_available.notify_all(); pending_calls }; diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs index 36e8cb8264..671386f1f5 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/host.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/host.rs @@ -613,6 +613,7 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); diff --git a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs index 6bec67ff45..d4b48f296a 100644 --- a/litebox_broker_transport_linux_userland/src/unix_socket/local.rs +++ b/litebox_broker_transport_linux_userland/src/unix_socket/local.rs @@ -22,8 +22,8 @@ use rustix::net::{ }; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, - BrokerRequest, BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, + BrokerResponse, }; use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, @@ -321,17 +321,10 @@ impl LocalCallChannel for UnixControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; let request_id = request.request_id; - let pending_call = if matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - ) { - association - .pending_calls - .register_with_reserved_capacity(request_id) - } else { - association.pending_calls.register(request_id) - } - .map_err(pending_calls_error)?; + let pending_call = association + .pending_calls + .register(request_id) + .map_err(pending_calls_error)?; let request_frame = encode_request(request); let write_result = { @@ -531,7 +524,7 @@ mod control_ring_tests { use std::os::unix::net::UnixListener; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Barrier, mpsc}; + use std::sync::mpsc; type Producer = ControlRingProducer; type Consumer = ControlRingConsumer; @@ -715,6 +708,7 @@ mod control_ring_tests { &encode_handshake_response(BrokerHandshakeResponse::Negotiated { broker_protocol_version: litebox_broker_protocol::BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }), None, @@ -773,81 +767,6 @@ mod control_ring_tests { )); } - #[test] - fn pending_capacity_preserves_process_start_failure_reports() { - use litebox_broker_transport::pending_calls::{ - MAX_ORDINARY_PENDING_CALLS, RESERVED_PENDING_CALL_CAPACITY, - }; - - let (channel, shutdown, mut responses, mut requests, _peer) = activate_local(|| {}); - let channel = Arc::new(channel); - let start = Arc::new(Barrier::new(MAX_PENDING_CALLS + 2)); - let mut callers = (0..=MAX_ORDINARY_PENDING_CALLS) - .map(|id| { - let channel = Arc::clone(&channel); - let start = Arc::clone(&start); - thread::spawn(move || { - start.wait(); - channel.call(request(id as u64)) - }) - }) - .collect::>(); - callers.extend((0..RESERVED_PENDING_CALL_CAPACITY).map(|index| { - let reserved_channel = Arc::clone(&channel); - let reserved_start = Arc::clone(&start); - thread::spawn(move || { - reserved_start.wait(); - reserved_channel.call(BrokerRequest { - request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), - operation: BrokerOperation::ReportProcessStartFailure( - litebox_broker_protocol::error::ErrorCode::UnsupportedOperation, - ), - }) - }) - })); - start.wait(); - - let mut published = Vec::new(); - for _ in 0..MAX_PENDING_CALLS { - published.push(read_request(&mut requests)); - } - assert!(published.iter().any(|request| matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - ))); - assert_eq!( - published - .iter() - .filter(|request| matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - )) - .count(), - RESERVED_PENDING_CALL_CAPACITY - ); - let released_request = published - .iter() - .find(|request| matches!(&request.operation, BrokerOperation::CloseObject(_))) - .unwrap(); - write_payload( - &mut responses, - &encode_response(response(released_request.request_id)), - ); - let released = read_request(&mut requests); - assert!( - !published - .iter() - .any(|request| request.request_id == released.request_id) - ); - - shutdown.shutdown().unwrap(); - let completed = callers - .into_iter() - .map(|caller| usize::from(caller.join().unwrap().is_ok())) - .sum::(); - assert_eq!(completed, 1); - } - #[test] fn unknown_duplicate_and_malformed_responses_fail_closed() { for payload_kind in 0..3 { diff --git a/litebox_broker_transport_windows_userland/src/local.rs b/litebox_broker_transport_windows_userland/src/local.rs index 42dddaf266..96d263d6ae 100644 --- a/litebox_broker_transport_windows_userland/src/local.rs +++ b/litebox_broker_transport_windows_userland/src/local.rs @@ -12,8 +12,8 @@ use std::thread; use std::time::{Duration, Instant}; use litebox_broker_protocol::message::{ - BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, - BrokerRequest, BrokerResponse, + BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerRequest, + BrokerResponse, }; use litebox_broker_protocol::wire::{ decode_handshake_response, decode_notification, decode_response, encode_handshake_request, @@ -230,17 +230,10 @@ impl LocalCallChannel for WindowsControlRingLocalCallChannel { fn call(&self, request: BrokerRequest) -> IoResult { let association = &self.association; let request_id = request.request_id; - let pending_call = if matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - ) { - association - .pending_calls - .register_with_reserved_capacity(request_id) - } else { - association.pending_calls.register(request_id) - } - .map_err(pending_calls_error)?; + let pending_call = association + .pending_calls + .register(request_id) + .map_err(pending_calls_error)?; let frame = encode_request(request); let write_result = { let mut producer = association diff --git a/litebox_broker_transport_windows_userland/src/named_pipe.rs b/litebox_broker_transport_windows_userland/src/named_pipe.rs index 67d9b96ebc..d796e31b83 100644 --- a/litebox_broker_transport_windows_userland/src/named_pipe.rs +++ b/litebox_broker_transport_windows_userland/src/named_pipe.rs @@ -138,7 +138,7 @@ fn wide_string(value: &OsStr) -> IoResult> { #[cfg(test)] mod tests { use super::*; - use std::sync::{Arc, Barrier}; + use std::sync::Arc; use std::time::{Duration, Instant}; use crate::setup::{OwnedEvent, read_pipe_until_cancelled, write_frame}; @@ -295,6 +295,7 @@ mod tests { host.send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); @@ -338,6 +339,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); @@ -415,6 +417,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); @@ -473,124 +476,6 @@ mod tests { host.join().unwrap(); } - #[test] - fn pending_capacity_preserves_process_start_calls() { - use litebox_broker_transport::pending_calls::{ - MAX_ORDINARY_PENDING_CALLS, RESERVED_PENDING_CALL_CAPACITY, - }; - - let control_name = pipe_name("pending-capacity-control"); - let control_listener = WindowsNamedPipeListener::bind(&control_name).unwrap(); - let (local_ring, host_ring) = control_rings(); - let host = std::thread::spawn(move || { - let mut setup = accept_host_guaranteed(control_listener); - assert!(matches!( - setup.recv_handshake_request().unwrap(), - HostReceive::Message(BrokerHandshakeRequest { .. }) - )); - setup - .send_handshake_response(&BrokerHandshakeResponse::Negotiated { - broker_protocol_version: BROKER_PROTOCOL_VERSION, - process_id: litebox_broker_protocol::ProcessId(1), - startup: None, - }) - .unwrap(); - let (mut requests, responses, _notifications, shutdown) = - setup.into_active(host_ring).unwrap(); - - let mut published = Vec::new(); - for _ in 0..litebox_broker_transport::pending_calls::MAX_PENDING_CALLS { - let HostReceive::Message(request) = requests.recv_request().unwrap() else { - panic!("expected pending request"); - }; - published.push(request); - } - assert!(published.iter().any(|request| matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - ))); - assert_eq!( - published - .iter() - .filter(|request| matches!( - &request.operation, - BrokerOperation::ReportProcessStartFailure(_) - )) - .count(), - RESERVED_PENDING_CALL_CAPACITY - ); - let released_request = published - .iter() - .find(|request| matches!(&request.operation, BrokerOperation::CloseObject(_))) - .unwrap(); - responses - .send_response(&BrokerResponse { - request_id: released_request.request_id, - result: BrokerResult::ObjectClosed, - }) - .unwrap(); - let HostReceive::Message(released) = requests.recv_request().unwrap() else { - panic!("expected released request"); - }; - assert!( - !published - .iter() - .any(|request| request.request_id == released.request_id) - ); - shutdown.shutdown().unwrap(); - }); - - let deadline = Instant::now() + Duration::from_secs(5); - let mut setup = - WindowsNamedPipeLocalSetupChannel::connect_with_setup_deadline(&control_name, deadline) - .unwrap(); - setup - .send_handshake_request(&BrokerHandshakeRequest { - protocol_version: BROKER_PROTOCOL_VERSION, - }) - .unwrap(); - setup.recv_handshake_response().unwrap().unwrap(); - let (calls, _notifications) = setup.into_active(local_ring).unwrap(); - let calls = Arc::new(calls); - let start = Arc::new(Barrier::new( - litebox_broker_transport::pending_calls::MAX_PENDING_CALLS + 2, - )); - let mut callers = (0..=MAX_ORDINARY_PENDING_CALLS) - .map(|id| { - let calls = Arc::clone(&calls); - let start = Arc::clone(&start); - std::thread::spawn(move || { - start.wait(); - calls.call(BrokerRequest { - request_id: RequestId(id as u64), - operation: BrokerOperation::CloseObject(ObjectHandle(id as u64)), - }) - }) - }) - .collect::>(); - callers.extend((0..RESERVED_PENDING_CALL_CAPACITY).map(|index| { - let reserved_calls = Arc::clone(&calls); - let reserved_start = Arc::clone(&start); - std::thread::spawn(move || { - reserved_start.wait(); - reserved_calls.call(BrokerRequest { - request_id: RequestId((MAX_ORDINARY_PENDING_CALLS + 1 + index) as u64), - operation: BrokerOperation::ReportProcessStartFailure( - litebox_broker_protocol::error::ErrorCode::UnsupportedOperation, - ), - }) - }) - })); - start.wait(); - - host.join().unwrap(); - let completed = callers - .into_iter() - .map(|caller| usize::from(caller.join().unwrap().is_ok())) - .sum::(); - assert_eq!(completed, 1); - } - #[test] fn host_shutdown_interrupts_blocked_request_receive() { let control_name = pipe_name("shutdown-control"); @@ -606,6 +491,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); diff --git a/litebox_broker_userland/src/runner.rs b/litebox_broker_userland/src/runner.rs index d777a4edeb..7a8cc6d926 100644 --- a/litebox_broker_userland/src/runner.rs +++ b/litebox_broker_userland/src/runner.rs @@ -113,8 +113,6 @@ enum RunnerShutdownState { } struct RunnerCompletion { - result: IoResult, - runner_success: Option, runner_signal: Option, runner_exit_code: Option, termination_provenance: TerminationProvenance, @@ -123,29 +121,15 @@ struct RunnerCompletion { } #[derive(Clone, Copy, Default)] -struct TerminationProvenance(u8); +struct TerminationProvenance(bool); impl TerminationProvenance { - const BROKER_TERMINATION: u8 = 1; - const REPORTED_START_FAILURE: u8 = 2; - - const fn new(broker_termination: bool, reported_start_failure: bool) -> Self { - let mut value = 0; - if broker_termination { - value |= Self::BROKER_TERMINATION; - } - if reported_start_failure { - value |= Self::REPORTED_START_FAILURE; - } - Self(value) + const fn new(broker_termination: bool) -> Self { + Self(broker_termination) } const fn broker_termination(self) -> bool { - self.0 & Self::BROKER_TERMINATION != 0 - } - - const fn reported_start_failure(self) -> bool { - self.0 & Self::REPORTED_START_FAILURE != 0 + self.0 } } @@ -353,24 +337,15 @@ impl RunnerInstance { if runner_status.is_err() { transaction.mark_abnormal(); } - let runner_success = runner_status.as_ref().ok().map(ExitStatus::success); let runner_signal = runner_status .as_ref() .ok() .copied() .and_then(runner_exit_signal); let runner_exit_code = runner_status.as_ref().ok().and_then(ExitStatus::code); - let termination_provenance = TerminationProvenance::new( - self.shutdown.termination_was_dispatched(), - shutdown_request.expected_start_failure_was_reported(), - ); - let result = runner_status.and_then(|status| { - association_result.result?; - Ok(status) - }); + let termination_provenance = + TerminationProvenance::new(self.shutdown.termination_was_dispatched()); RunnerCompletion { - result, - runner_success, runner_signal, runner_exit_code, termination_provenance, @@ -413,7 +388,7 @@ const fn runner_signal_is_abnormal(_signal: Option, _broker_termination: bo #[cfg(all(windows, target_arch = "x86_64"))] const fn runner_exit_code_is_crash(exit_code: Option) -> bool { - matches!(exit_code, Some(code) if code as u32 >= 0x8000_0000) + matches!(exit_code, Some(code) if code.cast_unsigned() >= 0x8000_0000) } #[cfg(not(all(windows, target_arch = "x86_64")))] @@ -421,35 +396,14 @@ const fn runner_exit_code_is_crash(_exit_code: Option) -> bool { false } -#[cfg(all(windows, target_arch = "x86_64"))] -const fn runner_exit_code_is_expected_shutdown( - exit_code: Option, - broker_termination: bool, -) -> bool { - broker_termination && matches!(exit_code, Some(1)) -} - #[cfg(all(windows, target_arch = "x86_64"))] const _: () = { - let access_violation = 0xc000_0005_u32 as i32; - let breakpoint = 0x8000_0003_u32 as i32; + let access_violation = 0xc000_0005_u32.cast_signed(); + let breakpoint = 0x8000_0003_u32.cast_signed(); assert!(runner_exit_code_is_crash(Some(access_violation))); assert!(runner_exit_code_is_crash(Some(breakpoint))); - assert!(!runner_exit_code_is_expected_shutdown( - Some(access_violation), - true - )); - assert!(runner_exit_code_is_expected_shutdown(Some(1), true)); }; -#[cfg(not(all(windows, target_arch = "x86_64")))] -const fn runner_exit_code_is_expected_shutdown( - _exit_code: Option, - _broker_termination: bool, -) -> bool { - false -} - fn accept_runner_channel( deadline: Instant, channel_name: &'static str, diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index b57cf8c06d..a3e15f78b3 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -7,7 +7,7 @@ use std::io::{Error as IoError, Result as IoResult}; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; -use litebox_broker_core::{BrokerCore, BrokerError, BrokerProcess}; +use litebox_broker_core::{BrokerCore, BrokerProcess}; use litebox_broker_host::{RequestFailure, copy_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; @@ -20,7 +20,7 @@ use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; use super::{ RunnerCompletion, RunnerConfig, RunnerInstance, RunnerShutdown, TerminationProvenance, - runner_exit_code_is_crash, runner_exit_code_is_expected_shutdown, runner_signal_is_abnormal, + runner_exit_code_is_crash, runner_signal_is_abnormal, }; const PROCESS_START_TIMEOUT: Duration = Duration::from_secs(5); @@ -38,6 +38,7 @@ pub(crate) struct RunnerProcessManager { /// Startup context for a runner whose broker process was created by its parent. pub(crate) struct RunnerStartup { process: Arc, + initial_thread_id: ThreadId, transaction: Arc, data: ProcessStartupData, } @@ -47,8 +48,10 @@ impl RunnerStartup { Arc::clone(&self.transaction) } - pub(crate) fn into_process_and_data(self) -> (Arc, ProcessStartupData) { - (self.process, self.data) + pub(crate) fn into_process_and_data( + self, + ) -> ((Arc, ThreadId), ProcessStartupData) { + ((self.process, self.initial_thread_id), self.data) } } @@ -64,6 +67,7 @@ pub(crate) type AssociationFailure = Arc; pub(super) struct ProcessStartTransaction { parent_id: ProcessId, process: Arc, + initial_thread_id: ThreadId, state: Mutex, changed: Condvar, } @@ -71,8 +75,6 @@ pub(super) struct ProcessStartTransaction { #[derive(Clone, Copy)] enum ProcessStartPhase { Starting, - Ready { initial_thread_id: Option }, - Completing { cancellation: Option }, Running, Failed(ErrorCode), } @@ -83,7 +85,6 @@ struct ProcessStartTransactionState { association_failure: Option, shutdown_request: ShutdownRequest, abnormal: bool, - start_completed: bool, finalization: FinalizationState, } @@ -99,17 +100,12 @@ enum FinalizationState { pub(super) enum ShutdownRequest { None, Expected, - ExpectedStartFailure, Unexpected, } impl ShutdownRequest { pub(super) const fn was_expected(self) -> bool { - matches!(self, Self::Expected | Self::ExpectedStartFailure) - } - - pub(super) const fn expected_start_failure_was_reported(self) -> bool { - matches!(self, Self::ExpectedStartFailure) + matches!(self, Self::Expected) } } @@ -152,16 +148,6 @@ impl RunnerProcessManager { }) .map(BrokerResult::ProcessStarted), ), - BrokerOperation::ReportProcessReady(initial_thread_id) => Some( - self.process_ready(process.id(), *initial_thread_id) - .map(|()| BrokerResult::ProcessReady) - .map_err(process_extension_error), - ), - BrokerOperation::ReportProcessStartFailure(error) => Some( - self.report_process_start_failure(process.id(), *error) - .map(|()| BrokerResult::ProcessStartFailed(*error)) - .map_err(process_extension_error), - ), _ => None, } } @@ -180,19 +166,26 @@ impl RunnerProcessManager { let (process, inherited_objects) = parent .create_child(requested_inherited_objects.as_slice()) .map_err(ErrorCode::from)?; + let initial_thread_id = match process.create_thread() { + Ok(initial_thread_id) => initial_thread_id, + Err(error) => { + process.cleanup(true); + return Err(ErrorCode::from(error)); + } + }; let inherited_objects = InheritedProcessObjects::new(&inherited_objects) .expect("child handle count must match the bounded inheritance request"); let process_id = process.id(); let transaction = Arc::new(ProcessStartTransaction { parent_id: parent.id(), process: Arc::clone(&process), + initial_thread_id, state: Mutex::new(ProcessStartTransactionState { phase: ProcessStartPhase::Starting, shutdown: None, association_failure: None, shutdown_request: ShutdownRequest::None, abnormal: false, - start_completed: false, finalization: FinalizationState::Active, }), changed: Condvar::new(), @@ -233,6 +226,7 @@ impl RunnerProcessManager { instance.run_started_process_to_completion( RunnerStartup { process, + initial_thread_id, transaction: Arc::clone(&thread_transaction), data: ProcessStartupData { format, @@ -249,11 +243,9 @@ impl RunnerProcessManager { Ok(Ok(result)) => { process_manager.runner_finished(&thread_transaction, result, false); } - Ok(Err(error)) => process_manager.runner_finished( + Ok(Err(_error)) => process_manager.runner_finished( &thread_transaction, RunnerCompletion { - result: Err(error), - runner_success: None, runner_signal: None, runner_exit_code: None, termination_provenance: TerminationProvenance::default(), @@ -265,8 +257,6 @@ impl RunnerProcessManager { Err(_) => process_manager.runner_finished( &thread_transaction, RunnerCompletion { - result: Err(IoError::other("runner process thread panicked")), - runner_success: None, runner_signal: None, runner_exit_code: None, termination_provenance: TerminationProvenance::default(), @@ -287,17 +277,10 @@ impl RunnerProcessManager { let deadline = Instant::now() + PROCESS_START_TIMEOUT; let result = transaction - .wait_until_ready(deadline) - .and_then(|initial_thread_id| { - if parent.is_cancellation_requested() { - transaction.abort(ErrorCode::PeerClosed, false, true); - return Err(ErrorCode::PeerClosed); - } - transaction.complete_start()?; - Ok(StartedProcess { - process_id, - initial_thread_id, - }) + .wait_until_running(deadline) + .map(|()| StartedProcess { + process_id, + initial_thread_id: transaction.initial_thread_id, }); self.remove_transaction(process_id); if let Some(abnormal) = transaction.protocol_finished() { @@ -306,26 +289,6 @@ impl RunnerProcessManager { result } - fn process_ready( - &self, - process_id: ProcessId, - initial_thread_id: Option, - ) -> Result<(), ErrorCode> { - self.find_transaction(process_id) - .ok_or(ErrorCode::PeerClosed)? - .ready_and_wait(initial_thread_id) - } - - fn report_process_start_failure( - &self, - process_id: ProcessId, - error: ErrorCode, - ) -> Result<(), ErrorCode> { - self.find_transaction(process_id) - .ok_or(ErrorCode::PeerClosed)? - .report_start_failure(error) - } - pub(crate) fn association_ending(&self, process_id: ProcessId) { let (children, transaction) = { let state = self @@ -361,6 +324,7 @@ impl RunnerProcessManager { &self, process_id: ProcessId, failure: AssociationFailure, + is_started_process: bool, ) -> IoResult<()> { let transaction = { let mut state = self @@ -387,8 +351,25 @@ impl RunnerProcessManager { .find(|transaction| transaction.process.id() == process_id) .cloned() }; - if let Some(transaction) = transaction { + if is_started_process { + let Some(transaction) = transaction else { + self.unregister_association(process_id); + return Err(IoError::other( + "started process has no pending start transaction", + )); + }; transaction.install_association_failure(failure); + if let Err(error) = transaction.activate() { + self.unregister_association(process_id); + return Err(IoError::other(format!( + "failed to activate broker process association: {error}" + ))); + } + } else if transaction.is_some() { + self.unregister_association(process_id); + return Err(IoError::other( + "root process unexpectedly has a pending start transaction", + )); } Ok(()) } @@ -407,16 +388,6 @@ impl RunnerProcessManager { } } - fn find_transaction(&self, process_id: ProcessId) -> Option> { - self.state - .lock() - .expect("runner process manager state mutex poisoned") - .transactions - .iter() - .find(|transaction| transaction.process.id() == process_id) - .cloned() - } - fn remove_transaction(&self, process_id: ProcessId) { let mut state = self .state @@ -437,14 +408,6 @@ impl RunnerProcessManager { result: RunnerCompletion, thread_panicked: bool, ) { - let unexpected_runner_failure = result.runner_success == Some(false) - && result.runner_signal.is_none() - && !transaction.start_completed() - && !result.termination_provenance.reported_start_failure() - && !runner_exit_code_is_expected_shutdown( - result.runner_exit_code, - result.termination_provenance.broker_termination(), - ); let unexpected_crash = runner_signal_is_abnormal( result.runner_signal, result.termination_provenance.broker_termination(), @@ -453,13 +416,8 @@ impl RunnerProcessManager { || result.association_panicked || result.shutdown_observation_failed || unexpected_crash - || runner_exit_code_is_crash(result.runner_exit_code) - || unexpected_runner_failure; - if result.result.is_err() || result.runner_success != Some(true) { - transaction.abort(ErrorCode::PeerClosed, abnormal, false); - } else if !transaction.start_completed() { - transaction.abort(ErrorCode::PeerClosed, false, false); - } + || runner_exit_code_is_crash(result.runner_exit_code); + transaction.abort(ErrorCode::PeerClosed, abnormal, false); if let Some(abnormal) = transaction.runner_finished(abnormal) { self.finish_transaction(transaction, abnormal); } @@ -512,20 +470,8 @@ const fn process_extension_error(error: ErrorCode) -> RequestFailure { } } -const fn process_start_failure_is_expected(error: ErrorCode) -> bool { - matches!( - error, - ErrorCode::UnsupportedOperation - | ErrorCode::PolicyDenied - | ErrorCode::InvalidRights - | ErrorCode::ResourceExhausted - | ErrorCode::WouldBlock - | ErrorCode::OutOfMemory - ) -} - impl ProcessStartTransaction { - fn wait_until_ready(&self, deadline: Instant) -> Result, ErrorCode> { + fn wait_until_running(&self, deadline: Instant) -> Result<(), ErrorCode> { let mut state = self .state .lock() @@ -551,53 +497,13 @@ impl ProcessStartTransaction { return Err(ErrorCode::Internal); } } - ProcessStartPhase::Ready { initial_thread_id } => return Ok(initial_thread_id), - ProcessStartPhase::Completing { .. } | ProcessStartPhase::Running => { - return Err(ErrorCode::ProtocolState); - } - ProcessStartPhase::Failed(error) => return Err(error), - } - } - } - - fn ready_and_wait(&self, initial_thread_id: Option) -> Result<(), ErrorCode> { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - if !matches!(state.phase, ProcessStartPhase::Starting) { - return Err(match state.phase { - ProcessStartPhase::Failed(error) => error, - _ => ErrorCode::ProtocolState, - }); - } - self.process - .mark_start_ready(initial_thread_id) - .map_err(|error| match error { - BrokerError::UnknownObject => ErrorCode::ProtocolState, - error => ErrorCode::from(error), - })?; - state.phase = ProcessStartPhase::Ready { initial_thread_id }; - self.changed.notify_all(); - loop { - match state.phase { - ProcessStartPhase::Ready { .. } | ProcessStartPhase::Completing { .. } => { - state = self - .changed - .wait(state) - .expect("process start transaction mutex poisoned"); - } ProcessStartPhase::Running => return Ok(()), ProcessStartPhase::Failed(error) => return Err(error), - ProcessStartPhase::Starting => unreachable!("ready state cannot regress"), } } } - fn report_start_failure(&self, error: ErrorCode) -> Result<(), ErrorCode> { - if !process_start_failure_is_expected(error) { - return Err(ErrorCode::ProtocolState); - } + fn activate(&self) -> Result<(), ErrorCode> { let mut state = self .state .lock() @@ -605,71 +511,24 @@ impl ProcessStartTransaction { match state.phase { ProcessStartPhase::Starting => {} ProcessStartPhase::Failed(error) => return Err(error), - _ => return Err(ErrorCode::ProtocolState), + ProcessStartPhase::Running => return Err(ErrorCode::ProtocolState), } - state.phase = ProcessStartPhase::Failed(error); - state.shutdown_request = ShutdownRequest::ExpectedStartFailure; - self.changed.notify_all(); - Ok(()) - } - - fn complete_start(&self) -> Result<(), ErrorCode> { - { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - match state.phase { - ProcessStartPhase::Ready { .. } => { - state.phase = ProcessStartPhase::Completing { cancellation: None }; - } - ProcessStartPhase::Failed(error) => return Err(error), - _ => return Err(ErrorCode::ProtocolState), + match self.process.complete_start().map_err(ErrorCode::from) { + Ok(()) => { + state.phase = ProcessStartPhase::Running; + self.changed.notify_all(); + Ok(()) } - } - - let completion = self.process.complete_start().map_err(ErrorCode::from); - let (result, association_failure, shutdown) = { - let mut state = self - .state - .lock() - .expect("process start transaction mutex poisoned"); - let ProcessStartPhase::Completing { cancellation } = state.phase else { - unreachable!("process completion phase cannot change independently"); - }; - match completion { - Ok(()) => { - state.start_completed = true; - if let Some(error) = cancellation { - state.phase = ProcessStartPhase::Failed(error); - (Err(error), None, None) - } else { - state.phase = ProcessStartPhase::Running; - (Ok(()), None, None) - } - } - Err(error) => { - state.phase = ProcessStartPhase::Failed(error); - state.abnormal |= error == ErrorCode::Internal; - if state.shutdown_request == ShutdownRequest::None { - state.shutdown_request = ShutdownRequest::Expected; - } - ( - Err(error), - state.association_failure.as_ref().map(Arc::clone), - state.shutdown.clone(), - ) + Err(error) => { + state.phase = ProcessStartPhase::Failed(error); + state.abnormal |= error == ErrorCode::Internal; + if state.shutdown_request == ShutdownRequest::None { + state.shutdown_request = ShutdownRequest::Expected; } + self.changed.notify_all(); + Err(error) } - }; - self.changed.notify_all(); - if let Some(association_failure) = association_failure { - association_failure(); - } - if let Some(shutdown) = shutdown { - shutdown.shutdown(); } - result } pub(super) fn install_shutdown(&self, shutdown: Arc) { @@ -707,19 +566,11 @@ impl ProcessStartTransaction { .lock() .expect("process start transaction mutex poisoned"); state.abnormal |= abnormal; - let should_terminate = match &mut state.phase { - ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { + let should_terminate = match state.phase { + ProcessStartPhase::Starting => { state.phase = ProcessStartPhase::Failed(error); true } - ProcessStartPhase::Completing { cancellation } => { - if cancellation.is_none() { - *cancellation = Some(error); - true - } else { - false - } - } ProcessStartPhase::Failed(_) | ProcessStartPhase::Running => false, }; if should_terminate && state.shutdown_request == ShutdownRequest::None { @@ -753,15 +604,10 @@ impl ProcessStartTransaction { .lock() .expect("process start transaction mutex poisoned"); state.association_failure = None; - match &mut state.phase { - ProcessStartPhase::Starting | ProcessStartPhase::Ready { .. } => { + match state.phase { + ProcessStartPhase::Starting => { state.phase = ProcessStartPhase::Failed(ErrorCode::PeerClosed); } - ProcessStartPhase::Completing { cancellation } => { - if cancellation.is_none() { - *cancellation = Some(ErrorCode::PeerClosed); - } - } ProcessStartPhase::Failed(_) | ProcessStartPhase::Running => {} } self.changed.notify_all(); @@ -791,13 +637,6 @@ impl ProcessStartTransaction { .shutdown_request } - fn start_completed(&self) -> bool { - self.state - .lock() - .expect("process start transaction mutex poisoned") - .start_completed - } - fn protocol_finished(&self) -> Option { let mut state = self .state @@ -847,7 +686,7 @@ mod tests { use litebox_broker_core::{BrokerCore, CallerCredential, ObjectRights, PolicyEngine}; use litebox_broker_protocol::error::ErrorCode; use std::path::PathBuf; - use std::sync::{Arc, Condvar, Mutex, mpsc}; + use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; fn starting_transaction() -> (BrokerCore, Arc) { @@ -861,19 +700,20 @@ mod tests { .unwrap(); let parent_id = parent.id(); let (process, _) = parent.create_child(&[]).unwrap(); + let initial_thread_id = process.create_thread().unwrap(); parent.cleanup(true); ( broker, Arc::new(ProcessStartTransaction { parent_id, process, + initial_thread_id, state: Mutex::new(ProcessStartTransactionState { phase: ProcessStartPhase::Starting, shutdown: None, association_failure: None, shutdown_request: ShutdownRequest::None, abnormal: false, - start_completed: false, finalization: FinalizationState::Active, }), changed: Condvar::new(), @@ -882,37 +722,21 @@ mod tests { } #[test] - fn ready_report_waits_for_broker_start_completion() { + fn association_activation_wakes_the_parent_request() { let (_broker, transaction) = starting_transaction(); - let ready = Arc::clone(&transaction); - let waiter = std::thread::spawn(move || ready.ready_and_wait(None)); - - assert_eq!( - transaction - .wait_until_ready(Instant::now() + Duration::from_secs(1)) - .unwrap(), - None - ); - transaction.complete_start().unwrap(); + let waiting = Arc::clone(&transaction); + let waiter = std::thread::spawn(move || { + waiting.wait_until_running(Instant::now() + Duration::from_secs(1)) + }); + transaction.activate().unwrap(); waiter.join().unwrap().unwrap(); assert!(transaction.process.is_running()); - transaction.process.cleanup(true); - } - - #[test] - fn reported_start_failure_wakes_the_parent_request() { - let (_broker, transaction) = starting_transaction(); - - transaction - .report_start_failure(ErrorCode::UnsupportedOperation) - .unwrap(); - - assert_eq!( - transaction.wait_until_ready(Instant::now() + Duration::from_secs(1)), - Err(ErrorCode::UnsupportedOperation) + assert!( + transaction + .process + .owns_thread(transaction.initial_thread_id) ); - assert!(transaction.shutdown_request().was_expected()); transaction.process.cleanup(true); } @@ -922,7 +746,7 @@ mod tests { let deadline = Instant::now() + Duration::from_millis(1); assert_eq!( - transaction.wait_until_ready(deadline), + transaction.wait_until_running(deadline), Err(ErrorCode::Internal) ); assert!(matches!( @@ -933,17 +757,16 @@ mod tests { } #[test] - fn parent_abort_releases_a_ready_child() { + fn parent_abort_prevents_association_activation() { let (_broker, transaction) = starting_transaction(); - let ready = Arc::clone(&transaction); - let waiter = std::thread::spawn(move || ready.ready_and_wait(None)); - transaction - .wait_until_ready(Instant::now() + Duration::from_secs(1)) - .unwrap(); transaction.abort(ErrorCode::PeerClosed, false, true); - assert_eq!(waiter.join().unwrap(), Err(ErrorCode::PeerClosed)); + assert_eq!(transaction.activate(), Err(ErrorCode::PeerClosed)); + assert_eq!( + transaction.wait_until_running(Instant::now() + Duration::from_secs(1)), + Err(ErrorCode::PeerClosed) + ); transaction.process.cleanup(true); } @@ -975,14 +798,14 @@ mod tests { manager.association_ending(parent_id); assert_eq!( - transaction.wait_until_ready(Instant::now() + PROCESS_START_TIMEOUT), + transaction.wait_until_running(Instant::now() + PROCESS_START_TIMEOUT), Err(ErrorCode::PeerClosed) ); transaction.process.cleanup(true); } #[test] - fn association_registration_installs_pending_failure_callback() { + fn association_registration_activates_the_process() { let (broker, transaction) = starting_transaction(); let process_id = transaction.process.id(); let manager = RunnerProcessManager { @@ -995,19 +818,35 @@ mod tests { }), drained: Condvar::new(), }; - let (failed, failure) = mpsc::sync_channel(1); - manager - .register_association( - process_id, - Arc::new(move || { - let _ = failed.send(()); - }), - ) + .register_association(process_id, Arc::new(|| {}), true) .unwrap(); - transaction.abort(ErrorCode::Internal, true, true); - failure.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(transaction.process.is_running()); + transaction.process.cleanup(true); + } + + #[test] + fn late_started_association_is_not_activated_as_a_root() { + let (broker, transaction) = starting_transaction(); + let process_id = transaction.process.id(); + let manager = RunnerProcessManager { + broker, + started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerState { + transactions: Vec::new(), + associations: Vec::new(), + active_instances: 1, + }), + drained: Condvar::new(), + }; + + assert!( + manager + .register_association(process_id, Arc::new(|| {}), true) + .is_err() + ); + assert!(!transaction.process.is_running()); transaction.process.cleanup(true); } } diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index e7a25dd918..ba1d441d53 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -293,6 +293,7 @@ where notification_channel, shutdown, process_manager, + is_started_process, !is_started_process && process_out.is_none(), panicked_out, abnormal_out, @@ -484,6 +485,7 @@ fn dispatch_requests>, + is_started_process: bool, finish_process: bool, panicked_out: Option<&AtomicBool>, abnormal_out: Option<&AtomicBool>, @@ -506,11 +508,22 @@ where "process association terminated by process-start control", )); }); - if let Err(error) = - process_manager.register_association(association.process_id(), association_failure) - { - failure_coordinator.report(error); - } + process_manager.register_association( + association.process_id(), + association_failure, + is_started_process, + )?; + } else if is_started_process { + return Err(IoError::other( + "started process association requires a process manager", + )); + } + if !is_started_process { + association.activate_process().map_err(|error| { + IoError::other(format!( + "failed to activate broker process association: {error}" + )) + })?; } let (request_sender, request_receiver) = sync_channel(REQUEST_QUEUE_CAPACITY); let request_receiver = Arc::new(Mutex::new(request_receiver)); @@ -849,6 +862,7 @@ mod tests { .send_handshake_response(&BrokerHandshakeResponse::Negotiated { broker_protocol_version: BROKER_PROTOCOL_VERSION, process_id: litebox_broker_protocol::ProcessId(1), + initial_thread_id: litebox_broker_protocol::ThreadId(2), startup: None, }) .unwrap(); @@ -980,6 +994,7 @@ mod tests { notifications, shutdown, None, + false, true, None, None, diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 46e69a538f..7ef274b1ff 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -9,8 +9,7 @@ use std::process::{Child, Command}; use std::sync::Arc; use std::time::{Duration, Instant}; -use litebox_broker_local::{BrokerLocal, BrokerLocalError}; -use litebox_broker_protocol::error::ErrorCode; +use litebox_broker_local::BrokerLocal; use litebox_broker_protocol::process::{ InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessStartupData, }; @@ -236,8 +235,8 @@ fn run_fake_runner(args: &[OsString]) { let bootstrap = marker.as_os_str().as_encoded_bytes(); let inherited_event = local.create_event_with_count(1).unwrap(); let inherited_objects = InheritedProcessObjects::new(&[inherited_event]).unwrap(); - assert!(matches!( - local.request_process_start( + let failed = local + .request_process_start( FAILING_BOOTSTRAP_FORMAT, ProcessBootstrapVersion(1), SharedBufferSequence::new( @@ -247,9 +246,9 @@ fn run_fake_runner(args: &[OsString]) { .unwrap(), bootstrap, inherited_objects, - ), - Err(BrokerLocalError::Broker(ErrorCode::UnsupportedOperation)) - )); + ) + .unwrap(); + assert_ne!(failed.process_id.0, failed.initial_thread_id.0); let started = local .request_process_start( TEST_BOOTSTRAP_FORMAT, @@ -263,7 +262,7 @@ fn run_fake_runner(args: &[OsString]) { inherited_objects, ) .unwrap(); - assert_eq!(started.initial_thread_id, None); + assert_ne!(started.process_id.0, started.initial_thread_id.0); wait_for_marker( marker, &format!("ready:{}\nstarted\nfinished\n", started.process_id.0), @@ -342,9 +341,6 @@ fn run_fake_child( bootstrap: ProcessStartupData, ) { if bootstrap.format == FAILING_BOOTSTRAP_FORMAT { - local - .report_process_start_failure(ErrorCode::UnsupportedOperation) - .unwrap(); return; } assert_eq!(bootstrap.format, TEST_BOOTSTRAP_FORMAT); @@ -357,11 +353,6 @@ fn run_fake_child( ReadinessFlags::READ | ReadinessFlags::WRITE ); std::fs::write(marker, format!("ready:{}\n", local.process_id().0)).unwrap(); - match local.process_ready(None) { - Ok(()) => {} - Err(BrokerLocalError::Broker(ErrorCode::PeerClosed)) => return, - Err(error) => panic!("child readiness failed: {error}"), - } std::fs::OpenOptions::new() .append(true) .open(marker) diff --git a/litebox_runner_linux_on_windows_userland/src/lib.rs b/litebox_runner_linux_on_windows_userland/src/lib.rs index 19ad8cecd4..f1ba2a0159 100644 --- a/litebox_runner_linux_on_windows_userland/src/lib.rs +++ b/litebox_runner_linux_on_windows_userland/src/lib.rs @@ -68,13 +68,23 @@ pub fn run(cli_args: CliArgs) -> Result<()> { .broker_control_channel .as_deref() .context("file operations require --broker-control-channel")?; + let (connection, startup) = broker::connect(control_pipe)?; + if let Some(bootstrap) = startup { + anyhow::bail!( + "unsupported child Linux process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + ); + } let broker::BrokerConnection { local, notifications, - } = broker::connect(control_pipe)?; + } = connection; let process_id = i32::try_from(local.process_id().0).context("process ID does not fit Linux pid_t")?; + let initial_thread_id = local.initial_thread_id(); let litebox = litebox::LiteBox::new_with_broker_local(platform, local); + let initial_thread = litebox.adopt_thread(initial_thread_id)?; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), @@ -109,7 +119,7 @@ pub fn run(cli_args: CliArgs) -> Result<()> { }; let program = shim - .load_program( + .load_program_with_initial_thread( litebox_common_linux::TaskParams { pid: process_id, ppid: 0, @@ -118,6 +128,7 @@ pub fn run(cli_args: CliArgs) -> Result<()> { euid: 1000, egid: 1000, }, + initial_thread, prog_path, argv, envp, diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 99d53e7851..11b3382bd3 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -3,7 +3,6 @@ use anyhow::{Context as _, Result, anyhow}; use clap::Parser; -use litebox_broker_protocol::error::ErrorCode; use litebox_platform_linux_userland::LinuxUserland as Platform; use std::path::PathBuf; @@ -96,9 +95,6 @@ pub fn run(cli_args: CliArgs) -> Result { broker::connect(control_socket_path) })?; if let Some(bootstrap) = startup { - connection - .local - .report_process_start_failure(ErrorCode::UnsupportedOperation)?; return Err(anyhow!( "unsupported child Linux process bootstrap format {:?} version {:?}", bootstrap.format, @@ -128,9 +124,11 @@ pub fn run(cli_args: CliArgs) -> Result { } = connection; let process_id = i32::try_from(broker_local.process_id().0) .context("process ID does not fit Linux pid_t")?; + let initial_thread_id = broker_local.initial_thread_id(); broker_positional_io_fds.extend(positional_io_fds); broker_shutdown_fds.push(shutdown_fd); let litebox = litebox::LiteBox::new_with_broker_local(platform, broker_local); + let initial_thread = litebox.adopt_thread(initial_thread_id)?; broker_association_coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( @@ -173,7 +171,8 @@ pub fn run(cli_args: CliArgs) -> Result { &broker_shutdown_fds, ); - let program = shim.load_program(task_params, prog_path, argv, envp)?; + let program = + shim.load_program_with_initial_thread(task_params, initial_thread, prog_path, argv, envp)?; #[cfg(feature = "lock_tracing")] litebox::sync::start_recording(); diff --git a/litebox_runner_optee_on_linux_userland/src/lib.rs b/litebox_runner_optee_on_linux_userland/src/lib.rs index a917b06fe5..1d75ab40ee 100644 --- a/litebox_runner_optee_on_linux_userland/src/lib.rs +++ b/litebox_runner_optee_on_linux_userland/src/lib.rs @@ -95,12 +95,20 @@ pub fn run(cli_args: CliArgs) -> Result<()> { // Leaked because the shim requires a `'static` session manager. let session_manager: &'static SessionManager = Box::leak(Box::new(SessionManager::new())); + let (connection, startup) = broker::connect(&cli_args.broker_control_channel)?; + if let Some(bootstrap) = startup { + return Err(anyhow::anyhow!( + "unsupported child OP-TEE process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } let broker::BrokerConnection { local, notifications, coordinator, .. - } = broker::connect(&cli_args.broker_control_channel)?; + } = connection; let litebox = litebox::LiteBox::new_with_broker_local(platform, local); coordinator.install_dispatch(litebox.broker_failure_dispatcher()); broker::start_notification_receiver( diff --git a/litebox_runner_windows_on_linux_userland/src/lib.rs b/litebox_runner_windows_on_linux_userland/src/lib.rs index 612f14f323..e730b4be6f 100644 --- a/litebox_runner_windows_on_linux_userland/src/lib.rs +++ b/litebox_runner_windows_on_linux_userland/src/lib.rs @@ -80,18 +80,28 @@ pub fn run(cli_args: CliArgs) -> Result<()> { .broker_control_channel .as_deref() .context("file operations require --broker-control-channel")?; + let (connection, startup) = + litebox_platform_linux_userland::with_guest_signals_blocked(|| { + broker::connect(control_socket) + })?; + if let Some(bootstrap) = startup { + return Err(anyhow::anyhow!( + "unsupported child Windows process bootstrap format {:?} version {:?}", + bootstrap.format, + bootstrap.version + )); + } let broker::BrokerConnection { local, notifications, coordinator, positional_io_fds: _broker_positional_io_fds, shutdown_fd: _broker_shutdown_fd, - } = litebox_platform_linux_userland::with_guest_signals_blocked(|| { - broker::connect(control_socket) - })?; + } = connection; let process_id = local.process_id().0 as usize; + let initial_thread_id = local.initial_thread_id(); let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.create_thread()?; + let initial_thread = litebox.adopt_thread(initial_thread_id)?; coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 9d5a0f8370..5f74a6225d 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -10,7 +10,6 @@ extern crate alloc; use anyhow::{Context as _, Result}; use clap::Parser; use litebox_broker_local_userland as broker; -use litebox_broker_protocol::error::ErrorCode; use litebox_platform_windows_userland::{GuestTlsMode, WindowsUserland}; /// Runs a Windows PE program with LiteBox on unmodified Windows and returns its exit code. @@ -61,9 +60,6 @@ pub fn run(cli_args: CliArgs) -> Result { .context("file operations require --broker-control-channel")?; let (connection, startup) = broker::connect(control_pipe)?; if let Some(bootstrap) = startup { - connection - .local - .report_process_start_failure(ErrorCode::UnsupportedOperation)?; anyhow::bail!( "unsupported child Windows process bootstrap format {:?} version {:?}", bootstrap.format, @@ -78,8 +74,9 @@ pub fn run(cli_args: CliArgs) -> Result { notifications, } = connection; let process_id = local.process_id().0 as usize; + let initial_thread_id = local.initial_thread_id(); let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.create_thread()?; + let initial_thread = litebox.adopt_thread(initial_thread_id)?; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), diff --git a/litebox_shim_linux/src/lib.rs b/litebox_shim_linux/src/lib.rs index 5534b60331..d82868de8a 100644 --- a/litebox_shim_linux/src/lib.rs +++ b/litebox_shim_linux/src/lib.rs @@ -266,6 +266,29 @@ impl LinuxShim { path: &str, argv: Vec, envp: Vec, + ) -> Result, loader::elf::ElfLoaderError> { + self.load_program_inner(task, None, path, argv, envp) + } + + /// Loads a program whose initial thread was allocated during broker negotiation. + pub fn load_program_with_initial_thread( + &self, + task: litebox_common_linux::TaskParams, + initial_thread: litebox::thread::Thread, + path: &str, + argv: Vec, + envp: Vec, + ) -> Result, loader::elf::ElfLoaderError> { + self.load_program_inner(task, Some(initial_thread), path, argv, envp) + } + + fn load_program_inner( + &self, + task: litebox_common_linux::TaskParams, + initial_thread: Option, + path: &str, + argv: Vec, + envp: Vec, ) -> Result, loader::elf::ElfLoaderError> { let litebox_common_linux::TaskParams { pid, @@ -295,7 +318,7 @@ impl LinuxShim { _not_send: core::marker::PhantomData, task: Task { global: self.0.clone(), - litebox_thread: Cell::new(None), + litebox_thread: Cell::new(initial_thread), thread: syscalls::process::ThreadState::new_process(pid), wait_state: wait::WaitState::new(self.0.platform), pid, From 00745bb82394b49f9e59caf5c98182986f1cbc57 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 18:59:10 -0700 Subject: [PATCH 15/21] Unify runner process creation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox/src/litebox.rs | 45 +++++- litebox/src/thread.rs | 13 +- litebox_broker_core/src/lib.rs | 34 +---- litebox_broker_core/src/process.rs | 139 ++++++++++-------- litebox_broker_host/src/lib.rs | 2 +- .../src/socket/tests/udp.rs | 2 +- litebox_broker_userland/src/runner/linux.rs | 6 +- .../src/runner/process_manager.rs | 29 ++-- litebox_broker_userland/src/runner/windows.rs | 6 +- litebox_broker_userland/src/runtime.rs | 18 +-- .../src/lib.rs | 8 +- litebox_runner_linux_userland/src/lib.rs | 8 +- .../src/lib.rs | 7 +- litebox_runner_windows_userland/src/lib.rs | 7 +- 14 files changed, 170 insertions(+), 154 deletions(-) diff --git a/litebox/src/litebox.rs b/litebox/src/litebox.rs index 6007e891af..9157db8e03 100644 --- a/litebox/src/litebox.rs +++ b/litebox/src/litebox.rs @@ -6,6 +6,7 @@ use alloc::sync::Arc; use litebox_broker_local::BrokerLocal; +use litebox_broker_protocol::ProcessId; use litebox_broker_protocol::message::BrokerNotification; use litebox_broker_transport::channel::LocalCallChannel; use litebox_platform::time::TimeProvider; @@ -14,6 +15,7 @@ use crate::{ broker, fd::Descriptors, sync::{RawSyncPrimitivesProvider, RwLock}, + thread::Thread, }; /// A full LiteBox system. @@ -48,16 +50,49 @@ impl LiteBox { platform: &'static Platform, broker_local: BrokerLocal, ) -> Self + where + Platform: TimeProvider, + Channel: LocalCallChannel + Send + Sync + 'static, + { + Self::new_with_broker_local_inner(platform, broker_local).0 + } + + fn new_with_broker_local_inner( + platform: &'static Platform, + broker_local: BrokerLocal, + ) -> (Self, Arc) where Platform: TimeProvider, Channel: LocalCallChannel + Send + Sync + 'static, { let broker_pollables = Arc::new(broker::BrokerPollableRegistry::new()); - let broker_control = Arc::new(broker::BrokerLocalControl::::new( - broker_local, - Arc::clone(&broker_pollables), - )); - Self::new_inner(platform, Some(broker_control), broker_pollables) + let broker_control: Arc = + Arc::new(broker::BrokerLocalControl::::new( + broker_local, + Arc::clone(&broker_pollables), + )); + let litebox = Self::new_inner( + platform, + Some(Arc::clone(&broker_control)), + broker_pollables, + ); + (litebox, broker_control) + } + + /// Creates a broker-backed process and its negotiated initial thread. + pub fn new_process_with_broker_local( + platform: &'static Platform, + broker_local: BrokerLocal, + ) -> (Self, ProcessId, Thread) + where + Platform: TimeProvider, + Channel: LocalCallChannel + Send + Sync + 'static, + { + let process_id = broker_local.process_id(); + let initial_thread_id = broker_local.initial_thread_id(); + let (litebox, broker) = Self::new_with_broker_local_inner(platform, broker_local); + let initial_thread = Thread::from_broker(initial_thread_id, broker); + (litebox, process_id, initial_thread) } fn new_inner( diff --git a/litebox/src/thread.rs b/litebox/src/thread.rs index d886959d13..bcd35d1790 100644 --- a/litebox/src/thread.rs +++ b/litebox/src/thread.rs @@ -50,6 +50,10 @@ pub struct Thread { } impl Thread { + pub(crate) fn from_broker(id: ThreadId, broker: Arc) -> Self { + Self { id, broker } + } + /// Returns the assigned thread ID. #[must_use] pub const fn id(&self) -> u32 { @@ -68,15 +72,6 @@ impl Thread { } impl LiteBox { - /// Adopts a broker thread allocated during process negotiation. - /// - /// The caller must pass the initial thread ID from the negotiated broker - /// association exactly once. - pub fn adopt_thread(&self, id: ThreadId) -> Result { - let broker = self.broker_control().ok_or(CreateError::Unavailable)?; - Ok(Thread { id, broker }) - } - /// Creates a thread belonging to this LiteBox process. /// /// # Panics diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 6d1d106226..3298a510cd 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -301,20 +301,7 @@ impl BrokerCore { Ok((first, second)) } - /// Allocates and registers one authenticated broker process. - /// - /// # Panics - /// - /// Panics if the shared ID allocator violates its range or uniqueness - /// invariants. - pub fn create_process( - &self, - caller_credential: CallerCredential, - ) -> Result> { - self.create_process_with_parent(None, caller_credential, process::ProcessState::Running) - } - - /// Allocates one authenticated broker process awaiting association activation. + /// Allocates one authenticated process awaiting association activation. /// /// The deployment must call [`BrokerProcess::complete_start`] after the /// process association becomes active. @@ -323,18 +310,9 @@ impl BrokerCore { /// /// Panics if the shared ID allocator violates its range or uniqueness /// invariants. - pub fn create_attaching_process( - &self, - caller_credential: CallerCredential, - ) -> Result> { - self.create_process_with_parent(None, caller_credential, process::ProcessState::Attaching) - } - - fn create_process_with_parent( + pub fn create_process( &self, - parent_id: Option, caller_credential: CallerCredential, - state: process::ProcessState, ) -> Result> { let mut processes = self.processes.write(); if processes.len() >= self.limits.max_processes { @@ -345,13 +323,7 @@ impl BrokerCore { .map_err(|_| BrokerError::OutOfMemory)?; let raw_id = self.ids.lock().allocate()?; let id = ProcessId(raw_id); - let process = Arc::new(BrokerProcess::new( - self.clone(), - id, - parent_id, - caller_credential, - state, - )); + let process = Arc::new(BrokerProcess::new(self.clone(), id, caller_credential)); assert!( processes.insert(id, Arc::downgrade(&process)).is_none(), "the ID allocator returned an occupied process ID" diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index cafad327c2..e076630d55 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -107,8 +107,6 @@ pub struct BrokerProcess { pub(crate) core: BrokerCore, /// Assigned process ID and internal authority. pub(crate) id: ProcessId, - /// Authoritative parent process ID, absent for a root process. - parent_id: Option, state: Mutex, /// Broker-entry-authenticated caller credential for this process. pub(crate) caller_credential: CallerCredential, @@ -136,15 +134,12 @@ impl BrokerProcess { pub(crate) fn new( core: BrokerCore, id: ProcessId, - parent_id: Option, caller_credential: CallerCredential, - state: ProcessState, ) -> Self { Self { core, id, - parent_id, - state: Mutex::new(state), + state: Mutex::new(ProcessState::Attaching), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -163,12 +158,6 @@ impl BrokerProcess { self.id } - /// Returns the current parent process ID, if any. - #[must_use] - pub const fn parent_id(&self) -> Option { - self.parent_id - } - /// Returns the credential authenticated for this process association. #[must_use] pub const fn caller_credential(&self) -> CallerCredential { @@ -194,39 +183,34 @@ impl BrokerProcess { } } - /// Creates one child process inheriting this process's authenticated credential. + /// Duplicates object references into another process while preserving rights. /// - /// Returned handles follow the requested inheritance order. If later host - /// launch or association setup fails normally, the caller must finish the - /// returned process; dropping it preserves its IDs as unwind protection. - pub fn create_child( + /// Returned handles follow the requested source order. If duplication + /// fails, references already created by this call are removed from the + /// target before the error is returned. + pub fn duplicate_object_references_to_preserving_rights( &self, - inherited_objects: &[ObjectHandle], - ) -> Result<(Arc, Vec)> { - let process = self.core.create_process_with_parent( - Some(self.id), - self.caller_credential, - ProcessState::Attaching, - )?; - let inherited_result = (|| { - let mut child_handles = Vec::new(); - child_handles - .try_reserve_exact(inherited_objects.len()) - .map_err(|_| BrokerError::OutOfMemory)?; - for handle in inherited_objects { - let child_handle = - self.duplicate_object_reference_to_preserving_rights(*handle, &process)?; - child_handles.push(child_handle); - } - Ok(child_handles) - })(); - match inherited_result { - Ok(child_handles) => Ok((process, child_handles)), - Err(error) => { - process.cleanup(true); - Err(error) + handles: &[ObjectHandle], + target: &BrokerProcess, + ) -> Result> { + let mut duplicates = Vec::new(); + duplicates + .try_reserve_exact(handles.len()) + .map_err(|_| BrokerError::OutOfMemory)?; + for handle in handles { + match self.duplicate_object_reference_to_preserving_rights(*handle, target) { + Ok(duplicate) => duplicates.push(duplicate), + Err(error) => { + for duplicate in duplicates.drain(..).rev() { + if target.close_object_reference(duplicate).is_err() { + return Err(BrokerError::Internal); + } + } + return Err(error); + } } } + Ok(duplicates) } /// Creates a broker thread belonging to this process. @@ -903,10 +887,8 @@ mod tests { .unwrap(); assert_eq!(first.id().0, 1); - assert_eq!(first.parent_id(), None); assert_eq!(thread.0, 2); assert_eq!(second.id().0, 3); - assert_eq!(second.parent_id(), None); } #[test] @@ -931,7 +913,7 @@ mod tests { } #[test] - fn child_is_parented_and_requires_start_completion() { + fn process_creation_and_reference_inheritance_are_separate() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -941,26 +923,55 @@ mod tests { .create_process(CallerCredential::Unauthenticated) .unwrap(); let source_handle = crate::event::create(&parent, 1).unwrap(); - let (child, inherited_objects) = parent.create_child(&[source_handle]).unwrap(); - let initial_thread_id = child.create_thread().unwrap(); + let process = broker.create_process(parent.caller_credential()).unwrap(); + let inherited_objects = parent + .duplicate_object_references_to_preserving_rights(&[source_handle], &process) + .unwrap(); + let initial_thread_id = process.create_thread().unwrap(); let inherited_handle = inherited_objects[0]; - assert_eq!(child.parent_id(), Some(parent.id())); - assert!(child.owns_thread(initial_thread_id)); + assert!(process.owns_thread(initial_thread_id)); assert_ne!(inherited_handle, source_handle); assert_eq!( - child.check_readiness(inherited_handle).unwrap(), + process.check_readiness(inherited_handle).unwrap(), ReadinessFlags::READ | ReadinessFlags::WRITE ); - assert!(!child.is_running()); + assert!(!process.is_running()); - child.complete_start().unwrap(); - assert!(child.is_running()); - assert_eq!(*child.state.lock(), ProcessState::Running); + process.complete_start().unwrap(); + assert!(process.is_running()); + assert_eq!(*process.state.lock(), ProcessState::Running); } #[test] - fn finished_attaching_child_releases_process_capacity() { + fn failed_reference_inheritance_rolls_back_target_references() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let source = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let target = broker.create_process(source.caller_credential()).unwrap(); + let source_handle = crate::event::create(&source, 1).unwrap(); + + assert_eq!( + source.duplicate_object_references_to_preserving_rights( + &[source_handle, ObjectHandle(u64::MAX)], + &target, + ), + Err(BrokerError::UnknownObject) + ); + assert!(target.references.lock().handles.is_empty()); + assert_eq!( + source.check_readiness(source_handle).unwrap(), + ReadinessFlags::READ | ReadinessFlags::WRITE + ); + } + + #[test] + fn finished_attaching_process_releases_process_capacity() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -970,18 +981,18 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let (child, _) = parent.create_child(&[]).unwrap(); + let process = broker.create_process(parent.caller_credential()).unwrap(); assert_eq!( - parent.create_child(&[]).err(), + broker.create_process(parent.caller_credential()).err(), Some(BrokerError::ResourceExhausted) ); - child.cleanup(true); - assert!(parent.create_child(&[]).is_ok()); + process.cleanup(true); + assert!(broker.create_process(parent.caller_credential()).is_ok()); } #[test] - fn child_cannot_complete_start_after_teardown() { + fn process_cannot_complete_start_after_teardown() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) @@ -991,13 +1002,13 @@ mod tests { let parent = broker .create_process(CallerCredential::Unauthenticated) .unwrap(); - let (child, _) = parent.create_child(&[]).unwrap(); + let process = broker.create_process(parent.caller_credential()).unwrap(); - child.cleanup(true); - child.cleanup(true); + process.cleanup(true); + process.cleanup(true); - assert_eq!(child.complete_start(), Err(BrokerError::PeerClosed)); - let (replacement, _) = parent.create_child(&[]).unwrap(); + assert_eq!(process.complete_start(), Err(BrokerError::PeerClosed)); + let replacement = broker.create_process(parent.caller_credential()).unwrap(); replacement.cleanup(true); } diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index d23b8a25fd..118e742dc4 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -330,7 +330,7 @@ where .map_err(BrokerHostError::Channel)?; return Ok(Err(ConnectionTermination::Rejected(error))); } - None => match core.create_attaching_process(caller_credential) { + None => match core.create_process(caller_credential) { Ok(process) => match process.create_thread() { Ok(initial_thread_id) => (process, initial_thread_id), Err( diff --git a/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs b/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs index e4eec9c3c3..4feb47c852 100644 --- a/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs +++ b/litebox_broker_platform_linux_userland/src/socket/tests/udp.rs @@ -1903,7 +1903,7 @@ fn udp_queued_datagrams_survive_source_process_teardown() { second_receiver, ReadinessFlags::READ, ); - source_process.finish(); + source_process.cleanup(true); assert_eq!(retirements.recv_timeout(TEST_TIMEOUT).unwrap(), source); assert_eq!(provider.reactor.udp_queued_datagram_count(), 1); assert!( diff --git a/litebox_broker_userland/src/runner/linux.rs b/litebox_broker_userland/src/runner/linux.rs index dec0a59cae..7be8911ec7 100644 --- a/litebox_broker_userland/src/runner/linux.rs +++ b/litebox_broker_userland/src/runner/linux.rs @@ -73,13 +73,13 @@ fn serve_association( startup: Option, process_manager: Arc, ) -> AssociationRunResult { - let is_started_process = startup.is_some(); + let has_parent_transaction = startup.is_some(); let (control_channel, setup_deadline) = match accept_control_channel(control_listener, runner) { Ok(connection) => connection, Err(error) => { let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit - } else if is_started_process + } else if has_parent_transaction && matches!( error.kind(), std::io::ErrorKind::BrokenPipe @@ -114,7 +114,7 @@ fn serve_association( UnixStreamHostSetupChannel::into_active, process_manager, ); - if is_started_process + if has_parent_transaction && result.failure_cause == AssociationFailureCause::Other && result .result diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index a3e15f78b3..01d34865e8 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -163,9 +163,20 @@ impl RunnerProcessManager { if !parent.is_running() { return Err(ErrorCode::ProtocolState); } - let (process, inherited_objects) = parent - .create_child(requested_inherited_objects.as_slice()) + let process = self + .broker + .create_process(parent.caller_credential()) .map_err(ErrorCode::from)?; + let inherited_objects = match parent.duplicate_object_references_to_preserving_rights( + requested_inherited_objects.as_slice(), + &process, + ) { + Ok(inherited_objects) => inherited_objects, + Err(error) => { + process.cleanup(true); + return Err(ErrorCode::from(error)); + } + }; let initial_thread_id = match process.create_thread() { Ok(initial_thread_id) => initial_thread_id, Err(error) => { @@ -324,7 +335,7 @@ impl RunnerProcessManager { &self, process_id: ProcessId, failure: AssociationFailure, - is_started_process: bool, + has_parent_transaction: bool, ) -> IoResult<()> { let transaction = { let mut state = self @@ -351,12 +362,10 @@ impl RunnerProcessManager { .find(|transaction| transaction.process.id() == process_id) .cloned() }; - if is_started_process { + if has_parent_transaction { let Some(transaction) = transaction else { self.unregister_association(process_id); - return Err(IoError::other( - "started process has no pending start transaction", - )); + return Err(IoError::other("process has no pending parent transaction")); }; transaction.install_association_failure(failure); if let Err(error) = transaction.activate() { @@ -368,7 +377,7 @@ impl RunnerProcessManager { } else if transaction.is_some() { self.unregister_association(process_id); return Err(IoError::other( - "root process unexpectedly has a pending start transaction", + "process without a parent transaction matched a pending transaction", )); } Ok(()) @@ -699,7 +708,7 @@ mod tests { .create_process(CallerCredential::Unauthenticated) .unwrap(); let parent_id = parent.id(); - let (process, _) = parent.create_child(&[]).unwrap(); + let process = broker.create_process(parent.caller_credential()).unwrap(); let initial_thread_id = process.create_thread().unwrap(); parent.cleanup(true); ( @@ -827,7 +836,7 @@ mod tests { } #[test] - fn late_started_association_is_not_activated_as_a_root() { + fn late_parented_association_requires_its_transaction() { let (broker, transaction) = starting_transaction(); let process_id = transaction.process.id(); let manager = RunnerProcessManager { diff --git a/litebox_broker_userland/src/runner/windows.rs b/litebox_broker_userland/src/runner/windows.rs index 19c7fee23f..da62d149e0 100644 --- a/litebox_broker_userland/src/runner/windows.rs +++ b/litebox_broker_userland/src/runner/windows.rs @@ -65,14 +65,14 @@ fn serve_association( startup: Option, process_manager: Arc, ) -> AssociationRunResult { - let is_started_process = startup.is_some(); + let has_parent_transaction = startup.is_some(); let (control_channel, _setup_deadline) = match accept_control_channel(control_listener, runner) { Ok(connection) => connection, Err(error) => { let failure_cause = if runner_has_exited(runner).unwrap_or(false) { AssociationFailureCause::RunnerExit - } else if is_started_process + } else if has_parent_transaction && matches!( error.kind(), std::io::ErrorKind::BrokenPipe @@ -110,7 +110,7 @@ fn serve_association( WindowsNamedPipeHostSetupChannel::into_active, process_manager, ); - if is_started_process + if has_parent_transaction && result.failure_cause == AssociationFailureCause::Other && result .result diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index ba1d441d53..40608d8753 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -222,7 +222,7 @@ where NotificationChannel: HostNotificationChannel + Send, Shutdown: HostAssociationShutdown + Send + Sync + 'static, { - let is_started_process = startup.is_some(); + let has_parent_transaction = startup.is_some(); let (process, startup) = match startup { Some(startup) => { let (process, data) = startup.into_process_and_data(); @@ -279,7 +279,7 @@ where match activate(control_channel, control_ring) { Ok(active) => active, Err(error) => { - if !is_started_process && process_out.is_none() { + if !has_parent_transaction && process_out.is_none() { association.finish(); } return Err(error); @@ -293,8 +293,8 @@ where notification_channel, shutdown, process_manager, - is_started_process, - !is_started_process && process_out.is_none(), + has_parent_transaction, + !has_parent_transaction && process_out.is_none(), panicked_out, abnormal_out, ) @@ -485,7 +485,7 @@ fn dispatch_requests>, - is_started_process: bool, + has_parent_transaction: bool, finish_process: bool, panicked_out: Option<&AtomicBool>, abnormal_out: Option<&AtomicBool>, @@ -511,14 +511,14 @@ where process_manager.register_association( association.process_id(), association_failure, - is_started_process, + has_parent_transaction, )?; - } else if is_started_process { + } else if has_parent_transaction { return Err(IoError::other( - "started process association requires a process manager", + "a process with a parent transaction requires a process manager", )); } - if !is_started_process { + if !has_parent_transaction { association.activate_process().map_err(|error| { IoError::other(format!( "failed to activate broker process association: {error}" diff --git a/litebox_runner_linux_on_windows_userland/src/lib.rs b/litebox_runner_linux_on_windows_userland/src/lib.rs index f1ba2a0159..071071b867 100644 --- a/litebox_runner_linux_on_windows_userland/src/lib.rs +++ b/litebox_runner_linux_on_windows_userland/src/lib.rs @@ -80,11 +80,9 @@ pub fn run(cli_args: CliArgs) -> Result<()> { local, notifications, } = connection; - let process_id = - i32::try_from(local.process_id().0).context("process ID does not fit Linux pid_t")?; - let initial_thread_id = local.initial_thread_id(); - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.adopt_thread(initial_thread_id)?; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = i32::try_from(process_id.0).context("process ID does not fit Linux pid_t")?; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), diff --git a/litebox_runner_linux_userland/src/lib.rs b/litebox_runner_linux_userland/src/lib.rs index 11b3382bd3..c2bf44d55a 100644 --- a/litebox_runner_linux_userland/src/lib.rs +++ b/litebox_runner_linux_userland/src/lib.rs @@ -122,13 +122,11 @@ pub fn run(cli_args: CliArgs) -> Result { positional_io_fds, shutdown_fd, } = connection; - let process_id = i32::try_from(broker_local.process_id().0) - .context("process ID does not fit Linux pid_t")?; - let initial_thread_id = broker_local.initial_thread_id(); broker_positional_io_fds.extend(positional_io_fds); broker_shutdown_fds.push(shutdown_fd); - let litebox = litebox::LiteBox::new_with_broker_local(platform, broker_local); - let initial_thread = litebox.adopt_thread(initial_thread_id)?; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, broker_local); + let process_id = i32::try_from(process_id.0).context("process ID does not fit Linux pid_t")?; broker_association_coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( diff --git a/litebox_runner_windows_on_linux_userland/src/lib.rs b/litebox_runner_windows_on_linux_userland/src/lib.rs index e730b4be6f..3c51818765 100644 --- a/litebox_runner_windows_on_linux_userland/src/lib.rs +++ b/litebox_runner_windows_on_linux_userland/src/lib.rs @@ -98,10 +98,9 @@ pub fn run(cli_args: CliArgs) -> Result<()> { positional_io_fds: _broker_positional_io_fds, shutdown_fd: _broker_shutdown_fd, } = connection; - let process_id = local.process_id().0 as usize; - let initial_thread_id = local.initial_thread_id(); - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.adopt_thread(initial_thread_id)?; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = process_id.0 as usize; coordinator.install_dispatch(litebox.broker_failure_dispatcher()); litebox_platform_linux_userland::with_guest_signals_blocked(|| { broker::start_notification_receiver( diff --git a/litebox_runner_windows_userland/src/lib.rs b/litebox_runner_windows_userland/src/lib.rs index 5f74a6225d..e9c72b762a 100644 --- a/litebox_runner_windows_userland/src/lib.rs +++ b/litebox_runner_windows_userland/src/lib.rs @@ -73,10 +73,9 @@ pub fn run(cli_args: CliArgs) -> Result { local, notifications, } = connection; - let process_id = local.process_id().0 as usize; - let initial_thread_id = local.initial_thread_id(); - let litebox = litebox::LiteBox::new_with_broker_local(platform, local); - let initial_thread = litebox.adopt_thread(initial_thread_id)?; + let (litebox, process_id, initial_thread) = + litebox::LiteBox::new_process_with_broker_local(platform, local); + let process_id = process_id.0 as usize; broker::start_notification_receiver( notifications, litebox.broker_notification_dispatcher(), From e69d190f86642e7c6f81fe0aacc91e3580683f83 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 19:25:12 -0700 Subject: [PATCH 16/21] Unify process startup activation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_core/src/lib.rs | 3 - litebox_broker_core/src/process.rs | 53 +++++------ .../src/runner/process_manager.rs | 87 +++++++++++++++---- litebox_broker_userland/src/runtime.rs | 4 +- 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/litebox_broker_core/src/lib.rs b/litebox_broker_core/src/lib.rs index 3298a510cd..0659eecf72 100644 --- a/litebox_broker_core/src/lib.rs +++ b/litebox_broker_core/src/lib.rs @@ -303,9 +303,6 @@ impl BrokerCore { /// Allocates one authenticated process awaiting association activation. /// - /// The deployment must call [`BrokerProcess::complete_start`] after the - /// process association becomes active. - /// /// # Panics /// /// Panics if the shared ID allocator violates its range or uniqueness diff --git a/litebox_broker_core/src/process.rs b/litebox_broker_core/src/process.rs index e076630d55..15b6ab970e 100644 --- a/litebox_broker_core/src/process.rs +++ b/litebox_broker_core/src/process.rs @@ -124,7 +124,7 @@ pub struct BrokerProcess { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ProcessState { - Attaching, + Starting, Running, Exiting, } @@ -139,7 +139,7 @@ impl BrokerProcess { Self { core, id, - state: Mutex::new(ProcessState::Attaching), + state: Mutex::new(ProcessState::Starting), caller_credential, references: Mutex::new(ProcessReferences { handles: Vec::new(), @@ -174,7 +174,7 @@ impl BrokerProcess { pub fn complete_start(&self) -> Result<()> { let mut state = self.state.lock(); match *state { - ProcessState::Attaching => { + ProcessState::Starting => { *state = ProcessState::Running; Ok(()) } @@ -183,12 +183,13 @@ impl BrokerProcess { } } - /// Duplicates object references into another process while preserving rights. + /// Duplicates object references into another process. /// - /// Returned handles follow the requested source order. If duplication - /// fails, references already created by this call are removed from the - /// target before the error is returned. - pub fn duplicate_object_references_to_preserving_rights( + /// Each duplicate retains its source reference's rights. Returned handles + /// follow the requested source order. If duplication fails, references + /// already created by this call are removed from the target before the + /// error is returned. + pub fn duplicate_object_references_to( &self, handles: &[ObjectHandle], target: &BrokerProcess, @@ -198,7 +199,10 @@ impl BrokerProcess { .try_reserve_exact(handles.len()) .map_err(|_| BrokerError::OutOfMemory)?; for handle in handles { - match self.duplicate_object_reference_to_preserving_rights(*handle, target) { + let duplicate = self + .object_reference_rights(*handle) + .and_then(|rights| self.duplicate_object_reference_to(*handle, target, rights)); + match duplicate { Ok(duplicate) => duplicates.push(duplicate), Err(error) => { for duplicate in duplicates.drain(..).rev() { @@ -394,20 +398,13 @@ impl BrokerProcess { target.create_object_reference_with_rights(object, rights) } - fn duplicate_object_reference_to_preserving_rights( - &self, - handle: ObjectHandle, - target: &BrokerProcess, - ) -> Result { - let rights = { - let references = self.core.references.read(); - let reference = references.get(&handle).ok_or(BrokerError::UnknownObject)?; - if reference.owner != self.id { - return Err(BrokerError::UnknownObject); - } - reference.rights - }; - self.duplicate_object_reference_to(handle, target, rights) + fn object_reference_rights(&self, handle: ObjectHandle) -> Result { + let references = self.core.references.read(); + let reference = references.get(&handle).ok_or(BrokerError::UnknownObject)?; + if reference.owner != self.id { + return Err(BrokerError::UnknownObject); + } + Ok(reference.rights) } pub(crate) fn create_object_reference_pair( @@ -925,7 +922,7 @@ mod tests { let source_handle = crate::event::create(&parent, 1).unwrap(); let process = broker.create_process(parent.caller_credential()).unwrap(); let inherited_objects = parent - .duplicate_object_references_to_preserving_rights(&[source_handle], &process) + .duplicate_object_references_to(&[source_handle], &process) .unwrap(); let initial_thread_id = process.create_thread().unwrap(); let inherited_handle = inherited_objects[0]; @@ -957,10 +954,8 @@ mod tests { let source_handle = crate::event::create(&source, 1).unwrap(); assert_eq!( - source.duplicate_object_references_to_preserving_rights( - &[source_handle, ObjectHandle(u64::MAX)], - &target, - ), + source + .duplicate_object_references_to(&[source_handle, ObjectHandle(u64::MAX)], &target,), Err(BrokerError::UnknownObject) ); assert!(target.references.lock().handles.is_empty()); @@ -971,7 +966,7 @@ mod tests { } #[test] - fn finished_attaching_process_releases_process_capacity() { + fn finished_starting_process_releases_process_capacity() { let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( ObjectRights::all(), )) diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index 01d34865e8..110a6ddfaa 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -167,10 +167,9 @@ impl RunnerProcessManager { .broker .create_process(parent.caller_credential()) .map_err(ErrorCode::from)?; - let inherited_objects = match parent.duplicate_object_references_to_preserving_rights( - requested_inherited_objects.as_slice(), - &process, - ) { + let inherited_objects = match parent + .duplicate_object_references_to(requested_inherited_objects.as_slice(), &process) + { Ok(inherited_objects) => inherited_objects, Err(error) => { process.cleanup(true); @@ -336,6 +335,7 @@ impl RunnerProcessManager { process_id: ProcessId, failure: AssociationFailure, has_parent_transaction: bool, + activate_process: impl FnOnce() -> Result<(), ErrorCode>, ) -> IoResult<()> { let transaction = { let mut state = self @@ -362,23 +362,26 @@ impl RunnerProcessManager { .find(|transaction| transaction.process.id() == process_id) .cloned() }; - if has_parent_transaction { + let activation = if has_parent_transaction { let Some(transaction) = transaction else { self.unregister_association(process_id); return Err(IoError::other("process has no pending parent transaction")); }; transaction.install_association_failure(failure); - if let Err(error) = transaction.activate() { - self.unregister_association(process_id); - return Err(IoError::other(format!( - "failed to activate broker process association: {error}" - ))); - } + transaction.activate(activate_process) } else if transaction.is_some() { self.unregister_association(process_id); return Err(IoError::other( "process without a parent transaction matched a pending transaction", )); + } else { + activate_process() + }; + if let Err(error) = activation { + self.unregister_association(process_id); + return Err(IoError::other(format!( + "failed to activate broker process association: {error}" + ))); } Ok(()) } @@ -512,7 +515,10 @@ impl ProcessStartTransaction { } } - fn activate(&self) -> Result<(), ErrorCode> { + fn activate( + &self, + activate_process: impl FnOnce() -> Result<(), ErrorCode>, + ) -> Result<(), ErrorCode> { let mut state = self .state .lock() @@ -522,7 +528,7 @@ impl ProcessStartTransaction { ProcessStartPhase::Failed(error) => return Err(error), ProcessStartPhase::Running => return Err(ErrorCode::ProtocolState), } - match self.process.complete_start().map_err(ErrorCode::from) { + match activate_process() { Ok(()) => { state.phase = ProcessStartPhase::Running; self.changed.notify_all(); @@ -738,7 +744,10 @@ mod tests { waiting.wait_until_running(Instant::now() + Duration::from_secs(1)) }); - transaction.activate().unwrap(); + let process = Arc::clone(&transaction.process); + transaction + .activate(move || process.complete_start().map_err(ErrorCode::from)) + .unwrap(); waiter.join().unwrap().unwrap(); assert!(transaction.process.is_running()); assert!( @@ -771,7 +780,10 @@ mod tests { transaction.abort(ErrorCode::PeerClosed, false, true); - assert_eq!(transaction.activate(), Err(ErrorCode::PeerClosed)); + assert_eq!( + transaction.activate(|| panic!("aborted transaction must not activate its process")), + Err(ErrorCode::PeerClosed) + ); assert_eq!( transaction.wait_until_running(Instant::now() + Duration::from_secs(1)), Err(ErrorCode::PeerClosed) @@ -827,14 +839,52 @@ mod tests { }), drained: Condvar::new(), }; + let process = Arc::clone(&transaction.process); manager - .register_association(process_id, Arc::new(|| {}), true) + .register_association(process_id, Arc::new(|| {}), true, move || { + process.complete_start().map_err(ErrorCode::from) + }) .unwrap(); assert!(transaction.process.is_running()); transaction.process.cleanup(true); } + #[test] + fn association_registration_activates_process_without_parent_transaction() { + let broker = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights( + ObjectRights::all(), + )) + .build() + .unwrap(); + let process = broker + .create_process(CallerCredential::Unauthenticated) + .unwrap(); + let process_id = process.id(); + let manager = RunnerProcessManager { + broker, + started_runner_config: RunnerConfig::new(PathBuf::new(), Vec::new()), + state: Mutex::new(RunnerProcessManagerState { + transactions: Vec::new(), + associations: Vec::new(), + active_instances: 0, + }), + drained: Condvar::new(), + }; + let process_for_activation = Arc::clone(&process); + + manager + .register_association(process_id, Arc::new(|| {}), false, move || { + process_for_activation + .complete_start() + .map_err(ErrorCode::from) + }) + .unwrap(); + + assert!(process.is_running()); + process.cleanup(true); + } + #[test] fn late_parented_association_requires_its_transaction() { let (broker, transaction) = starting_transaction(); @@ -849,10 +899,13 @@ mod tests { }), drained: Condvar::new(), }; + let process = Arc::clone(&transaction.process); assert!( manager - .register_association(process_id, Arc::new(|| {}), true) + .register_association(process_id, Arc::new(|| {}), true, move || { + process.complete_start().map_err(ErrorCode::from) + }) .is_err() ); assert!(!transaction.process.is_running()); diff --git a/litebox_broker_userland/src/runtime.rs b/litebox_broker_userland/src/runtime.rs index 40608d8753..974f128664 100644 --- a/litebox_broker_userland/src/runtime.rs +++ b/litebox_broker_userland/src/runtime.rs @@ -512,13 +512,13 @@ where association.process_id(), association_failure, has_parent_transaction, + || association.activate_process().map_err(ErrorCode::from), )?; } else if has_parent_transaction { return Err(IoError::other( "a process with a parent transaction requires a process manager", )); - } - if !has_parent_transaction { + } else { association.activate_process().map_err(|error| { IoError::other(format!( "failed to activate broker process association: {error}" From 9b190e578ad9c8d5cbbef637234e048305e5ff84 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Fri, 18 Sep 2026 19:27:09 -0700 Subject: [PATCH 17/21] Fix negotiated thread quota test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_shim_linux/src/syscalls/process.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litebox_shim_linux/src/syscalls/process.rs b/litebox_shim_linux/src/syscalls/process.rs index 3fd082bac6..cad5438716 100644 --- a/litebox_shim_linux/src/syscalls/process.rs +++ b/litebox_shim_linux/src/syscalls/process.rs @@ -1851,14 +1851,14 @@ mod tests { super::super::test_broker::MAX_TEST_BROKER_REFERENCES, BrokerCoreLimits::DEFAULT.max_total_pipe_capacity, ) - .with_thread_quotas(1, 1); + .with_thread_quotas(2, 2); let (litebox, process_id) = crate::syscalls::test_broker::litebox_with_limits(platform, limits); let shim_builder = crate::LinuxShimBuilder::new_with_litebox(platform, litebox, process_id); let leader = shim_builder.build().0.new_test_task(); let task = leader .clone_for_test() - .expect("the sole broker thread slot must be available"); + .expect("the thread slot after the negotiated initial thread must be available"); let old_tid = task.sys_gettid(); let process = task.process().clone(); From 0d730fb63c46126eefcbb00a63cd34abdc3d7185 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sat, 19 Sep 2026 07:17:31 -0700 Subject: [PATCH 18/21] Centralize shared buffer sequence copies Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_host/src/lib.rs | 62 +++++--------- litebox_broker_local/src/lib.rs | 37 ++------- litebox_broker_transport/src/shared_memory.rs | 82 ++++++++++++++++++- .../src/runner/process_manager.rs | 4 +- 4 files changed, 110 insertions(+), 75 deletions(-) diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 118e742dc4..79fd10b45c 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -63,7 +63,7 @@ use litebox_broker_protocol::stdio::{ }; use litebox_broker_protocol::{BROKER_PROTOCOL_VERSION, RequestId, ThreadId}; use litebox_broker_transport::channel::{HostReceive, HostSetupChannel, PeerCredential}; -use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; +use litebox_broker_transport::shared_memory::{SharedBufferError, SharedBufferPool, SharedMemory}; use spin::mutex::SpinMutex; mod error; @@ -597,7 +597,7 @@ fn handle_file_request( buffer, offset, }) => { - let data = copy_shared_buffer(shared_buffers, buffer, MAX_FILE_TRANSFER_SIZE)?; + let data = read_shared_buffer(shared_buffers, buffer, MAX_FILE_TRANSFER_SIZE)?; match litebox_broker_core::fs::write(process, handle, &data, offset) .map_err(RequestFailure::from)? { @@ -769,25 +769,17 @@ fn allocate_zeroed(length: u32) -> RequestResult> { Ok(data) } -/// Copies a validated operation-scoped shared-buffer sequence. -pub fn copy_shared_buffer( +/// Reads a validated operation-scoped shared-buffer sequence. +pub fn read_shared_buffer( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, max_length: u32, ) -> RequestResult> { validate_shared_buffer(buffer, max_length)?; let mut data = allocate_zeroed(buffer.length())?; - let mut offset = 0; - for descriptor in buffer - .descriptors(shared_buffers.layout()) - .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))? - { - let end = offset + descriptor.length as usize; - shared_buffers - .read(descriptor.slot_index, &mut data[offset..end]) - .map_err(|_| RequestFailure::Abort(ErrorCode::Internal))?; - offset = end; - } + shared_buffers + .read_sequence(buffer, &mut data) + .map_err(shared_buffer_access_failure)?; Ok(data) } @@ -798,35 +790,23 @@ fn write_shared_buffer( max_length: u32, ) -> RequestResult<()> { validate_shared_buffer(buffer, max_length)?; - if data.len() > buffer.length() as usize { - return Err(RequestFailure::Abort(ErrorCode::Internal)); - } - let mut offset = 0; - for descriptor in buffer - .descriptors(shared_buffers.layout()) - .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))? - { - if offset == data.len() { - break; - } - let length = (data.len() - offset).min(descriptor.length as usize); - let end = offset + length; - shared_buffers - .write(descriptor.slot_index, &data[offset..end]) - .map_err(|_| RequestFailure::Abort(ErrorCode::Internal))?; - offset = end; - } - if offset != data.len() { - return Err(RequestFailure::Abort(ErrorCode::Internal)); + shared_buffers + .write_sequence(buffer, data) + .map_err(shared_buffer_access_failure) +} + +fn shared_buffer_access_failure(error: SharedBufferError) -> RequestFailure { + match error { + SharedBufferError::Layout(_) => RequestFailure::Abort(ErrorCode::MalformedRequest), + _ => RequestFailure::Abort(ErrorCode::Internal), } - Ok(()) } fn read_file_path( shared_buffers: &SharedBufferPool, buffer: SharedBufferSequence, ) -> RequestResult { - let data = copy_shared_buffer(shared_buffers, buffer, SHARED_BUFFER_SLOT_SIZE)?; + let data = read_shared_buffer(shared_buffers, buffer, SHARED_BUFFER_SLOT_SIZE)?; let path = alloc::string::String::from_utf8(data) .map_err(|_| RequestFailure::Abort(ErrorCode::MalformedRequest))?; if !path.starts_with('/') { @@ -857,7 +837,7 @@ fn handle_stdio_request( })) } StdioRequest::Write(WriteStdioRequest { stream, buffer }) => { - let data = copy_shared_buffer(shared_buffers, buffer, MAX_STDIO_TRANSFER_SIZE)?; + let data = read_shared_buffer(shared_buffers, buffer, MAX_STDIO_TRANSFER_SIZE)?; let written = litebox_broker_core::stdio::write(process, stream, &data) .map_err(RequestFailure::from)?; Ok(StdioResponse::Write(WriteStdioResponse { @@ -948,7 +928,7 @@ fn handle_socket_request( return Err(RequestFailure::Abort(ErrorCode::MalformedRequest)); } let data = - copy_shared_buffer(shared_buffers, request.buffer, MAX_SOCKET_TRANSFER_SIZE)?; + read_shared_buffer(shared_buffers, request.buffer, MAX_SOCKET_TRANSFER_SIZE)?; match litebox_broker_core::socket::send(process, request.handle, data, request.flags) .map_err(RequestFailure::from)? { @@ -967,7 +947,7 @@ fn handle_socket_request( { return Err(RequestFailure::Abort(ErrorCode::MalformedRequest)); } - let data = copy_shared_buffer(shared_buffers, request.buffer, MAX_UDP_DATAGRAM_SIZE)?; + let data = read_shared_buffer(shared_buffers, request.buffer, MAX_UDP_DATAGRAM_SIZE)?; match litebox_broker_core::socket::send_to( process, request.handle, @@ -1144,7 +1124,7 @@ fn handle_pipe_request( })) } PipeRequest::Write(request) => { - let data = copy_shared_buffer(shared_buffers, request.buffer, MAX_PIPE_TRANSFER_SIZE)?; + let data = read_shared_buffer(shared_buffers, request.buffer, MAX_PIPE_TRANSFER_SIZE)?; litebox_broker_core::pipe::write(process, request.handle, &data) .map_err(RequestFailure::from) .and_then(|written| { diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 70f5db339f..3090b49479 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -273,17 +273,9 @@ impl BrokerLocal { buffer.length() as usize, "shared data must match its buffer sequence" ); - let mut offset = 0; - for descriptor in buffer - .descriptors(self.shared_buffers.layout()) - .expect("shared buffer sequence must identify valid slot ranges") - { - let end = offset + descriptor.length as usize; - self.shared_buffers - .write(descriptor.slot_index, &data[offset..end]) - .expect("validated shared buffer sequence must be accessible"); - offset = end; - } + self.shared_buffers + .write_sequence(buffer, data) + .expect("validated shared buffer sequence must be accessible"); } fn read_shared_buffer(&self, buffer: SharedBufferSequence, destination: &mut [u8]) { @@ -291,26 +283,9 @@ impl BrokerLocal { destination.len() <= buffer.length() as usize, "shared buffer sequence must cover the destination" ); - let mut offset = 0; - for descriptor in buffer - .descriptors(self.shared_buffers.layout()) - .expect("shared buffer sequence must identify valid slot ranges") - { - if offset == destination.len() { - break; - } - let length = (destination.len() - offset).min(descriptor.length as usize); - let end = offset + length; - self.shared_buffers - .read(descriptor.slot_index, &mut destination[offset..end]) - .expect("validated shared buffer sequence must be accessible"); - offset = end; - } - assert_eq!( - offset, - destination.len(), - "shared buffer sequence must cover the destination" - ); + self.shared_buffers + .read_sequence(buffer, destination) + .expect("validated shared buffer sequence must be accessible"); } /// Checks the current readiness of a broker-owned object. diff --git a/litebox_broker_transport/src/shared_memory.rs b/litebox_broker_transport/src/shared_memory.rs index df15ba0dd4..8e08e7ba2e 100644 --- a/litebox_broker_transport/src/shared_memory.rs +++ b/litebox_broker_transport/src/shared_memory.rs @@ -13,7 +13,7 @@ use alloc::sync::Arc; use thiserror::Error; use litebox_broker_protocol::shared_buffer::{ - SharedBufferLayout, SharedBufferLayoutError, SharedBufferSlotIndex, + SharedBufferLayout, SharedBufferLayoutError, SharedBufferSequence, SharedBufferSlotIndex, }; /// Error accessing a shared-memory resource. @@ -138,6 +138,9 @@ pub enum SharedBufferError { /// The backing shared-memory length does not exactly match the layout. #[error("shared-memory length does not match the shared-buffer layout")] MemoryLengthMismatch, + /// The requested transfer does not fit in the shared-buffer sequence. + #[error("shared-buffer sequence does not cover the requested transfer")] + TransferExceedsSequence, /// The backing shared-memory access failed. #[error("shared-memory access failed: {0}")] SharedMemory(#[from] SharedMemoryError), @@ -196,6 +199,58 @@ impl SharedBufferPool { self.memory.write(range.start, source)?; Ok(()) } + + /// Copies a sequence prefix into `destination`. + pub fn read_sequence( + &self, + sequence: SharedBufferSequence, + destination: &mut [u8], + ) -> Result<(), SharedBufferError> { + let descriptors = sequence.descriptors(self.layout)?; + if destination.len() > sequence.length() as usize { + return Err(SharedBufferError::TransferExceedsSequence); + } + let mut offset = 0; + for descriptor in descriptors { + if offset == destination.len() { + break; + } + let length = (destination.len() - offset).min(descriptor.length as usize); + let end = offset + length; + self.read(descriptor.slot_index, &mut destination[offset..end])?; + offset = end; + } + if offset != destination.len() { + return Err(SharedBufferError::TransferExceedsSequence); + } + Ok(()) + } + + /// Copies `source` into a sequence prefix. + pub fn write_sequence( + &self, + sequence: SharedBufferSequence, + source: &[u8], + ) -> Result<(), SharedBufferError> { + let descriptors = sequence.descriptors(self.layout)?; + if source.len() > sequence.length() as usize { + return Err(SharedBufferError::TransferExceedsSequence); + } + let mut offset = 0; + for descriptor in descriptors { + if offset == source.len() { + break; + } + let length = (source.len() - offset).min(descriptor.length as usize); + let end = offset + length; + self.write(descriptor.slot_index, &source[offset..end])?; + offset = end; + } + if offset != source.len() { + return Err(SharedBufferError::TransferExceedsSequence); + } + Ok(()) + } } #[cfg(test)] @@ -229,6 +284,31 @@ mod tests { ); } + #[test] + fn pool_copies_sequence_prefixes_across_slots() { + let layout = SharedBufferLayout::new(4, 3).unwrap(); + let memory = Arc::new(TestSharedMemory::new(layout.total_len())); + let pool = SharedBufferPool::new(Arc::clone(&memory), layout).unwrap(); + let sequence = + SharedBufferSequence::new(&[SharedBufferSlotIndex(0), SharedBufferSlotIndex(2)], 6) + .unwrap(); + + pool.write_sequence(sequence, &[1, 2, 3, 4, 5, 6]).unwrap(); + assert_eq!(memory.bytes(), [1, 2, 3, 4, 0, 0, 0, 0, 5, 6, 0, 0]); + + let mut destination = [0; 5]; + pool.read_sequence(sequence, &mut destination).unwrap(); + assert_eq!(destination, [1, 2, 3, 4, 5]); + assert_eq!( + pool.write_sequence(sequence, &[0; 7]), + Err(SharedBufferError::TransferExceedsSequence) + ); + assert_eq!( + pool.read_sequence(sequence, &mut [0; 7]), + Err(SharedBufferError::TransferExceedsSequence) + ); + } + struct TestSharedMemory(Mutex>); impl TestSharedMemory { diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index 110a6ddfaa..8de8ee2d4f 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; use litebox_broker_core::{BrokerCore, BrokerProcess}; -use litebox_broker_host::{RequestFailure, copy_shared_buffer}; +use litebox_broker_host::{RequestFailure, read_shared_buffer}; use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ @@ -135,7 +135,7 @@ impl RunnerProcessManager { ) -> Option> { match operation { BrokerOperation::StartProcess(request) => Some( - copy_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) + read_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) .and_then(|bootstrap| { self.start_process( process, From e27c4ddd33ccf1f4590a03d07d3588358f55dfbc Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sat, 19 Sep 2026 07:24:05 -0700 Subject: [PATCH 19/21] Simplify broker local process API Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_local/src/lib.rs | 40 ++++++++++++++- litebox_broker_local/src/process.rs | 49 ------------------- .../tests/userland_broker.rs | 4 +- 3 files changed, 40 insertions(+), 53 deletions(-) delete mode 100644 litebox_broker_local/src/process.rs diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 3090b49479..246718db82 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -24,7 +24,6 @@ mod error; mod event; mod fs; mod pipe; -mod process; mod random; mod socket; mod stdio; @@ -40,7 +39,10 @@ use litebox_broker_protocol::message::{ BrokerHandshakeRequest, BrokerHandshakeResponse, BrokerNotification, BrokerOperation, BrokerRequest, BrokerResponse, BrokerResult, }; -use litebox_broker_protocol::process::ProcessStartupData; +use litebox_broker_protocol::process::{ + InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, + ProcessBootstrapVersion, ProcessStartupData, ProcessStartupDescriptor, StartedProcess, +}; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; use litebox_broker_protocol::{ @@ -186,6 +188,40 @@ impl BrokerLocal { self.initial_thread_id } + /// Starts one child process. + /// + /// This call blocks until the child's broker association is active or + /// launch fails. The caller must retain exclusive ownership of the + /// bootstrap sequence until this method returns. + /// + /// # Panics + /// + /// Panics if the bootstrap length differs from the shared-buffer sequence + /// or the broker returns a response for another operation. + pub fn start_child_process( + &self, + format: ProcessBootstrapFormat, + version: ProcessBootstrapVersion, + buffer: SharedBufferSequence, + bootstrap: &[u8], + inherited_objects: InheritedProcessObjects, + ) -> Result { + if buffer.length() > MAX_PROCESS_BOOTSTRAP_SIZE { + return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); + } + self.write_shared_buffer(buffer, bootstrap); + match self.request(BrokerOperation::StartProcess(ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }))? { + BrokerResult::ProcessStarted(started) => Ok(started), + BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), + response => panic!("broker returned unexpected process-start response: {response:?}"), + } + } + /// Creates a broker thread belonging to this process. /// /// # Panics diff --git a/litebox_broker_local/src/process.rs b/litebox_broker_local/src/process.rs deleted file mode 100644 index ef5ee2637c..0000000000 --- a/litebox_broker_local/src/process.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -use litebox_broker_protocol::error::ErrorCode; -use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; -use litebox_broker_protocol::process::{ - InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartupDescriptor, StartedProcess, -}; -use litebox_broker_protocol::shared_buffer::SharedBufferSequence; -use litebox_broker_transport::channel::LocalCallChannel; - -use crate::{BrokerLocal, BrokerLocalError, Result}; - -impl BrokerLocal { - /// Requests materialization of one child process. - /// - /// This call blocks until the child's broker association is active or - /// launch fails. The caller must retain exclusive ownership of the - /// bootstrap sequence until this method returns. - /// - /// # Panics - /// - /// Panics if the bootstrap length differs from the shared-buffer sequence - /// or the broker returns a response for another operation. - pub fn request_process_start( - &self, - format: ProcessBootstrapFormat, - version: ProcessBootstrapVersion, - buffer: SharedBufferSequence, - bootstrap: &[u8], - inherited_objects: InheritedProcessObjects, - ) -> Result { - if buffer.length() > MAX_PROCESS_BOOTSTRAP_SIZE { - return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); - } - self.write_shared_buffer(buffer, bootstrap); - match self.request(BrokerOperation::StartProcess(ProcessStartupDescriptor { - format, - version, - buffer, - inherited_objects, - }))? { - BrokerResult::ProcessStarted(started) => Ok(started), - BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), - response => panic!("broker returned unexpected process-start response: {response:?}"), - } - } -} diff --git a/litebox_broker_userland/tests/userland_broker.rs b/litebox_broker_userland/tests/userland_broker.rs index 7ef274b1ff..78a61d8bf7 100644 --- a/litebox_broker_userland/tests/userland_broker.rs +++ b/litebox_broker_userland/tests/userland_broker.rs @@ -236,7 +236,7 @@ fn run_fake_runner(args: &[OsString]) { let inherited_event = local.create_event_with_count(1).unwrap(); let inherited_objects = InheritedProcessObjects::new(&[inherited_event]).unwrap(); let failed = local - .request_process_start( + .start_child_process( FAILING_BOOTSTRAP_FORMAT, ProcessBootstrapVersion(1), SharedBufferSequence::new( @@ -250,7 +250,7 @@ fn run_fake_runner(args: &[OsString]) { .unwrap(); assert_ne!(failed.process_id.0, failed.initial_thread_id.0); let started = local - .request_process_start( + .start_child_process( TEST_BOOTSTRAP_FORMAT, ProcessBootstrapVersion(1), SharedBufferSequence::new( From 901c000734000b14879b77050b875d7669cc5ba3 Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sat, 19 Sep 2026 11:55:11 -0700 Subject: [PATCH 20/21] Clarify child process start operation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_host/src/lib.rs | 2 +- litebox_broker_local/src/lib.rs | 14 ++++++---- litebox_broker_protocol/src/message.rs | 4 +-- litebox_broker_protocol/src/wire.rs | 28 ++++++++++--------- .../src/runner/process_manager.rs | 8 +++--- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/litebox_broker_host/src/lib.rs b/litebox_broker_host/src/lib.rs index 79fd10b45c..95debe318a 100644 --- a/litebox_broker_host/src/lib.rs +++ b/litebox_broker_host/src/lib.rs @@ -540,7 +540,7 @@ fn handle_request( BrokerOperation::File(request) => { handle_file_request(process, request, shared_buffers).map(BrokerResult::File) } - BrokerOperation::StartProcess(_) => { + BrokerOperation::StartChildProcess(_) => { Err(RequestFailure::Respond(ErrorCode::UnsupportedOperation)) } } diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 246718db82..53419e31e0 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -210,12 +210,14 @@ impl BrokerLocal { return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); } self.write_shared_buffer(buffer, bootstrap); - match self.request(BrokerOperation::StartProcess(ProcessStartupDescriptor { - format, - version, - buffer, - inherited_objects, - }))? { + match self.request(BrokerOperation::StartChildProcess( + ProcessStartupDescriptor { + format, + version, + buffer, + inherited_objects, + }, + ))? { BrokerResult::ProcessStarted(started) => Ok(started), BrokerResult::Error(error) => Err(BrokerLocalError::Broker(error)), response => panic!("broker returned unexpected process-start response: {response:?}"), diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index 663193aeee..ce6aaa50c2 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -66,7 +66,7 @@ pub enum BrokerOperation { /// File request family. File(FileRequest), /// Start one child process from an opaque platform bootstrap. - StartProcess(ProcessStartupDescriptor), + StartChildProcess(ProcessStartupDescriptor), } impl BrokerOperation { @@ -101,7 +101,7 @@ impl BrokerOperation { | FileRequest::Mkdir(MkdirFileRequest { path: buffer, .. }) | FileRequest::Rmdir(RmdirFileRequest { path: buffer, .. }), ) - | Self::StartProcess(ProcessStartupDescriptor { buffer, .. }) => Some(*buffer), + | Self::StartChildProcess(ProcessStartupDescriptor { buffer, .. }) => Some(*buffer), Self::CreateThread | Self::ExitThread(_) | Self::CloseObject(_) diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 597a8ca3a3..8df4c0ead1 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -48,7 +48,7 @@ const REQUEST_TAG_STDIO: u8 = 7; const REQUEST_TAG_FILE: u8 = 8; const REQUEST_TAG_CREATE_THREAD: u8 = 9; const REQUEST_TAG_EXIT_THREAD: u8 = 10; -const REQUEST_TAG_START_PROCESS: u8 = 11; +const REQUEST_TAG_START_CHILD_PROCESS: u8 = 11; // Tag 12 is reserved for the removed process-start acknowledgement. // Tags 13 and 14 are reserved for the removed process-ready protocol. @@ -126,7 +126,7 @@ pub fn decode_handshake_request(frame: &[u8]) -> Result { + | REQUEST_TAG_START_CHILD_PROCESS => { return Err(WireError::WrongMessagePhase); } _ => return Err(WireError::InvalidTag), @@ -195,13 +195,13 @@ pub fn encode_request(request: BrokerRequest) -> Vec { encoder.request_id(request_id); fs::encode_fs_request(&mut encoder, request); } - BrokerOperation::StartProcess(ProcessStartupDescriptor { + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { format, version, buffer, inherited_objects, }) => { - encoder.u8(REQUEST_TAG_START_PROCESS); + encoder.u8(REQUEST_TAG_START_CHILD_PROCESS); encoder.request_id(request_id); encoder.u32(format.0); encoder.u16(version.0); @@ -228,7 +228,7 @@ pub fn decode_request(frame: &[u8]) -> Result { | REQUEST_TAG_FILE | REQUEST_TAG_CREATE_THREAD | REQUEST_TAG_EXIT_THREAD - | REQUEST_TAG_START_PROCESS => {} + | REQUEST_TAG_START_CHILD_PROCESS => {} _ => return Err(WireError::InvalidTag), } let request_id = decoder.request_id()?; @@ -243,12 +243,14 @@ pub fn decode_request(frame: &[u8]) -> Result { REQUEST_TAG_FILL_RANDOM => BrokerOperation::FillRandom(decoder.shared_buffer_sequence()?), REQUEST_TAG_STDIO => BrokerOperation::Stdio(stdio::decode_stdio_request(&mut decoder)?), REQUEST_TAG_FILE => BrokerOperation::File(fs::decode_fs_request(&mut decoder)?), - REQUEST_TAG_START_PROCESS => BrokerOperation::StartProcess(ProcessStartupDescriptor { - format: ProcessBootstrapFormat(decoder.u32()?), - version: ProcessBootstrapVersion(decoder.u16()?), - buffer: decoder.shared_buffer_sequence()?, - inherited_objects: decode_inherited_objects(&mut decoder)?, - }), + REQUEST_TAG_START_CHILD_PROCESS => { + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { + format: ProcessBootstrapFormat(decoder.u32()?), + version: ProcessBootstrapVersion(decoder.u16()?), + buffer: decoder.shared_buffer_sequence()?, + inherited_objects: decode_inherited_objects(&mut decoder)?, + }) + } _ => unreachable!("active request tag was validated"), }; decoder.finish()?; @@ -655,7 +657,7 @@ mod tests { REQUEST_TAG_FILE, REQUEST_TAG_CREATE_THREAD, REQUEST_TAG_EXIT_THREAD, - REQUEST_TAG_START_PROCESS, + REQUEST_TAG_START_CHILD_PROCESS, ] ); assert_eq!( @@ -924,7 +926,7 @@ mod tests { name: TcpOptionName::KeepAlive, })), BrokerOperation::Socket(SocketRequest::Status(SocketStatusRequest { handle })), - BrokerOperation::StartProcess(ProcessStartupDescriptor { + BrokerOperation::StartChildProcess(ProcessStartupDescriptor { format: ProcessBootstrapFormat(u32::MAX), version: ProcessBootstrapVersion(u16::MAX), buffer: largest_sequence, diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index 8de8ee2d4f..0750ce4d3f 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -63,7 +63,7 @@ struct RunnerProcessManagerState { pub(crate) type AssociationFailure = Arc; -/// State for one blocking `StartProcess` request. +/// State for one blocking `StartChildProcess` request. pub(super) struct ProcessStartTransaction { parent_id: ProcessId, process: Arc, @@ -134,10 +134,10 @@ impl RunnerProcessManager { shared_buffers: &SharedBufferPool, ) -> Option> { match operation { - BrokerOperation::StartProcess(request) => Some( + BrokerOperation::StartChildProcess(request) => Some( read_shared_buffer(shared_buffers, request.buffer, MAX_PROCESS_BOOTSTRAP_SIZE) .and_then(|bootstrap| { - self.start_process( + self.start_child_process( process, request.format, request.version, @@ -152,7 +152,7 @@ impl RunnerProcessManager { } } - fn start_process( + fn start_child_process( self: &Arc, parent: &BrokerProcess, format: ProcessBootstrapFormat, From ba8e6754b12226a3f86cd57ad5fb9b06afa3d1be Mon Sep 17 00:00:00 2001 From: Weidong Cui Date: Sat, 19 Sep 2026 11:58:26 -0700 Subject: [PATCH 21/21] Clarify process identity result Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 05547201-01e0-4ff2-87c6-27739e6cc816 --- litebox_broker_local/src/lib.rs | 4 ++-- litebox_broker_protocol/src/message.rs | 4 ++-- litebox_broker_protocol/src/process.rs | 6 +++--- litebox_broker_protocol/src/wire.rs | 14 +++++++------- .../src/runner/process_manager.rs | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/litebox_broker_local/src/lib.rs b/litebox_broker_local/src/lib.rs index 53419e31e0..f8b05af910 100644 --- a/litebox_broker_local/src/lib.rs +++ b/litebox_broker_local/src/lib.rs @@ -41,7 +41,7 @@ use litebox_broker_protocol::message::{ }; use litebox_broker_protocol::process::{ InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartupData, ProcessStartupDescriptor, StartedProcess, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupData, ProcessStartupDescriptor, }; use litebox_broker_protocol::readiness::ReadinessFlags; use litebox_broker_protocol::shared_buffer::{SHARED_BUFFER_LAYOUT, SharedBufferSequence}; @@ -205,7 +205,7 @@ impl BrokerLocal { buffer: SharedBufferSequence, bootstrap: &[u8], inherited_objects: InheritedProcessObjects, - ) -> Result { + ) -> Result { if buffer.length() > MAX_PROCESS_BOOTSTRAP_SIZE { return Err(BrokerLocalError::Broker(ErrorCode::ResourceExhausted)); } diff --git a/litebox_broker_protocol/src/message.rs b/litebox_broker_protocol/src/message.rs index ce6aaa50c2..4e8a232622 100644 --- a/litebox_broker_protocol/src/message.rs +++ b/litebox_broker_protocol/src/message.rs @@ -17,7 +17,7 @@ use crate::pipe::{ CreatePipeRequest, CreatePipeResponse, ReadPipeRequest, ReadPipeResponse, WritePipeRequest, WritePipeResponse, }; -use crate::process::{ProcessStartupDescriptor, StartedProcess}; +use crate::process::{ProcessIdentity, ProcessStartupDescriptor}; use crate::readiness::ReadinessFlags; use crate::shared_buffer::SharedBufferSequence; use crate::socket::{ @@ -243,7 +243,7 @@ pub enum BrokerResult { /// File response family. File(FileResponse), /// A child established its broker association. - ProcessStarted(StartedProcess), + ProcessStarted(ProcessIdentity), /// Operation failed with an ABI-neutral broker error. Error(ErrorCode), } diff --git a/litebox_broker_protocol/src/process.rs b/litebox_broker_protocol/src/process.rs index ea7e48fb15..87e42e6b86 100644 --- a/litebox_broker_protocol/src/process.rs +++ b/litebox_broker_protocol/src/process.rs @@ -85,10 +85,10 @@ pub struct ProcessStartupData { pub inherited_objects: InheritedProcessObjects, } -/// Identifies a child process whose broker association was established. +/// Broker-assigned process and initial-thread identity. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct StartedProcess { - /// Broker-assigned child process ID. +pub struct ProcessIdentity { + /// Broker-assigned process ID. pub process_id: ProcessId, /// Broker-assigned initial thread ID. pub initial_thread_id: ThreadId, diff --git a/litebox_broker_protocol/src/wire.rs b/litebox_broker_protocol/src/wire.rs index 8df4c0ead1..c7c232a836 100644 --- a/litebox_broker_protocol/src/wire.rs +++ b/litebox_broker_protocol/src/wire.rs @@ -24,7 +24,7 @@ use crate::message::{ }; use crate::process::{ InheritedProcessObjects, MAX_INHERITED_PROCESS_OBJECTS, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartupDescriptor, StartedProcess, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupDescriptor, }; use crate::readiness::ReadinessFlags; @@ -408,7 +408,7 @@ pub fn encode_response(response: BrokerResponse) -> Vec { encoder.request_id(request_id); fs::encode_fs_response(&mut encoder, response); } - BrokerResult::ProcessStarted(StartedProcess { + BrokerResult::ProcessStarted(ProcessIdentity { process_id, initial_thread_id, }) => { @@ -461,7 +461,7 @@ pub fn decode_response(frame: &[u8]) -> Result { RESPONSE_TAG_RANDOM_FILLED => BrokerResult::RandomFilled, RESPONSE_TAG_STDIO => BrokerResult::Stdio(stdio::decode_stdio_response(&mut decoder)?), RESPONSE_TAG_FILE => BrokerResult::File(fs::decode_fs_response(&mut decoder)?), - RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(StartedProcess { + RESPONSE_TAG_PROCESS_STARTED => BrokerResult::ProcessStarted(ProcessIdentity { process_id: decoder.process_id()?, initial_thread_id: decoder.thread_id()?, }), @@ -583,8 +583,8 @@ mod tests { WritePipeResponse, }; use crate::process::{ - InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, - ProcessStartupDescriptor, StartedProcess, + InheritedProcessObjects, ProcessBootstrapFormat, ProcessBootstrapVersion, ProcessIdentity, + ProcessStartupDescriptor, }; use crate::shared_buffer::{SharedBufferSequence, SharedBufferSlotIndex}; use crate::socket::{ @@ -1283,11 +1283,11 @@ mod tests { BrokerResult::File(FileResponse::Mkdir), BrokerResult::File(FileResponse::Rmdir), BrokerResult::File(FileResponse::Failed(FileError::Io)), - BrokerResult::ProcessStarted(StartedProcess { + BrokerResult::ProcessStarted(ProcessIdentity { process_id: process_id(u32::MAX), initial_thread_id: thread_id(u32::MAX - 1), }), - BrokerResult::ProcessStarted(StartedProcess { + BrokerResult::ProcessStarted(ProcessIdentity { process_id: process_id(9), initial_thread_id: thread_id(11), }), diff --git a/litebox_broker_userland/src/runner/process_manager.rs b/litebox_broker_userland/src/runner/process_manager.rs index 0750ce4d3f..d19ba41ef8 100644 --- a/litebox_broker_userland/src/runner/process_manager.rs +++ b/litebox_broker_userland/src/runner/process_manager.rs @@ -13,7 +13,7 @@ use litebox_broker_protocol::error::ErrorCode; use litebox_broker_protocol::message::{BrokerOperation, BrokerResult}; use litebox_broker_protocol::process::{ InheritedProcessObjects, MAX_PROCESS_BOOTSTRAP_SIZE, ProcessBootstrapFormat, - ProcessBootstrapVersion, ProcessStartupData, StartedProcess, + ProcessBootstrapVersion, ProcessIdentity, ProcessStartupData, }; use litebox_broker_protocol::{ProcessId, ThreadId}; use litebox_broker_transport::shared_memory::{SharedBufferPool, SharedMemory}; @@ -159,7 +159,7 @@ impl RunnerProcessManager { version: ProcessBootstrapVersion, bootstrap: Vec, requested_inherited_objects: InheritedProcessObjects, - ) -> Result { + ) -> Result { if !parent.is_running() { return Err(ErrorCode::ProtocolState); } @@ -288,7 +288,7 @@ impl RunnerProcessManager { let deadline = Instant::now() + PROCESS_START_TIMEOUT; let result = transaction .wait_until_running(deadline) - .map(|()| StartedProcess { + .map(|()| ProcessIdentity { process_id, initial_thread_id: transaction.initial_thread_id, });