diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 200245df..7684eac7 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -132,17 +132,21 @@ def ensure_valid_worker_execution_options def ensure_fiber_workers_have_required_dependency return unless workers_options.any? { |options| fiber_worker?(options) } - SolidQueue::ExecutionPools::FiberPool.ensure_dependency! - rescue LoadError => error - errors.add(:base, error.message) + require "async" + require "async/semaphore" + rescue LoadError + errors.add(:base, "Fiber workers require the `async` gem. " \ + "Add `gem \"async\"` to your Gemfile to configure workers with `fibers`.") end def ensure_fiber_workers_use_supported_isolation_level return unless workers_options.any? { |options| fiber_worker?(options) } - SolidQueue::ExecutionPools::FiberPool.ensure_supported_isolation_level! - rescue ArgumentError => error - errors.add(:base, error.message) + unless ActiveSupport::IsolatedExecutionState.isolation_level == :fiber + errors.add(:base, "Fiber workers require fiber-scoped isolated execution state. " \ + "Set `config.active_support.isolation_level = :fiber` in your Rails configuration " \ + "(or `ActiveSupport::IsolatedExecutionState.isolation_level = :fiber` outside Rails).") + end end def default_options diff --git a/lib/solid_queue/execution_pools.rb b/lib/solid_queue/execution_pools.rb deleted file mode 100644 index 766cc189..00000000 --- a/lib/solid_queue/execution_pools.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module SolidQueue - module ExecutionPools - class << self - def build(type:, size:, on_state_change: nil) - case type - when :thread - ThreadPool.new(size, on_state_change: on_state_change) - when :fiber - FiberPool.new(size, on_state_change: on_state_change) - else - raise ArgumentError, "Unknown execution pool type #{type.inspect}. Expected one of: :thread, :fiber" - end - end - end - end -end diff --git a/lib/solid_queue/execution_pools/fiber_pool.rb b/lib/solid_queue/execution_pools/fiber_pool.rb deleted file mode 100644 index 72023d64..00000000 --- a/lib/solid_queue/execution_pools/fiber_pool.rb +++ /dev/null @@ -1,197 +0,0 @@ -# frozen_string_literal: true - -module SolidQueue - module ExecutionPools - class FiberPool - include AppExecutor - - class MissingDependencyError < LoadError - def initialize(error) - super( - "Fiber worker execution requires the `async` gem. " \ - "Add `gem \"async\"` to your Gemfile to configure workers with `fibers`. " \ - "Original error: #{error.message}" - ) - end - end - - class UnsupportedIsolationLevelError < ArgumentError - def initialize(level) - super( - "Fiber worker execution requires fiber-scoped isolated execution state. " \ - "Set `ActiveSupport::IsolatedExecutionState.isolation_level = :fiber` " \ - "(or `config.active_support.isolation_level = :fiber` in Rails). " \ - "Current isolation level: #{level.inspect}" - ) - end - end - - class << self - def ensure_dependency! - require "async" - require "async/semaphore" - rescue LoadError => error - raise MissingDependencyError.new(error) - end - - def ensure_supported_isolation_level! - return if supported_isolation_level? - - raise UnsupportedIsolationLevelError.new(ActiveSupport::IsolatedExecutionState.isolation_level) - end - - def supported_isolation_level? - ActiveSupport::IsolatedExecutionState.isolation_level == :fiber - end - end - - attr_reader :size - - def initialize(size, on_state_change: nil) - @size = size - @on_state_change = on_state_change - @available_capacity = size - @mutex = Mutex.new - @state_mutex = Mutex.new - @shutdown = false - @fatal_error = nil - @boot_queue = Thread::Queue.new - @pending_executions = Thread::Queue.new - - self.class.ensure_dependency! - self.class.ensure_supported_isolation_level! - - @reactor_thread = start_reactor - - boot_result = @boot_queue.pop - raise boot_result if boot_result.is_a?(Exception) - end - - def post(execution) - reserved = false - raise_if_fatal_error! - raise RuntimeError, "Execution pool is shutting down" if shutdown? - - reserve_capacity! - reserved = true - pending_executions << execution - rescue Exception - restore_capacity if reserved - raise - end - - def available_capacity - raise_if_fatal_error! - mutex.synchronize { @available_capacity } - end - - def idle? - available_capacity.positive? - end - - def shutdown - state_mutex.synchronize do - next false if @shutdown - - @shutdown = true - end.tap do |shut_down| - # Wake the reactor: already-queued executions are drained before the - # blocked pop in +wait_for_executions+ returns nil - pending_executions.close if shut_down - end - end - - def shutdown? - state_mutex.synchronize { @shutdown } - end - - def wait_for_termination(timeout) - reactor_thread.join(timeout) - end - - def metadata - { - fiber_pool_size: size, - inflight: size - available_capacity - } - end - - private - attr_reader :boot_queue, :mutex, :on_state_change, :pending_executions, :reactor_thread, :state_mutex - - def name - @name ||= "solid_queue-fiber-pool-#{object_id}" - end - - def start_reactor - create_thread do - Async do |task| - semaphore = Async::Semaphore.new(size, parent: task) - boot_queue << :ready - - # The reactor exits when all in-flight execution fibers, children - # of this task, have finished - wait_for_executions(semaphore) - end - rescue Exception => error - register_fatal_error(error) - raise - end - end - - def wait_for_executions(semaphore) - # Thread::Queue#pop is fiber-scheduler-aware: it suspends this fiber, letting - # execution fibers run, and wakes the reactor when the poller thread pushes new - # work or closes the queue on shutdown, after which it drains any remaining - # executions and returns nil - while execution = pending_executions.pop - semaphore.async(execution) do |_execution_task, scheduled_execution| - perform_execution(scheduled_execution) - end - end - end - - def perform_execution(execution) - wrap_in_app_executor { execution.perform } - rescue Async::Stop => error - handle_thread_error(error) - register_fatal_error(error) - rescue Exception => error - handle_thread_error(error) - ensure - restore_capacity - end - - def reserve_capacity! - mutex.synchronize do - raise RuntimeError, "Execution pool is at capacity" if @available_capacity <= 0 - - @available_capacity -= 1 - end - end - - def restore_capacity - should_notify = mutex.synchronize do - @available_capacity += 1 - @available_capacity.positive? - end - - on_state_change&.call if should_notify - end - - def register_fatal_error(error) - state_mutex.synchronize do - @fatal_error ||= error - end - - boot_queue << error if boot_queue.empty? - on_state_change&.call - end - - def raise_if_fatal_error! - error = state_mutex.synchronize { @fatal_error } - raise error if error - end - end - end -end diff --git a/lib/solid_queue/execution_pools/thread_pool.rb b/lib/solid_queue/execution_pools/thread_pool.rb deleted file mode 100644 index 5b7d1565..00000000 --- a/lib/solid_queue/execution_pools/thread_pool.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module SolidQueue - module ExecutionPools - class ThreadPool - include AppExecutor - - attr_reader :size - - delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor - - def initialize(size, on_state_change: nil) - @size = size - @on_state_change = on_state_change - @available_capacity = size - @mutex = Mutex.new - end - - def post(execution) - reserved = false - reserve_capacity! - reserved = true - - Concurrent::Promises.future_on(executor, execution) do |thread_execution| - wrap_in_app_executor { thread_execution.perform } - rescue Exception => error - handle_thread_error(error) - ensure - restore_capacity - end - rescue Exception - restore_capacity if reserved - raise - end - - def available_capacity - mutex.synchronize { @available_capacity } - end - - def idle? - available_capacity.positive? - end - - def metadata - { - inflight: size - available_capacity, - thread_pool_size: size - } - end - - private - attr_reader :mutex, :on_state_change - - DEFAULT_OPTIONS = { - min_threads: 0, - idletime: 60, - fallback_policy: :abort - } - - def executor - @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size) - end - - def reserve_capacity! - mutex.synchronize do - raise RuntimeError, "Execution pool is at capacity" if @available_capacity <= 0 - - @available_capacity -= 1 - end - end - - def restore_capacity - should_notify = mutex.synchronize do - @available_capacity += 1 - @available_capacity.positive? - end - - on_state_change&.call if should_notify - end - end - end -end diff --git a/lib/solid_queue/fiber_pool.rb b/lib/solid_queue/fiber_pool.rb new file mode 100644 index 00000000..0a29795f --- /dev/null +++ b/lib/solid_queue/fiber_pool.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +module SolidQueue + class FiberPool < Pool + def initialize(size, on_idle: nil) + super + + @state_mutex = Mutex.new + @shutdown = false + @fatal_error = nil + @boot_queue = Thread::Queue.new + @pending_executions = Thread::Queue.new + @reactor_thread = nil + end + + def post(execution) + raise_if_fatal_error! + raise RuntimeError, "Execution pool is shutting down" if shutdown? + + super + end + + def available_capacity + raise_if_fatal_error! + super + end + + def shutdown + state_mutex.synchronize do + next false if @shutdown + + @shutdown = true + end.tap do |shut_down| + # Wake the reactor: already-queued executions are drained before the + # blocked pop in +wait_for_executions+ returns nil + pending_executions.close if shut_down + end + end + + def shutdown? + state_mutex.synchronize { @shutdown } + end + + def wait_for_termination(timeout) + reactor_thread&.join(timeout) + end + + private + attr_reader :boot_queue, :pending_executions, :reactor_thread, :state_mutex + + def name + @name ||= "solid_queue-fiber-pool-#{object_id}" + end + + def schedule(execution) + start_reactor_if_needed + pending_executions << execution + end + + # The reactor thread is started lazily, when the first execution is posted, + # so that the pool can be safely built before forking: in the default fork + # supervisor mode, workers are instantiated in the supervisor process, and + # a thread started there wouldn't survive the fork. The async gem is also + # required lazily here, so that setups without fiber workers never load it. + def start_reactor_if_needed + @reactor_thread ||= begin + require "async" + require "async/semaphore" + + start_reactor.tap do + boot_result = boot_queue.pop + raise boot_result if boot_result.is_a?(Exception) + end + end + end + + def start_reactor + create_thread do + Async do |task| + semaphore = Async::Semaphore.new(size, parent: task) + boot_queue << :ready + + # The reactor exits when all in-flight execution fibers, children + # of this task, have finished + wait_for_executions(semaphore) + end + rescue Exception => error + register_fatal_error(error) + raise + end + end + + def wait_for_executions(semaphore) + # Thread::Queue#pop is fiber-scheduler-aware: it suspends this fiber, letting + # execution fibers run, and wakes the reactor when the poller thread pushes new + # work or closes the queue on shutdown, after which it drains any remaining + # executions and returns nil + while execution = pending_executions.pop + semaphore.async(execution) do |_execution_task, scheduled_execution| + perform_execution(scheduled_execution) + end + end + end + + def perform_execution(execution) + wrap_in_app_executor { execution.perform } + rescue Async::Stop => error + handle_thread_error(error) + register_fatal_error(error) + rescue Exception => error + handle_thread_error(error) + ensure + restore_capacity + end + + def register_fatal_error(error) + state_mutex.synchronize do + @fatal_error ||= error + end + + boot_queue << error if boot_queue.empty? + on_idle&.call + end + + def raise_if_fatal_error! + error = state_mutex.synchronize { @fatal_error } + raise error if error + end + end +end diff --git a/lib/solid_queue/pool.rb b/lib/solid_queue/pool.rb index 356bbaf3..f58ac8fc 100644 --- a/lib/solid_queue/pool.rb +++ b/lib/solid_queue/pool.rb @@ -1,5 +1,75 @@ # frozen_string_literal: true module SolidQueue - Pool = ExecutionPools::ThreadPool + class Pool + include AppExecutor + + def self.build(type:, size:, on_idle: nil) + SolidQueue.const_get("#{type.to_s.camelize}Pool").new(size, on_idle: on_idle) + end + + attr_reader :size + + def initialize(size, on_idle: nil) + @size = size + @on_idle = on_idle + @available_capacity = size + @mutex = Mutex.new + end + + def type + self.class.name.demodulize.delete_suffix("Pool").underscore.to_sym + end + + def post(execution) + reserve_capacity! + + begin + schedule(execution) + rescue Exception + restore_capacity + raise + end + end + + def available_capacity + mutex.synchronize { @available_capacity } + end + + def idle? + available_capacity.positive? + end + + private + attr_reader :mutex, :on_idle + + def schedule(execution) + raise NotImplementedError + end + + def perform_execution(execution) + wrap_in_app_executor { execution.perform } + rescue Exception => error + handle_thread_error(error) + ensure + restore_capacity + end + + def reserve_capacity! + mutex.synchronize do + raise RuntimeError, "Execution pool is at capacity" if @available_capacity <= 0 + + @available_capacity -= 1 + end + end + + def restore_capacity + should_notify = mutex.synchronize do + @available_capacity += 1 + @available_capacity.positive? + end + + on_idle&.call if should_notify + end + end end diff --git a/lib/solid_queue/thread_pool.rb b/lib/solid_queue/thread_pool.rb new file mode 100644 index 00000000..b1485264 --- /dev/null +++ b/lib/solid_queue/thread_pool.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module SolidQueue + class ThreadPool < Pool + delegate :shutdown, :shutdown?, :wait_for_termination, to: :executor + + private + DEFAULT_OPTIONS = { + min_threads: 0, + idletime: 60, + fallback_policy: :abort + } + + def schedule(execution) + Concurrent::Promises.future_on(executor, execution) do |thread_execution| + perform_execution(thread_execution) + end.on_rejection! do |error| + # Backstop for errors raised outside perform_execution's own rescue, + # such as when restoring capacity or waking up the worker + handle_thread_error(error) + end + end + + def executor + @executor ||= Concurrent::ThreadPoolExecutor.new DEFAULT_OPTIONS.merge(max_threads: size, max_queue: size) + end + end +end diff --git a/lib/solid_queue/worker.rb b/lib/solid_queue/worker.rb index 31945b95..9ba65bf3 100644 --- a/lib/solid_queue/worker.rb +++ b/lib/solid_queue/worker.rb @@ -11,29 +11,24 @@ class Worker < Processes::Poller attr_reader :queues, :pool def initialize(**options) - options = options.dup - validate_execution_options!(options) - execution_pool_type = options.key?(:fibers) ? :fiber : :thread - execution_pool_size = options[:fibers] || options[:threads] || SolidQueue::Configuration::WORKER_DEFAULTS[:threads] - options = options.with_defaults(worker_defaults_for(options)) + + options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS) + execution_pool_size = execution_pool_type == :fiber ? options[:fibers] : options[:threads] # Ensure that the queues array is deep frozen to prevent accidental modification @queues = Array(options[:queues]).map(&:freeze).freeze - @metadata_state_mutex = Mutex.new - @metadata_dirty = false - @pool_options = { + @pool = Pool.build \ type: execution_pool_type, size: execution_pool_size, - on_state_change: -> { mark_metadata_dirty; wake_up } - } + on_idle: -> { wake_up } super(**options) end def metadata - super.merge(queues: queues.join(",")).merge(pool.metadata) + super.merge(queues: queues.join(","), pool_type: pool.type, pool_size: pool.size) end private @@ -43,8 +38,6 @@ def poll pool.post(execution) end - reload_metadata_if_needed(executions.any?) - pool.idle? ? polling_interval : 10.minutes end end @@ -55,11 +48,6 @@ def claim_executions end end - def boot - build_pool - super - end - def shutdown pool.shutdown pool.wait_for_termination(SolidQueue.shutdown_timeout) @@ -71,49 +59,8 @@ def all_work_completed? SolidQueue::ReadyExecution.aggregated_count_across(queues).zero? end - def heartbeat - super - reload_metadata - end - def set_procline procline "waiting for jobs in #{queues.join(",")}" end - - def build_pool - @pool ||= ExecutionPools.build(**@pool_options) - end - - def validate_execution_options!(options) - if options.key?(:threads) && options.key?(:fibers) - raise ArgumentError, "Workers can specify either `threads` or `fibers`, but not both." - end - end - - def worker_defaults_for(options) - if options.key?(:fibers) - SolidQueue::Configuration::WORKER_DEFAULTS.except(:threads) - else - SolidQueue::Configuration::WORKER_DEFAULTS - end - end - - def mark_metadata_dirty - metadata_state_mutex.synchronize { @metadata_dirty = true } - end - - def metadata_state_mutex - @metadata_state_mutex - end - - def reload_metadata_if_needed(executions_claimed) - needs_reload = metadata_state_mutex.synchronize do - claimed_or_dirty = executions_claimed || @metadata_dirty - @metadata_dirty = false - claimed_or_dirty - end - - reload_metadata if needs_reload - end end end diff --git a/test/integration/fiber_processes_lifecycle_test.rb b/test/integration/fiber_processes_lifecycle_test.rb index 3a1eafe3..22e207e7 100644 --- a/test/integration/fiber_processes_lifecycle_test.rb +++ b/test/integration/fiber_processes_lifecycle_test.rb @@ -21,7 +21,7 @@ class FiberProcessesLifecycleTest < ActiveSupport::TestCase test "process jobs in fibers in a forked worker" do worker = find_processes_registered_as("Worker").first - assert_metadata worker, fiber_pool_size: 3 + assert_metadata worker, pool_type: "fiber", pool_size: 3 6.times { |i| enqueue_store_result_job("job_#{i}") } diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index b25bc066..4a0d7c06 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -228,15 +228,13 @@ class ConfigurationTest < ActiveSupport::TestCase # Fiber workers require fiber-scoped isolated execution state configuration = SolidQueue::Configuration.new(skip_recurring: true, dispatchers: [], workers: [ { fibers: 3 } ]) assert_not configuration.valid? - assert_match /requires fiber-scoped isolated execution state/, configuration.errors.full_messages.first + assert_match /require fiber-scoped isolated execution state/, configuration.errors.full_messages.first # Fiber workers require the async gem with_execution_isolation(:fiber) do - load_error = LoadError.new("cannot load such file -- async") - missing_dependency_error = SolidQueue::ExecutionPools::FiberPool::MissingDependencyError.new(load_error) - SolidQueue::ExecutionPools::FiberPool.expects(:ensure_dependency!).raises(missing_dependency_error) - configuration = SolidQueue::Configuration.new(skip_recurring: true, dispatchers: [], workers: [ { fibers: 3 } ]) + configuration.expects(:require).with("async").raises(LoadError.new("cannot load such file -- async")) + assert_not configuration.valid? assert_match /gem "async"/, configuration.errors.full_messages.first end diff --git a/test/unit/execution_pools/fiber_pool_test.rb b/test/unit/fiber_pool_test.rb similarity index 66% rename from test/unit/execution_pools/fiber_pool_test.rb rename to test/unit/fiber_pool_test.rb index 16e8f75b..f82694ba 100644 --- a/test/unit/execution_pools/fiber_pool_test.rb +++ b/test/unit/fiber_pool_test.rb @@ -16,37 +16,17 @@ def perform end end - def test_raises_a_clear_error_when_the_async_gem_is_unavailable - load_error = LoadError.new("cannot load such file -- async") - - SolidQueue::ExecutionPools::FiberPool.expects(:require).with("async").raises(load_error) - - error = assert_raises SolidQueue::ExecutionPools::FiberPool::MissingDependencyError do - SolidQueue::ExecutionPools::FiberPool.new(3) - end - - assert_match /gem "async"/, error.message - end - def test_builds_a_fiber_pool pool = mock - SolidQueue::ExecutionPools::FiberPool.expects(:new).with(5, on_state_change: nil).returns(pool) - - assert_equal pool, SolidQueue::ExecutionPools.build(type: :fiber, size: 5) - end - - def test_raises_a_clear_error_when_isolation_level_is_not_fiber - error = assert_raises SolidQueue::ExecutionPools::FiberPool::UnsupportedIsolationLevelError do - SolidQueue::ExecutionPools::FiberPool.new(3) - end + SolidQueue::FiberPool.expects(:new).with(5, on_idle: nil).returns(pool) - assert_match /isolation_level = :fiber/, error.message + assert_equal pool, SolidQueue::Pool.build(type: :fiber, size: 5) end def test_executes_jobs_as_fibers_on_a_single_reactor_thread with_execution_isolation(:fiber) do - pool = SolidQueue::ExecutionPools::FiberPool.new(2) + pool = SolidQueue::FiberPool.new(2) results = Thread::Queue.new pool.post Execution.new(nil, results, 0.05) @@ -57,7 +37,6 @@ def test_executes_jobs_as_fibers_on_a_single_reactor_thread assert_equal 1, entries.map(&:first).uniq.count assert_equal 2, entries.map(&:last).uniq.count assert_equal 2, pool.available_capacity - assert_equal 0, pool.metadata[:inflight] ensure pool&.shutdown pool&.wait_for_termination(1.second) @@ -66,7 +45,7 @@ def test_executes_jobs_as_fibers_on_a_single_reactor_thread def test_waits_for_in_flight_executions_during_shutdown with_execution_isolation(:fiber) do - pool = SolidQueue::ExecutionPools::FiberPool.new(1) + pool = SolidQueue::FiberPool.new(1) started = Thread::Queue.new pool.post Execution.new(started, nil, 0.1) @@ -84,7 +63,11 @@ def test_waits_for_in_flight_executions_during_shutdown def test_shutdown_wakes_the_reactor_when_idle with_execution_isolation(:fiber) do - pool = SolidQueue::ExecutionPools::FiberPool.new(1) + pool = SolidQueue::FiberPool.new(1) + results = Thread::Queue.new + + pool.post Execution.new(nil, results, nil) + Timeout.timeout(1.second) { results.pop } pool.shutdown @@ -95,6 +78,27 @@ def test_shutdown_wakes_the_reactor_when_idle end end + def test_starts_the_reactor_lazily_so_the_pool_can_be_built_before_forking + with_execution_isolation(:fiber) do + pool = SolidQueue::FiberPool.new(1) + + pid = fork do + results = Thread::Queue.new + pool.post Execution.new(nil, results, nil) + Timeout.timeout(1.second) { results.pop } + pool.shutdown + pool.wait_for_termination(1.second) + exit!(0) + end + + _, status = Process.waitpid2(pid) + assert_equal 0, status.exitstatus + ensure + pool&.shutdown + pool&.wait_for_termination(1.second) + end + end + def test_marks_the_pool_as_fatal_when_an_execution_is_cancelled with_execution_isolation(:fiber) do notifications = Thread::Queue.new @@ -103,7 +107,7 @@ def test_marks_the_pool_as_fatal_when_an_execution_is_cancelled original_on_thread_error = SolidQueue.on_thread_error SolidQueue.on_thread_error = ->(error) { reported_errors << error.class.name } - pool = SolidQueue::ExecutionPools::FiberPool.new(1, on_state_change: -> { notifications << :changed }) + pool = SolidQueue::FiberPool.new(1, on_idle: -> { notifications << :changed }) pool.post CancelledExecution.new(started) Timeout.timeout(1.second) { started.pop } @@ -111,7 +115,6 @@ def test_marks_the_pool_as_fatal_when_an_execution_is_cancelled error = assert_raises(Async::Stop) { pool.available_capacity } assert_equal [ error.class.name ], reported_errors - assert_raises(Async::Stop) { pool.metadata } ensure SolidQueue.on_thread_error = original_on_thread_error pool&.shutdown diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index a7baecb2..c21cd574 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -8,19 +8,11 @@ class WorkerTest < ActiveSupport::TestCase EXECUTION_MODES = [ { name: "thread", - options: { threads: 3 }, - expected_metadata: { - inflight: 0, - thread_pool_size: 3 - } + options: { threads: 3 } }, { name: "fiber", - options: { fibers: 3 }, - expected_metadata: { - inflight: 0, - fiber_pool_size: 3 - } + options: { fibers: 3 } } ].freeze @@ -43,10 +35,7 @@ class WorkerTest < ActiveSupport::TestCase process = SolidQueue::Process.first assert_equal "Worker", process.kind - assert_metadata process, { - queues: "background", - polling_interval: 0.2 - }.merge(mode[:expected_metadata]) + assert_metadata process, queues: "background", polling_interval: 0.2, pool_type: mode[:name], pool_size: 3 ensure worker&.stop wait_for_registered_processes(0, timeout: 1.second) @@ -77,60 +66,6 @@ class WorkerTest < ActiveSupport::TestCase wait_for_registered_processes(0, timeout: 1.second) end end - - test "updates inflight metadata after jobs finish in #{mode[:name]} mode" do - StoreResultJob.perform_later(:slow, pause: 0.1.second) - - with_worker_execution_support(mode[:options]) do - worker = SolidQueue::Worker.new(queues: "background", polling_interval: 0.05, **mode[:options]) - - worker.start - wait_for_registered_processes(1, timeout: 1.second) - - process = SolidQueue::Process.first - - wait_for(timeout: 2.seconds) { process.reload.metadata["inflight"] == 1 } - assert_equal 1, process.reload.metadata["inflight"] - - wait_for(timeout: 2.seconds) do - process.reload.metadata["inflight"] == 0 && - JobResult.where(queue_name: :background, status: "completed", value: :slow).count == 1 - end - assert_equal 0, process.reload.metadata["inflight"] - assert_equal 1, JobResult.where(queue_name: :background, status: "completed", value: :slow).count - ensure - worker&.stop - wait_for_registered_processes(0, timeout: 1.second) - end - end - end - - test "builds the execution pool on boot instead of initialize" do - pool = SolidQueue::ExecutionPools::ThreadPool.new(3) - - SolidQueue::ExecutionPools.expects(:build).once.with do |**options| - options[:type] == :thread && options[:size] == 3 && options[:on_state_change].respond_to?(:call) - end.returns(pool) - - worker = SolidQueue::Worker.new(queues: "background", threads: 3, polling_interval: 0.2) - - assert_nil worker.pool - - worker.start - wait_for_registered_processes(1, timeout: 1.second) - - assert_equal pool, worker.pool - ensure - worker&.stop - wait_for_registered_processes(0, timeout: 1.second) - end - - test "rejects multiple worker pool size options" do - error = assert_raises ArgumentError do - SolidQueue::Worker.new(queues: "background", threads: 3, fibers: 3, polling_interval: 0.2) - end - - assert_match /either `threads` or `fibers`/, error.message end test "defaults thread workers to the configured thread pool size" do @@ -140,7 +75,7 @@ class WorkerTest < ActiveSupport::TestCase wait_for_registered_processes(1, timeout: 1.second) assert_equal 3, worker.pool.size - assert_metadata SolidQueue::Process.first, thread_pool_size: 3, inflight: 0 + assert_metadata SolidQueue::Process.first, pool_type: "thread", pool_size: 3 ensure worker&.stop wait_for_registered_processes(0, timeout: 1.second)