From 9d8e1565ef88c8badab1523e041b8f468f882335 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Sat, 12 Sep 2026 07:02:27 -0700 Subject: [PATCH] fix: preserve committed turns in v0.14.6 Track SQL commitment before retry and failure handling so Rails callback errors cannot relabel a completed turn or mask its original exception. Carry the committed phase through coordination handlers and restore the original error at synchronous caller and worker boundaries. Cover real transaction callbacks, rollback, domain rejection, busy and deadline failures, waiting callers, and subsequent execution with strict red/green and implementation-reversal evidence. Document the error boundary and bump the version and generated lockfile to 0.14.6. --- CHANGELOG.md | 9 + Gemfile.lock | 4 +- docs/architecture.md | 31 +- docs/correctness.md | 25 +- lib/solid_objects/database_adapter.rb | 4 + lib/solid_objects/database_adapters/sqlite.rb | 4 + lib/solid_objects/errors.rb | 17 + lib/solid_objects/executor.rb | 10 +- lib/solid_objects/synchronous_invocation.rb | 2 + lib/solid_objects/version.rb | 2 +- lib/solid_objects/worker.rb | 2 + sig/generated/lib/solid_objects/errors.rbs | 12 + sig/generated/lib/solid_objects/executor.rbs | 2 + test/integration/post_commit_test.rb | 366 ++++++++++++++++++ 14 files changed, 483 insertions(+), 7 deletions(-) create mode 100644 test/integration/post_commit_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 09254f3..8b99442 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.14.6 - 2026-09-12 + +- Preserve committed turns when an Active Record after-commit callback raises. + Caller assistance and workers expose the original callback exception without + restoring obsolete state, retrying/rejecting the completed message, or + masking the error as a missing claim. Track actual SQL commitment before + SQLite retries and deadline translation; retain pre-commit rollback, retry, + and domain-rejection behavior. + ## 0.14.5 - 2026-09-03 - Split broadcast claiming into separate pending and stale-processing probes, diff --git a/Gemfile.lock b/Gemfile.lock index 1c8a95e..8d500f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_objects (0.14.5) + solid_objects (0.14.6) actioncable (>= 7.1) actionpack (>= 7.1) actionview (>= 7.1) @@ -384,7 +384,7 @@ CHECKSUMS rubocop-rails-omakase (1.1.0) sha256=2af73ac8ee5852de2919abbd2618af9c15c19b512c4cfc1f9a5d3b6ef009109d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 - solid_objects (0.14.5) + solid_objects (0.14.6) sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d sqlite3 (2.9.5-arm-linux-gnu) sha256=1bdfca0c7d63998c60b0f4a8e3c8df2d33800ccc4abd2d612eddbbbc92a4c48b diff --git a/docs/architecture.md b/docs/architecture.md index 4f885c4..23b36b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -314,9 +314,30 @@ Any lease or message predicate failure raises `LostActivation` and rolls back ev The state version can advance because of state migration even when the message itself makes no state change. +Active Record may raise from an application `after_commit` callback while the +transaction call unwinds, after SQL has committed. The adapter retains the +actual transaction object and checks its fully committed state before any +deadline conversion or SQLite retry. Reaching the end of the block, releasing +a savepoint, or observing an in-memory message update is not proof of SQL +commitment. A failing `before_commit` callback follows the rollback path. + +An internal `CommittedTransactionError` carries the original error through +executor, SQLite retry, and coordination rescue handlers. The synchronous +invocation and worker boundaries re-raise the original exception with its +original backtrace and cause. The wrapper prevents a callback's `Rejected`, +`LostActivation`, or database error from being mistaken for a pre-commit +outcome. It is not a durable message error or a new delivery mechanism. + +After commitment, actor state, application writes, message completion/result, +and outboxes remain durable. The executor preserves the committed activation +state and does not retry, reject, or dead-letter that turn. Caller assistance +still deactivates and releases its lease during cleanup; cached worker state +can process later messages. A worker running its continuous loop exposes the +error and runs its existing shutdown cleanup. + ## Failure path -Actor exceptions roll back all in-memory changes by restoring the pre-turn state. A separate short transaction conditionally owned by the current generation: +Before commitment, actor and commit-action exceptions roll back all in-memory changes by restoring the pre-turn state. A separate short transaction conditionally owned by the current generation: - Stores a sanitized error - Deletes claimed membership @@ -403,9 +424,15 @@ outer commit, and callers timing out on work they indirectly block. waiting and immediately returns a `MessageReference`. Runtime workers process it normally. +An executing caller receives an inline after-commit callback error even though +the turn committed. An independently waiting caller observes the durable +result and may return before that callback raises in the worker. Completed +history is not changed retroactively; callback failures must be observed in +the executing process. Later Rails callbacks may not run after one raises. + ## Domain rejection -Actor code can call `reject` for a validation or business-rule outcome that +Before commitment, actor code can call `reject` for a validation or business-rule outcome that must not retry. The executor restores pre-turn state, discards staged intents, stores the structured rejection, completes the claimed membership, and continues with the next sequence in one fenced transaction. Synchronous callers diff --git a/docs/correctness.md b/docs/correctness.md index 665ef93..47b29f6 100644 --- a/docs/correctness.md +++ b/docs/correctness.md @@ -125,6 +125,29 @@ it is available only when Solid Objects and `ActiveRecord::Base` share one connection pool. Commit actions must contain only bounded database work. External I/O belongs in the effect outbox. +### Errors after SQL commit + +Active Record `after_commit` callbacks registered by commit actions run after +SQL commitment. If one raises, the application writes, actor state, message +result/completion, and claimed-membership deletion remain committed. The +executor does not restore the pre-turn snapshot, retry the business action, +reject the message, or create a dead letter. A `before_commit` callback can +still roll back everything even after the fenced transaction block finishes. + +The executing synchronous caller or `Worker#run_once` receives the original +callback exception, including its identity, backtrace, and cause. This also +applies when the callback raises `Rejected`, `LostActivation`, or a database +deadline/lock error: its class does not change a committed turn into a failed +one. Synchronous cleanup releases the activation; a worker retains the +committed state until normal deactivation or shutdown, so later messages can +continue from that state. + +A separate waiting caller can observe the durable result before the callback +finishes. There is no retroactive failure delivery or durable callback-error +result. Inspect and report errors in the executing process. Rails may skip +later callbacks when one raises; these callbacks are not a durable delivery +mechanism. Use an idempotent effect for work that needs independent retries. + ## Reactive components A successful fenced turn advances `instances.state_revision` to that message's @@ -205,7 +228,7 @@ wakes the caller, and raises `SolidObjects::ActorDestroyed`. ## Domain rejection -`reject` is a terminal domain outcome, not an infrastructure failure. It rolls +`reject` before commitment is a terminal domain outcome, not an infrastructure failure. It rolls back in-memory state and staged intents, stores a structured rejection on the message, removes claimed membership, and lets the next sequence run. It is never retried or dead-lettered. The synchronous caller receives diff --git a/lib/solid_objects/database_adapter.rb b/lib/solid_objects/database_adapter.rb index 0d9c07e..26cfa92 100644 --- a/lib/solid_objects/database_adapter.rb +++ b/lib/solid_objects/database_adapter.rb @@ -123,17 +123,21 @@ def with_lock_probe # @rbs () { () -> untyped } -> untyped def transaction(&block) + active_transaction = nil raise DatabaseDeadlineExceeded, "synchronous invocation deadline expired" if SyncDeadline.expired? with_connection do |connection| with_transaction_deadline(connection) do connection.transaction(requires_new: true) do + active_transaction = connection.current_transaction configure_transaction_deadline(connection) with_transaction_clock { block.call } end end end rescue => error + raise CommittedTransactionError.new(error) if active_transaction&.state&.fully_committed? + raise unless deadline_error?(error) raise DatabaseDeadlineExceeded, diff --git a/lib/solid_objects/database_adapters/sqlite.rb b/lib/solid_objects/database_adapters/sqlite.rb index 4c39b28..ef1e3d5 100644 --- a/lib/solid_objects/database_adapters/sqlite.rb +++ b/lib/solid_objects/database_adapters/sqlite.rb @@ -33,6 +33,8 @@ def with_busy_retry attempts = 0 begin yield + rescue CommittedTransactionError + raise rescue => error raise unless busy_error?(error) @@ -53,6 +55,8 @@ def with_lock_retry with_connection do |connection| with_transaction_deadline(connection) { yield } end + rescue CommittedTransactionError + raise rescue DatabaseDeadlineExceeded raise if SyncDeadline.expired? diff --git a/lib/solid_objects/errors.rb b/lib/solid_objects/errors.rb index fc2593b..d3f3cbe 100644 --- a/lib/solid_objects/errors.rb +++ b/lib/solid_objects/errors.rb @@ -67,6 +67,23 @@ class ActorDestroyed < LostActivation class DatabaseDeadlineExceeded < Error end + class CommittedTransactionError < Error + # @rbs @original_error: StandardError + + attr_reader :original_error + + # @rbs (StandardError) -> void + def initialize(original_error) + @original_error = original_error + super(original_error.message) + end + + # @rbs () -> bot + def reraise + raise original_error, cause: original_error.cause + end + end + class SyncEnqueueTimeout < Error # @rbs @timeout: Numeric # @rbs @actor_type: String diff --git a/lib/solid_objects/executor.rb b/lib/solid_objects/executor.rb index 2006af2..4c652f2 100644 --- a/lib/solid_objects/executor.rb +++ b/lib/solid_objects/executor.rb @@ -4,6 +4,7 @@ module SolidObjects class Executor # @rbs @activation: Activation # @rbs @message: Message + # @rbs @completion_transaction: untyped # @rbs (activation: Activation, message: Message) -> void def initialize(activation:, message:) @@ -35,7 +36,7 @@ def call state_changed: state_after.value != state_before ) true - rescue LostActivation + rescue CommittedTransactionError, LostActivation raise rescue Rejected => rejection activation.restore_state(state_before) if state_before @@ -96,6 +97,7 @@ def complete(result, observable_changes, state_after:, state_changed:) moved_reminders = [] activation.lease.fenced_transaction do |instance| + @completion_transaction = Record.connection.current_transaction # A busy database makes the adapter retry this whole block, so an # attempt that was rolled back must not leave its work in the lists the # reporting below reads. Each attempt starts from empty. @@ -154,6 +156,12 @@ def complete(result, observable_changes, state_after:, state_changed:) report_large_state(state_after.byte_size) SolidObjects.instrument_after_commit(:"message.completed", **instrumentation_payload) SolidObjects.wake_up.signal + rescue CommittedTransactionError + raise + rescue => error + raise unless @completion_transaction&.state&.fully_committed? + + raise CommittedTransactionError.new(error) end # @rbs (Integer) -> void diff --git a/lib/solid_objects/synchronous_invocation.rb b/lib/solid_objects/synchronous_invocation.rb index df81bc8..25b6b0e 100644 --- a/lib/solid_objects/synchronous_invocation.rb +++ b/lib/solid_objects/synchronous_invocation.rb @@ -20,6 +20,8 @@ def call(message_reference, timeout:) SyncDeadline.with(timeout:) do call_before_deadline(message_reference, timeout:) end + rescue CommittedTransactionError => error + error.reraise rescue ActiveRecord::RecordNotFound raise ActorDestroyed, "actor was destroyed while waiting for its result" end diff --git a/lib/solid_objects/version.rb b/lib/solid_objects/version.rb index f90d21e..6796d1a 100644 --- a/lib/solid_objects/version.rb +++ b/lib/solid_objects/version.rb @@ -1,5 +1,5 @@ # rbs_inline: enabled module SolidObjects - VERSION = "0.14.5" + VERSION = "0.14.6" end diff --git a/lib/solid_objects/worker.rb b/lib/solid_objects/worker.rb index 7f0e53d..888aefc 100644 --- a/lib/solid_objects/worker.rb +++ b/lib/solid_objects/worker.rb @@ -59,6 +59,8 @@ def run_once ).around { activation.drain } release_activation(activation) if activation.pass_exhausted? processed + rescue CommittedTransactionError => error + error.reraise rescue ActorDestroyed release_activation(activation) if activation 0 diff --git a/sig/generated/lib/solid_objects/errors.rbs b/sig/generated/lib/solid_objects/errors.rbs index d858b25..b3705a3 100644 --- a/sig/generated/lib/solid_objects/errors.rbs +++ b/sig/generated/lib/solid_objects/errors.rbs @@ -67,6 +67,18 @@ module SolidObjects class DatabaseDeadlineExceeded < Error end + class CommittedTransactionError < Error + @original_error: StandardError + + attr_reader original_error: untyped + + # @rbs (StandardError) -> void + def initialize: (StandardError) -> void + + # @rbs () -> bot + def reraise: () -> bot + end + class SyncEnqueueTimeout < Error @timeout: Numeric diff --git a/sig/generated/lib/solid_objects/executor.rbs b/sig/generated/lib/solid_objects/executor.rbs index d580054..63e3ff7 100644 --- a/sig/generated/lib/solid_objects/executor.rbs +++ b/sig/generated/lib/solid_objects/executor.rbs @@ -2,6 +2,8 @@ module SolidObjects class Executor + @completion_transaction: untyped + @message: Message @activation: Activation diff --git a/test/integration/post_commit_test.rb b/test/integration/post_commit_test.rb new file mode 100644 index 0000000..06e1027 --- /dev/null +++ b/test/integration/post_commit_test.rb @@ -0,0 +1,366 @@ +# rbs_inline: enabled + +require "database_test_helper" +require "timeout" + +class PostCommitTest < ActiveSupport::TestCase + class CallbackFailure < StandardError + end + + class CallbackRecord < SolidObjectsTestDomainRecord + # @rbs @commit_callback: Proc? + # @rbs @before_commit_callback: Proc? + + attr_accessor :commit_callback, :before_commit_callback + + before_commit :run_before_commit_callback + after_commit :run_commit_callback + + # @rbs () -> void + def run_before_commit_callback + before_commit_callback&.call + end + + # @rbs () -> void + def run_commit_callback + commit_callback&.call + end + end + + class Counter < SolidObjects::Actor + actor_type "post-commit-counter" + + attribute :count, default: 0 + + class << self + # @rbs @before_increment: Proc? + + attr_accessor :before_increment + end + + # @rbs () -> Integer + def increment + self.class.before_increment&.call + self.count += 1 + commit_action(:write_counter, count:) + count + end + end + + class WaitingWakeUp + # @rbs @waiting: Thread::Queue + # @rbs @release: Thread::Queue + + attr_reader :waiting, :release + + # @rbs () -> void + def initialize + @waiting = Queue.new + @release = Queue.new + end + + # @rbs () -> void + def signal + end + + # @rbs (timeout: Numeric) -> void + def wait(timeout:) + waiting << true + release.pop + end + end + + test "caller assistance exposes the original after commit error and preserves the completed turn" do + failure = CallbackFailure.new("distinctive committed callback failure") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + + error = assert_raises(StandardError) { Counter.ref("inline").increment } + + assert_equal [ "1" ], CallbackRecord.pluck(:name) + instance = SolidObjects::Instance.sole + assert_equal({ "count" => 1 }, instance.state) + message = SolidObjects::Message.sole + assert message.completed? + assert_equal 1, message.result + assert_equal 1, message.attempt_count + assert_nil message.error + assert_nil message.rejected_at + assert_nil message.last_failed_at + assert_empty SolidObjects::ReadyMessage.all + assert_empty SolidObjects::ClaimedMessage.all + assert_empty SolidObjects::DeadLetter.all + assert_nil instance.activation_owner_id + assert_instance_of CallbackFailure, error + assert_same failure, error + assert_equal "distinctive committed callback failure", error.message + assert error.backtrace.any? { |line| line.include?("run_commit_callback") } + end + + test "a rejection raised after commit remains a callback error instead of a domain outcome" do + failure = SolidObjects::Rejected.new(code: "callback", message: "rejected after commitment") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + + error = assert_raises(StandardError) { Counter.ref("rejection").increment } + + assert_same failure, error + assert_nil SolidObjects::Message.sole.rejected_at + assert_equal 1, SolidObjects::Message.sole.result + assert_equal({ "count" => 1 }, SolidObjects::Instance.sole.state) + assert_empty SolidObjects::DeadLetter.all + end + + test "a successful callback runs outside the transaction and later messages advance the same actor" do + depths = [] + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, + commit_callback: -> { depths << ActiveRecord::Base.connection.open_transactions }) + end + reference = Counter.ref("success") + + assert_equal 1, reference.sync(idempotency_key: "once").increment + assert_equal 1, reference.sync(idempotency_key: "once").increment + assert_equal 2, reference.increment + assert_equal [ 0, 0 ], depths + assert_equal [ "1", "2" ], CallbackRecord.order(:id).pluck(:name) + assert_equal [ 1, 2 ], SolidObjects::Message.order(:sequence).pluck(:result) + assert_equal({ "count" => 2 }, SolidObjects::Instance.sole.state) + end + + test "a before commit callback failure rolls back even after the fenced block finishes" do + SolidObjects.configuration.max_attempts = 1 + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, + before_commit_callback: -> { raise CallbackFailure, "failed before SQL commit" }) + end + + error = assert_raises(SolidObjects::MessageFailed) { Counter.ref("before").increment } + + assert_equal "PostCommitTest::CallbackFailure", error.details.fetch("class") + assert_equal "failed before SQL commit", error.details.fetch("message") + assert_empty CallbackRecord.all + assert_equal({}, SolidObjects::Instance.sole.state) + message = SolidObjects::Message.sole + refute message.completed? + assert message.dead? + assert_nil message.result + assert_equal 1, message.attempt_count + assert_empty SolidObjects::ReadyMessage.all + assert_empty SolidObjects::ClaimedMessage.all + assert_equal 1, SolidObjects::DeadLetter.count + end + + test "a rolled back callback retries from the previous actor state" do + attempts = 0 + SolidObjects.configuration.max_attempts = 2 + SolidObjects.configuration.retry_delay = ->(_attempt) { 0 } + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + attempts += 1 + CallbackRecord.create!(name: arguments.fetch("count").to_s, + before_commit_callback: -> { raise CallbackFailure, "retry before commit" if attempts == 1 }) + end + + assert_equal 1, Counter.ref("retry").increment + assert_equal 2, attempts + assert_equal [ "1" ], CallbackRecord.pluck(:name) + assert_equal({ "count" => 1 }, SolidObjects::Instance.sole.state) + assert_equal 2, SolidObjects::Message.sole.attempt_count + assert_nil SolidObjects::Message.sole.error + assert_empty SolidObjects::DeadLetter.all + end + + test "a rejection before commit rolls back and lets the next message proceed" do + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, + before_commit_callback: -> { raise SolidObjects::Rejected.new(code: "invalid", message: "before commit") }) + end + reference = Counter.ref("domain") + + error = assert_raises(SolidObjects::Rejected) { reference.increment } + assert_equal "invalid", error.code + assert SolidObjects::Message.sole.rejected? + assert_nil SolidObjects::Message.sole.result + assert_equal({}, SolidObjects::Instance.sole.state) + assert_empty CallbackRecord.all + assert_empty SolidObjects::DeadLetter.all + + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s) + end + assert_equal 1, reference.increment + end + + test "a waiting caller receives completion before a separate worker exposes its callback failure" do + started = Queue.new + run_handler = Queue.new + committed = Queue.new + run_callback = Queue.new + wake_up = WaitingWakeUp.new + SolidObjects.configuration.wake_up_adapter = wake_up + Counter.before_increment = lambda do + started << true + run_handler.pop + end + failure = CallbackFailure.new("worker callback failed after caller returned") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + callback = lambda do + next unless arguments.fetch("count") == 1 + + committed << true + run_callback.pop + raise failure + end + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: callback) + end + reference = Counter.ref("worker") + message_reference = SolidObjects::Mailbox.new.enqueue( + reference:, operation: :increment, arguments: {}, delivery_mode: "sync" + ) + worker = SolidObjects::Worker.new + worker_thread = Thread.new do + ActiveRecord::Base.connection_pool.with_connection { worker.run_once } + rescue => error + error + end + Timeout.timeout(10) { started.pop } + caller_thread = Thread.new do + ActiveRecord::Base.connection_pool.with_connection { message_reference.wait(timeout: 10) } + end + Timeout.timeout(10) { wake_up.waiting.pop } + run_handler << true + Timeout.timeout(10) { committed.pop } + wake_up.release << true + + assert_equal 1, Timeout.timeout(10) { caller_thread.value } + assert worker_thread.alive? + run_callback << true + assert_same failure, Timeout.timeout(10) { worker_thread.value } + assert failure.backtrace.any? { |line| line.include?("run_commit_callback") } + message = SolidObjects::Message.find(message_reference.id) + assert message.completed? + assert_equal 1, message.result + assert_equal 1, message.attempt_count + assert_nil message.error + assert_nil message.last_failed_at + assert_nil message.rejected_at + assert_empty SolidObjects::ReadyMessage.all + assert_empty SolidObjects::ClaimedMessage.all + assert_empty SolidObjects::DeadLetter.all + + Counter.before_increment = nil + reference.async.increment + assert_equal 1, worker.run_once + assert_equal [ "1", "2" ], CallbackRecord.order(:id).pluck(:name) + assert_equal({ "count" => 2 }, SolidObjects::Instance.sole.state) + assert_equal [ 1, 1 ], SolidObjects::Message.order(:sequence).pluck(:attempt_count) + ensure + Counter.before_increment = nil + run_handler&.push(true) + run_callback&.push(true) + wake_up&.release&.push(true) + Timeout.timeout(10) do + worker_thread&.join + caller_thread&.join + end + worker&.stop + end + + test "a lost activation exception from an after commit callback escapes the worker unchanged" do + failure = SolidObjects::LostActivation.new("callback raised lost activation after commit") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + reference = Counter.ref("lost") + reference.async.increment + worker = SolidObjects::Worker.new + + error = assert_raises(SolidObjects::LostActivation) { worker.run_once } + + assert_same failure, error + assert SolidObjects::Message.sole.completed? + assert_nil SolidObjects::Message.sole.error + assert_equal({ "count" => 1 }, SolidObjects::Instance.sole.state) + assert_empty SolidObjects::DeadLetter.all + ensure + worker&.stop + end + + test "a deadline exception after commit is not retried or replaced by caller coordination" do + failure = SolidObjects::DatabaseDeadlineExceeded.new("callback deadline after commit") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + + error = assert_raises(StandardError) { Counter.ref("deadline").increment } + + assert_same failure, error + assert_equal [ "1" ], CallbackRecord.pluck(:name) + assert_equal 1, SolidObjects::Message.sole.result + assert_equal 1, SolidObjects::Message.sole.attempt_count + assert_empty SolidObjects::DeadLetter.all + end + + test "caller assistance never retries a database busy error raised after commit" do + failure = if database_family == :sqlite + SQLite3::BusyException.new("callback database busy after commit") + else + ActiveRecord::LockWaitTimeout.new("callback database busy after commit") + end + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + + error = assert_raises(StandardError) { Counter.ref("busy").increment } + + assert_same failure, error + assert_equal [ "1" ], CallbackRecord.pluck(:name) + assert_equal 1, SolidObjects::Message.sole.result + assert_nil SolidObjects::Message.sole.error + assert_empty SolidObjects::ReadyMessage.all + assert_empty SolidObjects::DeadLetter.all + end + + test "a worker never retries a database busy error raised after commit" do + failure = if database_family == :sqlite + SQLite3::BusyException.new("worker callback database busy after commit") + else + ActiveRecord::LockWaitTimeout.new("worker callback database busy after commit") + end + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, commit_callback: -> { raise failure }) + end + Counter.ref("worker-busy").async.increment + worker = SolidObjects::Worker.new + + error = assert_raises(StandardError) { worker.run_once } + + assert_same failure, error + assert_equal [ "1" ], CallbackRecord.pluck(:name) + assert SolidObjects::Message.sole.completed? + assert_nil SolidObjects::Message.sole.error + assert_empty SolidObjects::ReadyMessage.all + assert_empty SolidObjects::DeadLetter.all + ensure + worker&.stop + end + + test "caller cleanup preserves a callback record lookup error and its original cause" do + original_cause = CallbackFailure.new("original callback cause") + failure = ActiveRecord::RecordNotFound.new("callback record missing after commit") + SolidObjects.register_commit_action(:write_counter) do |arguments, _context| + CallbackRecord.create!(name: arguments.fetch("count").to_s, + commit_callback: -> { raise failure, cause: original_cause }) + end + + error = assert_raises(ActiveRecord::RecordNotFound) { Counter.ref("lookup").increment } + + assert_same failure, error + assert_same original_cause, error.cause + assert error.backtrace.any? { |line| line.include?("run_commit_callback") } + assert_nil SolidObjects::Instance.sole.activation_owner_id + assert SolidObjects::Message.sole.completed? + end +end