From cca52e28716483eabcdc1d40d0908de3143620dd Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Mon, 12 Oct 2026 07:35:28 -0700 Subject: [PATCH 1/2] fix: renew leases and preserve claimed fences Apply the parity repair for independently reported lease defects. Renew between drain turns when due, without relying on the renewal thread, and keep the activation fence while a message claim remains. Before the fix, regressions reported an expired lease during draining and a cleared expiry with an outstanding claim. Confirm renewal, recovery with max_attempts: 1, and replacement-owner fencing on SQLite, PostgreSQL, mysql2, and Trilogy. Preserve existing cross-pass renewal and NULL-expiry claim recovery behavior. --- CHANGELOG.md | 9 + docs/roadmap.md | 3 +- lib/solid_objects/activation.rb | 1 + lib/solid_objects/lease.rb | 1 + .../activation_lease_recovery_test.rb | 155 ++++++++++++++++++ 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 test/integration/activation_lease_recovery_test.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8fb28..47ecc7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +- Renew activations between turns in a drain pass when renewal is due, + including when the renewal thread has not run. +- Keep the activation fence and expiry while an outstanding message claim + remains. The next worker pass recovers an expired turn in mailbox order, + including with `max_attempts: 1`. Reclaiming increments `attempt_count`; + lease loss alone does not create a dead letter. + ## 0.17.3 - 2026-10-09 - Add four problem guides in `docs/guides/`: preventing race conditions in diff --git a/docs/roadmap.md b/docs/roadmap.md index 75cb615..5d771d1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -23,7 +23,8 @@ instead of 12. Concurrent creation causes no deadlock on SQLite, PostgreSQL, or MySQL. MySQL needs a shared read after a duplicate key, because it uses repeatable read. The tests count statements, and do not measure latency. -- Activation leases, renewal, unique activation tokens, generations, and +- Activation leases, renewal between turns as well as in the renewal thread, + unique activation tokens, generations, and fenced commits - Bounded activation passes, idle cache, hot-actor yield, and process records - At-least-once retries, terminal domain rejection, strict poison ordering, diff --git a/lib/solid_objects/activation.rb b/lib/solid_objects/activation.rb index a87b243..e663177 100644 --- a/lib/solid_objects/activation.rb +++ b/lib/solid_objects/activation.rb @@ -139,6 +139,7 @@ def drain_messages(message_id: nil, deadline: nil) break end + renew_lease if processed_count.positive? && lease_renewal_due? message = claim_next_message break unless message diff --git a/lib/solid_objects/lease.rb b/lib/solid_objects/lease.rb index 0fac7e2..a4efc5d 100644 --- a/lib/solid_objects/lease.rb +++ b/lib/solid_objects/lease.rb @@ -123,6 +123,7 @@ def release instance = Instance.lock.find_by(id: instance_id) next false unless instance next false unless owned_generation?(instance) + next false if ClaimedMessage.where(instance_id:).exists? instance.update!( activation_owner_id: nil, diff --git a/test/integration/activation_lease_recovery_test.rb b/test/integration/activation_lease_recovery_test.rb new file mode 100644 index 0000000..8f2f582 --- /dev/null +++ b/test/integration/activation_lease_recovery_test.rb @@ -0,0 +1,155 @@ +# rbs_inline: enabled + +require "database_test_helper" + +class ActivationLeaseRecoveryTest < ActiveSupport::TestCase + class Counter < SolidObjects::Actor + actor_type "activation-lease-recovery" + attribute :count, default: 0 + + class << self + attr_accessor :before_commit + end + + def increment + self.count += 1 + self.class.before_commit&.call + count + end + end + + setup do + SolidObjects.configuration.lease_duration = 0.2 + SolidObjects.configuration.lease_renewal_interval = 0.05 + SolidObjects.configuration.max_attempts = 1 + @clock = { now: SolidObjects.database_adapter.database_now } + clock = @clock + SolidObjects.database_adapter.define_singleton_method(:database_now) { clock.fetch(:now) } + Counter.before_commit = nil + @events = [] + @subscription = ActiveSupport::Notifications.subscribe(/solid_objects\.(message|activation|recovery)\./) do |event| + @events << event.name + end + end + + teardown do + ActiveSupport::Notifications.unsubscribe(@subscription) + SolidObjects.database_adapter.singleton_class.remove_method(:database_now) + Counter.before_commit = nil + end + + test "keeps renewing across short worker passes for five lease durations" do + worker = SolidObjects::Worker.new + Counter.before_commit = -> { @clock[:now] += 0.03 } + messages = [] + first_expiration = nil + + 34.times do + messages << Counter.ref("continuous").async.increment + assert_equal 1, worker.run_once + first_expiration ||= instance.activation_expires_at + end + + assert_operator instance.activation_expires_at, :>, first_expiration + 0.8 + assert_equal 1, instance.activation_generation + messages.each_with_index do |reference, index| + message = SolidObjects::Message.find(reference.id) + assert_predicate message, :completed? + assert_equal index + 1, message.result + assert_equal 1, message.attempt_count + end + assert_equal 34, @events.count("solid_objects.message.completed") + ensure + worker&.stop + end + + test "renews between short turns while draining without a renewal thread" do + messages = 34.times.map { Counter.ref("draining").async.increment } + registry = SolidObjects::ProcessRegistry.new + process = registry.register + activation = SolidObjects::ActivationManager.new(owner_id: process.id).claim_next + first_expiration = instance.activation_expires_at + Counter.before_commit = -> { @clock[:now] += 0.03 } + + assert_equal 34, activation.drain + assert_operator instance.activation_expires_at, :>, first_expiration + 0.8 + assert_equal 1, instance.activation_generation + assert messages.all? { |reference| SolidObjects::Message.find(reference.id).completed? } + ensure + activation&.deactivate + registry&.stop + end + + test "keeps the expired lease fence while a message claim remains" do + worker = SolidObjects::Worker.new + first = Counter.ref("expired").async.increment + second = Counter.ref("expired").async.increment + Counter.before_commit = -> { @clock[:now] += 0.22 } + + assert_equal 0, worker.run_once + assert_empty instance.state + assert SolidObjects::ClaimedMessage.exists?(message_id: first.id) + assert_not_nil instance.activation_expires_at + assert_operator instance.activation_expires_at, :<=, @clock.fetch(:now) + Counter.before_commit = nil + + assert_equal 2, worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 1, SolidObjects::Message.find(second.id).attempt_count + assert_empty SolidObjects::DeadLetter.all + assert_equal 2, @events.count("solid_objects.message.completed") + ensure + worker&.stop + end + + test "recovers on the same worker and preserves mailbox order with one allowed attempt" do + worker = SolidObjects::Worker.new + first = Counter.ref("recoverable").async.increment + second = Counter.ref("recoverable").async.increment + Counter.before_commit = -> { @clock[:now] += 0.22 } + + assert_equal 0, worker.run_once + assert_empty instance.state + Counter.before_commit = nil + + assert_equal 2, worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_empty SolidObjects::DeadLetter.all + ensure + worker&.stop + end + + test "a stale worker cannot release a second worker's activation or repeat its commits" do + first_worker = SolidObjects::Worker.new + second_worker = SolidObjects::Worker.new + first = Counter.ref("takeover").async.increment + second = Counter.ref("takeover").async.increment + Counter.before_commit = -> do + Counter.before_commit = nil + @clock[:now] += 0.22 + assert_equal 2, Thread.new { second_worker.run_once }.value + end + + assert_equal 0, first_worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_equal 2, @events.count("solid_objects.message.completed") + third = Counter.ref("takeover").async.increment + assert_equal 1, second_worker.run_once + assert_equal 3, SolidObjects::Message.find(third.id).result + ensure + first_worker&.stop + second_worker&.stop + end + + private + + def instance + SolidObjects::Instance.find_by!(actor_type: Counter.actor_type) + end +end From 6db0de8b8939397854218be1389356ef33a4aa35 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Mon, 12 Oct 2026 08:27:37 -0700 Subject: [PATCH 2/2] chore: prepare version 0.17.4 Align the gem version and lockfile and date the activation lease release notes. Review the consumer guides and validate the complete suite and built gem before preparing the release for tagging after merge. See #89 --- CHANGELOG.md | 2 +- Gemfile.lock | 4 ++-- lib/solid_objects/version.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ecc7a..56cabce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.17.4 - 2026-10-12 - Renew activations between turns in a drain pass when renewal is due, including when the renewal thread has not run. diff --git a/Gemfile.lock b/Gemfile.lock index 4ed22ff..50fa84c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_objects (0.17.3) + solid_objects (0.17.4) actioncable (>= 7.1) actionpack (>= 7.1) actionview (>= 7.1) @@ -392,7 +392,7 @@ CHECKSUMS rubocop-rails-omakase (1.1.0) sha256=2af73ac8ee5852de2919abbd2618af9c15c19b512c4cfc1f9a5d3b6ef009109d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 - solid_objects (0.17.3) + solid_objects (0.17.4) sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d sqlite3 (2.9.5-arm-linux-gnu) sha256=1bdfca0c7d63998c60b0f4a8e3c8df2d33800ccc4abd2d612eddbbbc92a4c48b diff --git a/lib/solid_objects/version.rb b/lib/solid_objects/version.rb index da3f728..9816c7d 100644 --- a/lib/solid_objects/version.rb +++ b/lib/solid_objects/version.rb @@ -1,5 +1,5 @@ # rbs_inline: enabled module SolidObjects - VERSION = "0.17.3" + VERSION = "0.17.4" end