diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f8fb28..56cabce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 0.17.4 - 2026-10-12 + +- Renew activations between turns in a drain pass when renewal is due, + including when the renewal thread has not run. +- Keep the activation fence and expiry while an outstanding message claim + remains. The next worker pass recovers an expired turn in mailbox order, + including with `max_attempts: 1`. Reclaiming increments `attempt_count`; + lease loss alone does not create a dead letter. + ## 0.17.3 - 2026-10-09 - Add four problem guides in `docs/guides/`: preventing race conditions in diff --git a/Gemfile.lock b/Gemfile.lock index 4ed22ff..50fa84c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_objects (0.17.3) + solid_objects (0.17.4) actioncable (>= 7.1) actionpack (>= 7.1) actionview (>= 7.1) @@ -392,7 +392,7 @@ CHECKSUMS rubocop-rails-omakase (1.1.0) sha256=2af73ac8ee5852de2919abbd2618af9c15c19b512c4cfc1f9a5d3b6ef009109d ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33 securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1 - solid_objects (0.17.3) + solid_objects (0.17.4) sqlite3 (2.9.5-aarch64-linux-gnu) sha256=78075b6337d3d182c6d2b4691049ed45cd220826160c9ea18946bf6a1de200dc sqlite3 (2.9.5-aarch64-linux-musl) sha256=18c801185deb4adc01ddb281e8f672a39e3d1729979ca91e39439cd3eac0402d sqlite3 (2.9.5-arm-linux-gnu) sha256=1bdfca0c7d63998c60b0f4a8e3c8df2d33800ccc4abd2d612eddbbbc92a4c48b diff --git a/docs/roadmap.md b/docs/roadmap.md index 75cb615..5d771d1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -23,7 +23,8 @@ instead of 12. Concurrent creation causes no deadlock on SQLite, PostgreSQL, or MySQL. MySQL needs a shared read after a duplicate key, because it uses repeatable read. The tests count statements, and do not measure latency. -- Activation leases, renewal, unique activation tokens, generations, and +- Activation leases, renewal between turns as well as in the renewal thread, + unique activation tokens, generations, and fenced commits - Bounded activation passes, idle cache, hot-actor yield, and process records - At-least-once retries, terminal domain rejection, strict poison ordering, diff --git a/lib/solid_objects/activation.rb b/lib/solid_objects/activation.rb index a87b243..e663177 100644 --- a/lib/solid_objects/activation.rb +++ b/lib/solid_objects/activation.rb @@ -139,6 +139,7 @@ def drain_messages(message_id: nil, deadline: nil) break end + renew_lease if processed_count.positive? && lease_renewal_due? message = claim_next_message break unless message diff --git a/lib/solid_objects/lease.rb b/lib/solid_objects/lease.rb index 0fac7e2..a4efc5d 100644 --- a/lib/solid_objects/lease.rb +++ b/lib/solid_objects/lease.rb @@ -123,6 +123,7 @@ def release instance = Instance.lock.find_by(id: instance_id) next false unless instance next false unless owned_generation?(instance) + next false if ClaimedMessage.where(instance_id:).exists? instance.update!( activation_owner_id: nil, diff --git a/lib/solid_objects/version.rb b/lib/solid_objects/version.rb index da3f728..9816c7d 100644 --- a/lib/solid_objects/version.rb +++ b/lib/solid_objects/version.rb @@ -1,5 +1,5 @@ # rbs_inline: enabled module SolidObjects - VERSION = "0.17.3" + VERSION = "0.17.4" end diff --git a/test/integration/activation_lease_recovery_test.rb b/test/integration/activation_lease_recovery_test.rb new file mode 100644 index 0000000..8f2f582 --- /dev/null +++ b/test/integration/activation_lease_recovery_test.rb @@ -0,0 +1,155 @@ +# rbs_inline: enabled + +require "database_test_helper" + +class ActivationLeaseRecoveryTest < ActiveSupport::TestCase + class Counter < SolidObjects::Actor + actor_type "activation-lease-recovery" + attribute :count, default: 0 + + class << self + attr_accessor :before_commit + end + + def increment + self.count += 1 + self.class.before_commit&.call + count + end + end + + setup do + SolidObjects.configuration.lease_duration = 0.2 + SolidObjects.configuration.lease_renewal_interval = 0.05 + SolidObjects.configuration.max_attempts = 1 + @clock = { now: SolidObjects.database_adapter.database_now } + clock = @clock + SolidObjects.database_adapter.define_singleton_method(:database_now) { clock.fetch(:now) } + Counter.before_commit = nil + @events = [] + @subscription = ActiveSupport::Notifications.subscribe(/solid_objects\.(message|activation|recovery)\./) do |event| + @events << event.name + end + end + + teardown do + ActiveSupport::Notifications.unsubscribe(@subscription) + SolidObjects.database_adapter.singleton_class.remove_method(:database_now) + Counter.before_commit = nil + end + + test "keeps renewing across short worker passes for five lease durations" do + worker = SolidObjects::Worker.new + Counter.before_commit = -> { @clock[:now] += 0.03 } + messages = [] + first_expiration = nil + + 34.times do + messages << Counter.ref("continuous").async.increment + assert_equal 1, worker.run_once + first_expiration ||= instance.activation_expires_at + end + + assert_operator instance.activation_expires_at, :>, first_expiration + 0.8 + assert_equal 1, instance.activation_generation + messages.each_with_index do |reference, index| + message = SolidObjects::Message.find(reference.id) + assert_predicate message, :completed? + assert_equal index + 1, message.result + assert_equal 1, message.attempt_count + end + assert_equal 34, @events.count("solid_objects.message.completed") + ensure + worker&.stop + end + + test "renews between short turns while draining without a renewal thread" do + messages = 34.times.map { Counter.ref("draining").async.increment } + registry = SolidObjects::ProcessRegistry.new + process = registry.register + activation = SolidObjects::ActivationManager.new(owner_id: process.id).claim_next + first_expiration = instance.activation_expires_at + Counter.before_commit = -> { @clock[:now] += 0.03 } + + assert_equal 34, activation.drain + assert_operator instance.activation_expires_at, :>, first_expiration + 0.8 + assert_equal 1, instance.activation_generation + assert messages.all? { |reference| SolidObjects::Message.find(reference.id).completed? } + ensure + activation&.deactivate + registry&.stop + end + + test "keeps the expired lease fence while a message claim remains" do + worker = SolidObjects::Worker.new + first = Counter.ref("expired").async.increment + second = Counter.ref("expired").async.increment + Counter.before_commit = -> { @clock[:now] += 0.22 } + + assert_equal 0, worker.run_once + assert_empty instance.state + assert SolidObjects::ClaimedMessage.exists?(message_id: first.id) + assert_not_nil instance.activation_expires_at + assert_operator instance.activation_expires_at, :<=, @clock.fetch(:now) + Counter.before_commit = nil + + assert_equal 2, worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 1, SolidObjects::Message.find(second.id).attempt_count + assert_empty SolidObjects::DeadLetter.all + assert_equal 2, @events.count("solid_objects.message.completed") + ensure + worker&.stop + end + + test "recovers on the same worker and preserves mailbox order with one allowed attempt" do + worker = SolidObjects::Worker.new + first = Counter.ref("recoverable").async.increment + second = Counter.ref("recoverable").async.increment + Counter.before_commit = -> { @clock[:now] += 0.22 } + + assert_equal 0, worker.run_once + assert_empty instance.state + Counter.before_commit = nil + + assert_equal 2, worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_empty SolidObjects::DeadLetter.all + ensure + worker&.stop + end + + test "a stale worker cannot release a second worker's activation or repeat its commits" do + first_worker = SolidObjects::Worker.new + second_worker = SolidObjects::Worker.new + first = Counter.ref("takeover").async.increment + second = Counter.ref("takeover").async.increment + Counter.before_commit = -> do + Counter.before_commit = nil + @clock[:now] += 0.22 + assert_equal 2, Thread.new { second_worker.run_once }.value + end + + assert_equal 0, first_worker.run_once + assert_equal 1, SolidObjects::Message.find(first.id).result + assert_equal 2, SolidObjects::Message.find(second.id).result + assert_equal 2, SolidObjects::Message.find(first.id).attempt_count + assert_equal 2, @events.count("solid_objects.message.completed") + third = Counter.ref("takeover").async.increment + assert_equal 1, second_worker.run_once + assert_equal 3, SolidObjects::Message.find(third.id).result + ensure + first_worker&.stop + second_worker&.stop + end + + private + + def instance + SolidObjects::Instance.find_by!(actor_type: Counter.actor_type) + end +end