From 9b3492ae36dd140496867404ccad2bd9f679601b Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:20:02 -0800 Subject: [PATCH 1/4] feat: report external storage metrics on workflow task completions --- .../internal/external_storage_runner.rb | 25 ++++- .../internal/storage_operation_metrics.rb | 73 ++++++++++++ .../internal/worker/workflow_worker.rb | 104 ++++++++++++------ .../internal/external_storage_runner.rbi | 16 ++- .../internal/storage_operation_metrics.rbi | 27 +++++ .../internal/worker/workflow_worker.rbi | 6 + .../internal/external_storage_runner.rbs | 10 +- .../internal/storage_operation_metrics.rbs | 25 +++++ .../internal/worker/workflow_worker.rbs | 2 + .../test/external_storage_runner_test.rb | 47 ++++++++ temporalio/test/in_memory_storage_driver.rb | 6 +- .../test/sig/worker_external_storage_test.rbs | 3 + .../test/worker_external_storage_test.rb | 53 +++++++++ 13 files changed, 352 insertions(+), 45 deletions(-) create mode 100644 temporalio/lib/temporalio/internal/storage_operation_metrics.rb create mode 100644 temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi create mode 100644 temporalio/sig/temporalio/internal/storage_operation_metrics.rbs diff --git a/temporalio/lib/temporalio/internal/external_storage_runner.rb b/temporalio/lib/temporalio/internal/external_storage_runner.rb index e25ed3a5..e508daf7 100644 --- a/temporalio/lib/temporalio/internal/external_storage_runner.rb +++ b/temporalio/lib/temporalio/internal/external_storage_runner.rb @@ -9,6 +9,7 @@ require 'temporalio/converters/storage_driver_store_context' require 'temporalio/error' require 'temporalio/internal/external_storage_references' +require 'temporalio/internal/storage_operation_metrics' module Temporalio module Internal @@ -33,8 +34,9 @@ def initialize(storage, namespace:) end # @param payloads [Array] Codec-encoded payloads. Not mutated. + # @param metrics [StorageOperationMetrics, nil] Records each driver call that stores payloads. # @return [Array] Payloads with every eligible one replaced by a reference. - def store(payloads) + def store(payloads, metrics: nil) storage = @storage return payloads unless storage @@ -69,8 +71,17 @@ def store(payloads) # Drivers get copies, since a driver may keep a payload after returning and the SDK modifies some payloads in # place copies = indexes_and_serialized.map { |_index, serialized| Api::Common::V1::Payload.decode(serialized) } + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) claims = driver.store(context, copies) + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) check_result_count(driver, 'store', indexes_and_serialized.size, claims) + metrics&.record( + driver_name: driver.name, + payload_count: claims.size, + total_size_bytes: indexes_and_serialized.sum { |_index, serialized| serialized.bytesize }, + start_time:, + end_time: + ) indexes_and_serialized.zip(claims) do |(index, serialized), claim| result[index] = ExternalStorageReferences.create_reference_payload( @@ -82,9 +93,10 @@ def store(payloads) end # @param payloads [Array] Payloads that may contain references. Not mutated. + # @param metrics [StorageOperationMetrics, nil] Records each driver call that retrieves payloads. # @return [Array] Payloads with every reference replaced by what it points at. # @raise [Error::ExternalStorageNotConfiguredError] If there is a reference but no storage is configured. - def retrieve(payloads) + def retrieve(payloads, metrics: nil) storage = @storage # Driver => [[index, claim], ...] for each reference that driver stored. Every driver is looked up here, so an # unregistered name fails before any request is made. @@ -105,8 +117,17 @@ def retrieve(payloads) context = Converters::StorageDriverRetrieveContext.new( cancellation: Cancellation.new, limiter: UNBOUNDED_LIMITER ) + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) retrieved = driver.retrieve(context, indexes_and_claims.map { |_index, claim| claim }) + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) check_result_count(driver, 'retrieve', indexes_and_claims.size, retrieved) + metrics&.record( + driver_name: driver.name, + payload_count: retrieved.size, + total_size_bytes: retrieved.sum { |payload| payload.to_proto.bytesize }, + start_time:, + end_time: + ) indexes_and_claims.zip(retrieved) { |(index, _claim), payload| result[index] = payload } end diff --git a/temporalio/lib/temporalio/internal/storage_operation_metrics.rb b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb new file mode 100644 index 00000000..5c1bf503 --- /dev/null +++ b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'temporalio/internal/bridge/api' +require 'temporalio/internal/proto_utils' + +module Temporalio + module Internal + # Gathers what external storage moved during one store or retrieve pass, for core to report on the activation + # completion. + # + # @!visibility private + class StorageOperationMetrics + def initialize + @mutex = Mutex.new + @payload_count = 0 + @total_size_bytes = 0 + @driver_names = Set.new + @spans = [] + end + + # Record one driver call that moved payloads. + # + # @param driver_name [String] Driver that stored or retrieved the payloads. + # @param payload_count [Integer] Number of payloads moved. + # @param total_size_bytes [Integer] Encoded size of the payloads moved. + # @param start_time [Float] Monotonic clock reading taken before the driver call. + # @param end_time [Float] Monotonic clock reading taken after the driver call. + def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:) + @mutex.synchronize do + @driver_names << driver_name + @payload_count += payload_count + @total_size_bytes += total_size_bytes + @spans << [start_time, end_time] + end + end + + # @return [Bridge::Api::Common::ExternalStorageMetrics, nil] Metrics for the pass, or nil when it moved nothing, + # so the completion field stays unset. + def to_proto + @mutex.synchronize do + return nil if @payload_count.zero? + + Bridge::Api::Common::ExternalStorageMetrics.new( + payload_count: @payload_count, + total_size_bytes: @total_size_bytes, + total_duration: ProtoUtils.seconds_to_duration(union_seconds), + driver_names: @driver_names.sort + ) + end + end + + private + + def union_seconds + first, *rest = @spans.sort + return 0.0 unless first + + total = 0.0 + span_start, span_end = first + rest.each do |start_time, end_time| + if start_time > span_end + total += span_end - span_start + span_start = start_time + span_end = end_time + elsif end_time > span_end + span_end = end_time + end + end + total + (span_end - span_start) + end + end + end +end diff --git a/temporalio/lib/temporalio/internal/worker/workflow_worker.rb b/temporalio/lib/temporalio/internal/worker/workflow_worker.rb index 50ccce83..48a950c6 100644 --- a/temporalio/lib/temporalio/internal/worker/workflow_worker.rb +++ b/temporalio/lib/temporalio/internal/worker/workflow_worker.rb @@ -4,6 +4,7 @@ require 'temporalio/error' require 'temporalio/internal/bridge/api' require 'temporalio/internal/external_storage_runner' +require 'temporalio/internal/storage_operation_metrics' require 'temporalio/internal/worker/workflow_instance' require 'temporalio/scoped_logger' require 'temporalio/workflow' @@ -90,30 +91,22 @@ def initialize( 'storage and not using fibers' end - # If there is a payload codec or external storage, we need to build encoding and decoding visitors. Storage - # comes after the codec when encoding and before it when decoding. These may block, so they run in the - # background. - storage_runner = ExternalStorageRunner.new(external_storage, namespace:) + # If there is a payload codec or external storage, payloads are encoded and decoded by visitors built per + # pass, so each pass can measure its own storage operations. Storage comes after the codec when encoding and + # before it when decoding. These may block, so they run in the background. + @payload_codec = payload_codec + @external_storage = external_storage + @storage_runner = ExternalStorageRunner.new(external_storage, namespace:) @decode_in_background = !payload_codec.nil? || !external_storage.nil? - if @decode_in_background - @payload_encoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| - replace_payloads_on_visit(payload_or_payloads) do |payloads| - payloads = payload_codec.encode(payloads) if payload_codec - payloads = storage_runner.store(payloads.to_a) if external_storage - payloads - end - end - @payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| - replace_payloads_on_visit(payload_or_payloads) do |payloads| - payloads = storage_runner.retrieve(payloads.to_a) - payloads = payload_codec.decode(payloads) if payload_codec - payloads - end - end - else + # Download metrics by run ID, waiting for the completion of the activation that measured them. Core never has + # two activations of one run outstanding, so the run ID identifies the activation. + @pending_download_metrics = {} + @pending_download_metrics_mutex = Mutex.new + unless @decode_in_background # Without a codec or storage, decoding only rejects references that were never retrieved. That needs no # I/O, so it runs inline and existing workers need no thread pool. - @payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + storage_runner = @storage_runner + @payload_rejecting_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| storage_runner.retrieve( payload_or_payloads.is_a?(Api::Common::V1::Payload) ? [payload_or_payloads] : payload_or_payloads.to_a ) @@ -176,7 +169,7 @@ def handle_activation(runner:, activation:, decoded:) # Returns whether completion was submitted to the bridge # rubocop:disable-next Naming/PredicateMethod def handle_activation_complete(runner:, activation_completion:, encoded:, completion_complete_queue:) - if @payload_encoding_visitor && !encoded + if @decode_in_background && !encoded if Fiber.current_scheduler Fiber.schedule { encode_activation_completion(runner, activation_completion) } else @@ -203,7 +196,17 @@ def on_shutdown_complete # activation as failed. Otherwise the run waits forever for a completion that never comes. def decode_activation(runner, activation) - @payload_decoding_visitor.run(activation) + if @decode_in_background + metrics = nil + if @external_storage + metrics = StorageOperationMetrics.new + # Stored before the visit, so the completion reports what was retrieved even if decoding then fails + @pending_download_metrics_mutex.synchronize { @pending_download_metrics[activation.run_id] = metrics } + end + payload_decoding_visitor(metrics).run(activation) + else + @payload_rejecting_visitor&.run(activation) + end runner.apply_workflow_activation_decoded(workflow_worker: self, activation:) rescue Exception => e # rubocop:disable Lint/RescueException activation_completion = WorkflowInstance.new_completion_with_failure( @@ -217,18 +220,51 @@ def decode_activation(runner, activation) end def encode_activation_completion(runner, activation_completion) - @payload_encoding_visitor.run(activation_completion) - runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true) - rescue Exception => e # rubocop:disable Lint/RescueException - # Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads could - # fail the same way and retry forever - activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new( - run_id: activation_completion.run_id, - failed: Bridge::Api::WorkflowCompletion::Failure.new( - failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}") + download_metrics = @pending_download_metrics_mutex.synchronize do + @pending_download_metrics.delete(activation_completion.run_id) + end + upload_metrics = StorageOperationMetrics.new if @external_storage + begin + payload_encoding_visitor(upload_metrics).run(activation_completion) + rescue Exception => e # rubocop:disable Lint/RescueException + # Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads + # could fail the same way and retry forever + activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new( + run_id: activation_completion.run_id, + failed: Bridge::Api::WorkflowCompletion::Failure.new( + failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}") + ) ) - ) - runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: false) + end + # Reported even when a pass failed, since what it moved before failing still took time + activation_completion.payload_download_metrics = download_metrics&.to_proto + activation_completion.payload_upload_metrics = upload_metrics&.to_proto + runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true) + end + + def payload_encoding_visitor(metrics) + payload_codec = @payload_codec + external_storage = @external_storage + storage_runner = @storage_runner + Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + replace_payloads_on_visit(payload_or_payloads) do |payloads| + payloads = payload_codec.encode(payloads) if payload_codec + payloads = storage_runner.store(payloads.to_a, metrics:) if external_storage + payloads + end + end + end + + def payload_decoding_visitor(metrics) + payload_codec = @payload_codec + storage_runner = @storage_runner + Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + replace_payloads_on_visit(payload_or_payloads) do |payloads| + payloads = storage_runner.retrieve(payloads.to_a, metrics:) + payloads = payload_codec.decode(payloads) if payload_codec + payloads + end + end end def replace_payloads_on_visit(payload_or_payloads, &) diff --git a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi index 8e79afcb..7a119b10 100644 --- a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi +++ b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi @@ -9,14 +9,18 @@ class Temporalio::Internal::ExternalStorageRunner def initialize(storage, namespace:); end sig do - params(payloads: T::Array[Temporalio::Api::Common::V1::Payload]) - .returns(T::Array[Temporalio::Api::Common::V1::Payload]) + params( + payloads: T::Array[Temporalio::Api::Common::V1::Payload], + metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics) + ).returns(T::Array[Temporalio::Api::Common::V1::Payload]) end - def store(payloads); end + def store(payloads, metrics: nil); end sig do - params(payloads: T::Array[Temporalio::Api::Common::V1::Payload]) - .returns(T::Array[Temporalio::Api::Common::V1::Payload]) + params( + payloads: T::Array[Temporalio::Api::Common::V1::Payload], + metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics) + ).returns(T::Array[Temporalio::Api::Common::V1::Payload]) end - def retrieve(payloads); end + def retrieve(payloads, metrics: nil); end end diff --git a/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi b/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi new file mode 100644 index 00000000..68bfc267 --- /dev/null +++ b/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi @@ -0,0 +1,27 @@ +# typed: true + +class Temporalio::Internal::StorageOperationMetrics + extend T::Sig + + sig { void } + def initialize; end + + sig do + params( + driver_name: String, + payload_count: Integer, + total_size_bytes: Integer, + start_time: Float, + end_time: Float + ).void + end + def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:); end + + sig { returns(T.nilable(Temporalio::Internal::Bridge::Api::Common::ExternalStorageMetrics)) } + def to_proto; end + + private + + sig { returns(Float) } + def union_seconds; end +end diff --git a/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi b/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi index fbf1b985..3302afd3 100644 --- a/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi +++ b/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi @@ -92,6 +92,12 @@ class Temporalio::Internal::Worker::WorkflowWorker sig { params(runner: Temporalio::Internal::Worker::MultiRunner, activation_completion: Object).void } def encode_activation_completion(runner, activation_completion); end + sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) } + def payload_encoding_visitor(metrics); end + + sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) } + def payload_decoding_visitor(metrics); end + sig { params(payload_or_payloads: Object, block: T.proc.params(arg0: Object).returns(Object)).void } def replace_payloads_on_visit(payload_or_payloads, &block); end end diff --git a/temporalio/sig/temporalio/internal/external_storage_runner.rbs b/temporalio/sig/temporalio/internal/external_storage_runner.rbs index 9aa2f9f9..7d2757a7 100644 --- a/temporalio/sig/temporalio/internal/external_storage_runner.rbs +++ b/temporalio/sig/temporalio/internal/external_storage_runner.rbs @@ -6,8 +6,14 @@ module Temporalio def initialize: (Converters::ExternalStorage? storage, namespace: String?) -> void - def store: (Array[Api::Common::V1::Payload] payloads) -> Array[Api::Common::V1::Payload] - def retrieve: (Array[Api::Common::V1::Payload] payloads) -> Array[Api::Common::V1::Payload] + def store: ( + Array[Api::Common::V1::Payload] payloads, + ?metrics: StorageOperationMetrics? + ) -> Array[Api::Common::V1::Payload] + def retrieve: ( + Array[Api::Common::V1::Payload] payloads, + ?metrics: StorageOperationMetrics? + ) -> Array[Api::Common::V1::Payload] private def check_result_count: ( Converters::StorageDriver driver, diff --git a/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs b/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs new file mode 100644 index 00000000..a09bffa2 --- /dev/null +++ b/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs @@ -0,0 +1,25 @@ +module Temporalio + module Internal + class StorageOperationMetrics + @mutex: Mutex + @payload_count: Integer + @total_size_bytes: Integer + @driver_names: Set[String] + @spans: Array[[Float, Float]] + + def initialize: -> void + + def record: ( + driver_name: String, + payload_count: Integer, + total_size_bytes: Integer, + start_time: Float, + end_time: Float + ) -> void + + def to_proto: -> Bridge::Api::Common::ExternalStorageMetrics? + + private def union_seconds: -> Float + end + end +end diff --git a/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs b/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs index b584edb5..c434c02a 100644 --- a/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs +++ b/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs @@ -50,6 +50,8 @@ module Temporalio def decode_activation: (MultiRunner runner, untyped activation) -> void def encode_activation_completion: (MultiRunner runner, untyped activation_completion) -> void + def payload_encoding_visitor: (StorageOperationMetrics? metrics) -> Api::PayloadVisitor + def payload_decoding_visitor: (StorageOperationMetrics? metrics) -> Api::PayloadVisitor def replace_payloads_on_visit: (untyped payload_or_payloads) { (untyped) -> Enumerable[untyped] } -> void class State diff --git a/temporalio/test/external_storage_runner_test.rb b/temporalio/test/external_storage_runner_test.rb index 3e9d56a8..b491395d 100644 --- a/temporalio/test/external_storage_runner_test.rb +++ b/temporalio/test/external_storage_runner_test.rb @@ -5,6 +5,7 @@ require 'temporalio/error' require 'temporalio/internal/external_storage_references' require 'temporalio/internal/external_storage_runner' +require 'temporalio/internal/storage_operation_metrics' require 'test' class ExternalStorageRunnerTest < Test @@ -129,6 +130,52 @@ def test_without_storage_retrieve_raises_on_reference assert_raises(Temporalio::Error::ExternalStorageNotConfiguredError) { Runner.new(nil, namespace: nil).retrieve([reference]) } end + def test_store_and_retrieve_record_metrics_for_each_driver + delay = 0.05 + one = InMemoryStorageDriver.new(name: 'one', delay:) + two = InMemoryStorageDriver.new(name: 'two', delay:) + runner = Runner.new(Temporalio::Converters::ExternalStorage.new( + drivers: [two, one], + driver_selector: ->(_context, payload) { payload.data == '"a"' ? one : two }, + payload_size_threshold: 0 + ), namespace: nil) + originals = [payload('a'), payload('b'), payload('c')] + original_size = originals.sum { |p| p.to_proto.bytesize } + + store_metrics = Temporalio::Internal::StorageOperationMetrics.new + stored = runner.store(originals, metrics: store_metrics) + retrieve_metrics = Temporalio::Internal::StorageOperationMetrics.new + runner.retrieve(stored, metrics: retrieve_metrics) + + [store_metrics, retrieve_metrics].each do |metrics| + proto = metrics.to_proto or flunk('Expected metrics') + assert_equal 3, proto.payload_count + assert_equal original_size, proto.total_size_bytes + assert_equal %w[one two], proto.driver_names.to_a + # Drivers are called one after another, so both calls' time counts + assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(proto.total_duration).to_f, :>=, 2 * delay + end + end + + def test_metrics_are_unset_when_nothing_is_moved + runner = Runner.new(storage(InMemoryStorageDriver.new, payload_size_threshold: 1_000_000), namespace: nil) + metrics = Temporalio::Internal::StorageOperationMetrics.new + + runner.retrieve(runner.store([payload('a')], metrics:), metrics:) + + assert_nil metrics.to_proto + end + + def test_metrics_duration_counts_overlapping_calls_once + metrics = Temporalio::Internal::StorageOperationMetrics.new + metrics.record(driver_name: 'a', payload_count: 1, total_size_bytes: 1, start_time: 10.0, end_time: 12.0) + metrics.record(driver_name: 'b', payload_count: 1, total_size_bytes: 1, start_time: 11.0, end_time: 13.0) + metrics.record(driver_name: 'a', payload_count: 1, total_size_bytes: 1, start_time: 20.0, end_time: 20.5) + + duration = metrics.to_proto&.total_duration or flunk('Expected a duration') + assert_equal 3.5, Temporalio::Internal::ProtoUtils.duration_to_seconds(duration) + end + private def storage(driver, payload_size_threshold: 0) diff --git a/temporalio/test/in_memory_storage_driver.rb b/temporalio/test/in_memory_storage_driver.rb index 1aacef35..80ba7bd3 100644 --- a/temporalio/test/in_memory_storage_driver.rb +++ b/temporalio/test/in_memory_storage_driver.rb @@ -13,10 +13,12 @@ class InMemoryStorageDriver < Temporalio::Converters::StorageDriver attr_reader :name, :type - def initialize(name: 'mem', type: 'test.memdriver') + # @param delay [Float] Seconds each store and retrieve sleeps, so tests can assert a lower bound on measured time. + def initialize(name: 'mem', type: 'test.memdriver', delay: 0) super() @name = name @type = type + @delay = delay @mutex = Mutex.new @stored = {} @store_calls = [] @@ -37,6 +39,7 @@ def retrieve_calls def store(context, payloads) @mutex.synchronize { @store_calls << [context, payloads.size] } + sleep(@delay) if @delay.positive? payloads.map do |payload| key = SecureRandom.uuid @mutex.synchronize { @stored[key] = payload.to_proto } @@ -46,6 +49,7 @@ def store(context, payloads) def retrieve(_context, claims) @mutex.synchronize { @retrieve_calls << claims.size } + sleep(@delay) if @delay.positive? claims.map do |claim| bytes = @mutex.synchronize { @stored.fetch(claim.claim_data.fetch('key')) } Temporalio::Api::Common::V1::Payload.decode(bytes) diff --git a/temporalio/test/sig/worker_external_storage_test.rbs b/temporalio/test/sig/worker_external_storage_test.rbs index a089d6a1..88ff88dd 100644 --- a/temporalio/test/sig/worker_external_storage_test.rbs +++ b/temporalio/test/sig/worker_external_storage_test.rbs @@ -80,6 +80,9 @@ class WorkerExternalStorageTest < Test def change: (untyped input) -> untyped end + def run_echo_workflow_capturing_completions: ( + Temporalio::Converters::StorageDriver driver + ) -> Array[Temporalio::Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion] def assert_storage_failure_retried: (Temporalio::Converters::StorageDriver driver, String failure_message) -> void def client_with_storage: ( Temporalio::Converters::StorageDriver driver, diff --git a/temporalio/test/worker_external_storage_test.rb b/temporalio/test/worker_external_storage_test.rb index 94ad1e90..0ec9d2d6 100644 --- a/temporalio/test/worker_external_storage_test.rb +++ b/temporalio/test/worker_external_storage_test.rb @@ -456,8 +456,61 @@ def test_worker_without_storage_fails_the_task_on_an_update_reference end end + def test_workflow_task_completions_report_storage_metrics + delay = 0.05 + driver = InMemoryStorageDriver.new(delay:) + completions = run_echo_workflow_capturing_completions(driver) + + # The first task retrieves the input and stores the result + first = completions.first or flunk('No completions') + stored_sizes = driver.stored.values.map(&:bytesize).sort + download = first.payload_download_metrics or flunk('No download metrics') + upload = first.payload_upload_metrics or flunk('No upload metrics') + assert_equal 1, download.payload_count + assert_equal 1, upload.payload_count + assert_equal stored_sizes, [download.total_size_bytes, upload.total_size_bytes].sort + assert_equal ['mem'], download.driver_names.to_a + assert_equal ['mem'], upload.driver_names.to_a + assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(download.total_duration).to_f, :>=, delay + assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(upload.total_duration).to_f, :>=, delay + end + + def test_failed_store_still_reports_download_metrics + completions = run_echo_workflow_capturing_completions(FailOnceStorageDriver.new(fail_store_containing: 'done')) + + failed = completions.find(&:failed) or flunk('No failed completion') + download = failed.payload_download_metrics or flunk('No download metrics') + assert_equal 1, download.payload_count + assert_equal ['mem'], download.driver_names.to_a + assert_nil failed.payload_upload_metrics + end + private + def run_echo_workflow_capturing_completions(driver) + client = client_with_storage(driver) + worker = Temporalio::Worker.new( + client:, + task_queue: "tq-#{SecureRandom.uuid}", + workflows: [EchoWorkflow], + workflow_payload_codec_thread_pool: Temporalio::Worker::ThreadPool.default + ) + completions = Queue.new + bridge_worker = worker._bridge_worker + complete = bridge_worker.method(:async_complete_workflow_activation) + bridge_worker.define_singleton_method(:async_complete_workflow_activation) do |run_id, proto, queue| + completions << Temporalio::Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.decode(proto) + complete.call(run_id, proto, queue) + end + worker.run do + result = client.execute_workflow( + EchoWorkflow, 'start-input', id: "wf-#{SecureRandom.uuid}", task_queue: worker.task_queue + ) + assert_equal 'start-input done', result + end + Array.new(completions.size) { completions.pop } + end + def assert_storage_failure_retried(driver, failure_message) execute_workflow( EchoWorkflow, 'start-input', From 03d470fe64ce21710f5daf138e38b52630c5b3a9 Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:39:00 -0800 Subject: [PATCH 2/4] refactor(extstore): build the storage metrics proto outside the lock --- .../internal/storage_operation_metrics.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/temporalio/lib/temporalio/internal/storage_operation_metrics.rb b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb index 5c1bf503..f78f352e 100644 --- a/temporalio/lib/temporalio/internal/storage_operation_metrics.rb +++ b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb @@ -37,16 +37,19 @@ def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_tim # @return [Bridge::Api::Common::ExternalStorageMetrics, nil] Metrics for the pass, or nil when it moved nothing, # so the completion field stays unset. def to_proto - @mutex.synchronize do + payload_count, total_size_bytes, duration_seconds, driver_names = @mutex.synchronize do return nil if @payload_count.zero? - Bridge::Api::Common::ExternalStorageMetrics.new( - payload_count: @payload_count, - total_size_bytes: @total_size_bytes, - total_duration: ProtoUtils.seconds_to_duration(union_seconds), - driver_names: @driver_names.sort - ) + [@payload_count, @total_size_bytes, union_seconds, @driver_names.sort] end + # Built outside the lock, since creating a protobuf message takes a process-wide lock that would otherwise also + # hold up record + Bridge::Api::Common::ExternalStorageMetrics.new( + payload_count:, + total_size_bytes:, + total_duration: ProtoUtils.seconds_to_duration(duration_seconds), + driver_names: + ) end private From a3351c8b3ddddfee90e5e30c4e12e0987018f86c Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:52:25 -0800 Subject: [PATCH 3/4] test(extstore): replace driver sleeps with an injected clock --- .../internal/external_storage_runner.rb | 15 ++++--- .../internal/external_storage_runner.rbi | 8 +++- .../internal/external_storage_runner.rbs | 5 ++- .../test/external_storage_runner_test.rb | 44 ++++++++++++++++--- temporalio/test/external_storage_test.rb | 4 +- temporalio/test/in_memory_storage_driver.rb | 10 ++--- .../test/sig/external_storage_runner_test.rbs | 14 ++++++ .../test/sig/in_memory_storage_driver.rbs | 17 +++++++ .../test/worker_external_storage_test.rb | 5 +-- 9 files changed, 96 insertions(+), 26 deletions(-) create mode 100644 temporalio/test/sig/in_memory_storage_driver.rbs diff --git a/temporalio/lib/temporalio/internal/external_storage_runner.rb b/temporalio/lib/temporalio/internal/external_storage_runner.rb index e508daf7..a6a3f8e9 100644 --- a/temporalio/lib/temporalio/internal/external_storage_runner.rb +++ b/temporalio/lib/temporalio/internal/external_storage_runner.rb @@ -24,13 +24,18 @@ module Internal # # @!visibility private class ExternalStorageRunner + MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + private_constant :MONOTONIC_CLOCK + # @param storage [Converters::ExternalStorage, nil] Configured storage. When nil, storing does nothing and # retrieving raises if it finds a reference, so a reference is never mistaken for an ordinary payload. # @param namespace [String, nil] Namespace of the client or worker storing through this runner, or nil when it is # not known. - def initialize(storage, namespace:) + # @param clock [Proc] Returns monotonic seconds, read around each driver call to time it. + def initialize(storage, namespace:, clock: MONOTONIC_CLOCK) @storage = storage @namespace = namespace + @clock = clock end # @param payloads [Array] Codec-encoded payloads. Not mutated. @@ -71,9 +76,9 @@ def store(payloads, metrics: nil) # Drivers get copies, since a driver may keep a payload after returning and the SDK modifies some payloads in # place copies = indexes_and_serialized.map { |_index, serialized| Api::Common::V1::Payload.decode(serialized) } - start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start_time = @clock.call claims = driver.store(context, copies) - end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + end_time = @clock.call check_result_count(driver, 'store', indexes_and_serialized.size, claims) metrics&.record( driver_name: driver.name, @@ -117,9 +122,9 @@ def retrieve(payloads, metrics: nil) context = Converters::StorageDriverRetrieveContext.new( cancellation: Cancellation.new, limiter: UNBOUNDED_LIMITER ) - start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + start_time = @clock.call retrieved = driver.retrieve(context, indexes_and_claims.map { |_index, claim| claim }) - end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + end_time = @clock.call check_result_count(driver, 'retrieve', indexes_and_claims.size, retrieved) metrics&.record( driver_name: driver.name, diff --git a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi index 7a119b10..295adb3c 100644 --- a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi +++ b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi @@ -4,9 +4,13 @@ class Temporalio::Internal::ExternalStorageRunner extend T::Sig sig do - params(storage: T.nilable(Temporalio::Converters::ExternalStorage), namespace: T.nilable(String)).void + params( + storage: T.nilable(Temporalio::Converters::ExternalStorage), + namespace: T.nilable(String), + clock: T.proc.returns(Float) + ).void end - def initialize(storage, namespace:); end + def initialize(storage, namespace:, clock: T.unsafe(nil)); end sig do params( diff --git a/temporalio/sig/temporalio/internal/external_storage_runner.rbs b/temporalio/sig/temporalio/internal/external_storage_runner.rbs index 7d2757a7..1c2707a9 100644 --- a/temporalio/sig/temporalio/internal/external_storage_runner.rbs +++ b/temporalio/sig/temporalio/internal/external_storage_runner.rbs @@ -3,8 +3,11 @@ module Temporalio class ExternalStorageRunner @storage: Converters::ExternalStorage? @namespace: String? + @clock: ^() -> Float - def initialize: (Converters::ExternalStorage? storage, namespace: String?) -> void + MONOTONIC_CLOCK: ^() -> Float + + def initialize: (Converters::ExternalStorage? storage, namespace: String?, ?clock: ^() -> Float) -> void def store: ( Array[Api::Common::V1::Payload] payloads, diff --git a/temporalio/test/external_storage_runner_test.rb b/temporalio/test/external_storage_runner_test.rb index b491395d..35317121 100644 --- a/temporalio/test/external_storage_runner_test.rb +++ b/temporalio/test/external_storage_runner_test.rb @@ -12,6 +12,39 @@ class ExternalStorageRunnerTest < Test Runner = Temporalio::Internal::ExternalStorageRunner References = Temporalio::Internal::ExternalStorageReferences + class FakeClock + def initialize + @now = 0.0 + end + + def call + @now + end + + def advance(seconds) + @now += seconds + end + end + + # Moves the fake clock forward during each call, so the time measured around a call is exactly the given seconds. + class ClockAdvancingDriver < InMemoryStorageDriver + def initialize(clock:, seconds:, name:) + super(name:) + @clock = clock + @seconds = seconds + end + + def store(context, payloads) + @clock.advance(@seconds) + super + end + + def retrieve(context, claims) + @clock.advance(@seconds) + super + end + end + def test_store_replaces_only_payloads_at_or_above_threshold driver = InMemoryStorageDriver.new small = payload('a') @@ -131,14 +164,14 @@ def test_without_storage_retrieve_raises_on_reference end def test_store_and_retrieve_record_metrics_for_each_driver - delay = 0.05 - one = InMemoryStorageDriver.new(name: 'one', delay:) - two = InMemoryStorageDriver.new(name: 'two', delay:) + clock = FakeClock.new + one = ClockAdvancingDriver.new(clock:, seconds: 2.0, name: 'one') + two = ClockAdvancingDriver.new(clock:, seconds: 3.0, name: 'two') runner = Runner.new(Temporalio::Converters::ExternalStorage.new( drivers: [two, one], driver_selector: ->(_context, payload) { payload.data == '"a"' ? one : two }, payload_size_threshold: 0 - ), namespace: nil) + ), namespace: nil, clock:) originals = [payload('a'), payload('b'), payload('c')] original_size = originals.sum { |p| p.to_proto.bytesize } @@ -152,8 +185,7 @@ def test_store_and_retrieve_record_metrics_for_each_driver assert_equal 3, proto.payload_count assert_equal original_size, proto.total_size_bytes assert_equal %w[one two], proto.driver_names.to_a - # Drivers are called one after another, so both calls' time counts - assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(proto.total_duration).to_f, :>=, 2 * delay + assert_equal 5.0, Temporalio::Internal::ProtoUtils.duration_to_seconds(proto.total_duration) end end diff --git a/temporalio/test/external_storage_test.rb b/temporalio/test/external_storage_test.rb index 862b211e..b23f1087 100644 --- a/temporalio/test/external_storage_test.rb +++ b/temporalio/test/external_storage_test.rb @@ -121,7 +121,9 @@ def test_empty_driver_name_rejected def test_symbol_driver_name_rejected # A Symbol passes a bare empty? check but cannot be looked up by String or written to the proto. err = assert_raises(ArgumentError) do - Temporalio::Converters::ExternalStorage.new(drivers: [InMemoryStorageDriver.new(name: :mem)]) + Temporalio::Converters::ExternalStorage.new( + drivers: [InMemoryStorageDriver.new(name: :mem)] # steep:ignore ArgumentTypeMismatch + ) end assert_includes err.message, 'must be a non-empty String' end diff --git a/temporalio/test/in_memory_storage_driver.rb b/temporalio/test/in_memory_storage_driver.rb index 80ba7bd3..eaf5a32b 100644 --- a/temporalio/test/in_memory_storage_driver.rb +++ b/temporalio/test/in_memory_storage_driver.rb @@ -13,12 +13,10 @@ class InMemoryStorageDriver < Temporalio::Converters::StorageDriver attr_reader :name, :type - # @param delay [Float] Seconds each store and retrieve sleeps, so tests can assert a lower bound on measured time. - def initialize(name: 'mem', type: 'test.memdriver', delay: 0) + def initialize(name: 'mem', type: 'test.memdriver') super() @name = name @type = type - @delay = delay @mutex = Mutex.new @stored = {} @store_calls = [] @@ -38,8 +36,7 @@ def retrieve_calls end def store(context, payloads) - @mutex.synchronize { @store_calls << [context, payloads.size] } - sleep(@delay) if @delay.positive? + @mutex.synchronize { @store_calls << [context, payloads.count] } payloads.map do |payload| key = SecureRandom.uuid @mutex.synchronize { @stored[key] = payload.to_proto } @@ -48,8 +45,7 @@ def store(context, payloads) end def retrieve(_context, claims) - @mutex.synchronize { @retrieve_calls << claims.size } - sleep(@delay) if @delay.positive? + @mutex.synchronize { @retrieve_calls << claims.count } claims.map do |claim| bytes = @mutex.synchronize { @stored.fetch(claim.claim_data.fetch('key')) } Temporalio::Api::Common::V1::Payload.decode(bytes) diff --git a/temporalio/test/sig/external_storage_runner_test.rbs b/temporalio/test/sig/external_storage_runner_test.rbs index 56bc96cd..738f24b9 100644 --- a/temporalio/test/sig/external_storage_runner_test.rbs +++ b/temporalio/test/sig/external_storage_runner_test.rbs @@ -1,4 +1,18 @@ class ExternalStorageRunnerTest < Test + class FakeClock + @now: Float + + def call: -> Float + def advance: (Float seconds) -> Float + end + + class ClockAdvancingDriver < InMemoryStorageDriver + @clock: FakeClock + @seconds: Float + + def initialize: (clock: FakeClock, seconds: Float, name: String) -> void + end + def storage: ( Temporalio::Converters::StorageDriver driver, ?payload_size_threshold: Integer diff --git a/temporalio/test/sig/in_memory_storage_driver.rbs b/temporalio/test/sig/in_memory_storage_driver.rbs new file mode 100644 index 00000000..9b116dbb --- /dev/null +++ b/temporalio/test/sig/in_memory_storage_driver.rbs @@ -0,0 +1,17 @@ +class InMemoryStorageDriver < Temporalio::Converters::StorageDriver + @name: String + @type: String + @mutex: Mutex + @stored: Hash[String, String] + @store_calls: Array[[Temporalio::Converters::StorageDriverStoreContext, Integer]] + @retrieve_calls: Array[Integer] + + attr_reader name: String + attr_reader type: String + + def initialize: (?name: String, ?type: String) -> void + + def stored: -> Hash[String, String] + def store_calls: -> Array[[Temporalio::Converters::StorageDriverStoreContext, Integer]] + def retrieve_calls: -> Array[Integer] +end diff --git a/temporalio/test/worker_external_storage_test.rb b/temporalio/test/worker_external_storage_test.rb index 0ec9d2d6..48ceae54 100644 --- a/temporalio/test/worker_external_storage_test.rb +++ b/temporalio/test/worker_external_storage_test.rb @@ -457,8 +457,7 @@ def test_worker_without_storage_fails_the_task_on_an_update_reference end def test_workflow_task_completions_report_storage_metrics - delay = 0.05 - driver = InMemoryStorageDriver.new(delay:) + driver = InMemoryStorageDriver.new completions = run_echo_workflow_capturing_completions(driver) # The first task retrieves the input and stores the result @@ -471,8 +470,6 @@ def test_workflow_task_completions_report_storage_metrics assert_equal stored_sizes, [download.total_size_bytes, upload.total_size_bytes].sort assert_equal ['mem'], download.driver_names.to_a assert_equal ['mem'], upload.driver_names.to_a - assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(download.total_duration).to_f, :>=, delay - assert_operator Temporalio::Internal::ProtoUtils.duration_to_seconds(upload.total_duration).to_f, :>=, delay end def test_failed_store_still_reports_download_metrics From c8a6f4b940c1da6a985abacdf11726a44f29cea6 Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:38:30 -0800 Subject: [PATCH 4/4] test(extstore): pass the fake clock to the runner as a lambda --- temporalio/test/external_storage_runner_test.rb | 8 +++----- temporalio/test/sig/external_storage_runner_test.rbs | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/temporalio/test/external_storage_runner_test.rb b/temporalio/test/external_storage_runner_test.rb index 35317121..686aff97 100644 --- a/temporalio/test/external_storage_runner_test.rb +++ b/temporalio/test/external_storage_runner_test.rb @@ -13,14 +13,12 @@ class ExternalStorageRunnerTest < Test References = Temporalio::Internal::ExternalStorageReferences class FakeClock + attr_reader :now + def initialize @now = 0.0 end - def call - @now - end - def advance(seconds) @now += seconds end @@ -171,7 +169,7 @@ def test_store_and_retrieve_record_metrics_for_each_driver drivers: [two, one], driver_selector: ->(_context, payload) { payload.data == '"a"' ? one : two }, payload_size_threshold: 0 - ), namespace: nil, clock:) + ), namespace: nil, clock: -> { clock.now }) originals = [payload('a'), payload('b'), payload('c')] original_size = originals.sum { |p| p.to_proto.bytesize } diff --git a/temporalio/test/sig/external_storage_runner_test.rbs b/temporalio/test/sig/external_storage_runner_test.rbs index 738f24b9..30d25c62 100644 --- a/temporalio/test/sig/external_storage_runner_test.rbs +++ b/temporalio/test/sig/external_storage_runner_test.rbs @@ -2,7 +2,7 @@ class ExternalStorageRunnerTest < Test class FakeClock @now: Float - def call: -> Float + attr_reader now: Float def advance: (Float seconds) -> Float end