diff --git a/temporalio/lib/temporalio/internal/external_storage_runner.rb b/temporalio/lib/temporalio/internal/external_storage_runner.rb index e25ed3a5..a6a3f8e9 100644 --- a/temporalio/lib/temporalio/internal/external_storage_runner.rb +++ b/temporalio/lib/temporalio/internal/external_storage_runner.rb @@ -9,6 +9,7 @@ require 'temporalio/converters/storage_driver_store_context' require 'temporalio/error' require 'temporalio/internal/external_storage_references' +require 'temporalio/internal/storage_operation_metrics' module Temporalio module Internal @@ -23,18 +24,24 @@ module Internal # # @!visibility private class ExternalStorageRunner + MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + private_constant :MONOTONIC_CLOCK + # @param storage [Converters::ExternalStorage, nil] Configured storage. When nil, storing does nothing and # retrieving raises if it finds a reference, so a reference is never mistaken for an ordinary payload. # @param namespace [String, nil] Namespace of the client or worker storing through this runner, or nil when it is # not known. - def initialize(storage, namespace:) + # @param clock [Proc] Returns monotonic seconds, read around each driver call to time it. + def initialize(storage, namespace:, clock: MONOTONIC_CLOCK) @storage = storage @namespace = namespace + @clock = clock end # @param payloads [Array] Codec-encoded payloads. Not mutated. + # @param metrics [StorageOperationMetrics, nil] Records each driver call that stores payloads. # @return [Array] Payloads with every eligible one replaced by a reference. - def store(payloads) + def store(payloads, metrics: nil) storage = @storage return payloads unless storage @@ -69,8 +76,17 @@ def store(payloads) # Drivers get copies, since a driver may keep a payload after returning and the SDK modifies some payloads in # place copies = indexes_and_serialized.map { |_index, serialized| Api::Common::V1::Payload.decode(serialized) } + start_time = @clock.call claims = driver.store(context, copies) + end_time = @clock.call check_result_count(driver, 'store', indexes_and_serialized.size, claims) + metrics&.record( + driver_name: driver.name, + payload_count: claims.size, + total_size_bytes: indexes_and_serialized.sum { |_index, serialized| serialized.bytesize }, + start_time:, + end_time: + ) indexes_and_serialized.zip(claims) do |(index, serialized), claim| result[index] = ExternalStorageReferences.create_reference_payload( @@ -82,9 +98,10 @@ def store(payloads) end # @param payloads [Array] Payloads that may contain references. Not mutated. + # @param metrics [StorageOperationMetrics, nil] Records each driver call that retrieves payloads. # @return [Array] Payloads with every reference replaced by what it points at. # @raise [Error::ExternalStorageNotConfiguredError] If there is a reference but no storage is configured. - def retrieve(payloads) + def retrieve(payloads, metrics: nil) storage = @storage # Driver => [[index, claim], ...] for each reference that driver stored. Every driver is looked up here, so an # unregistered name fails before any request is made. @@ -105,8 +122,17 @@ def retrieve(payloads) context = Converters::StorageDriverRetrieveContext.new( cancellation: Cancellation.new, limiter: UNBOUNDED_LIMITER ) + start_time = @clock.call retrieved = driver.retrieve(context, indexes_and_claims.map { |_index, claim| claim }) + end_time = @clock.call check_result_count(driver, 'retrieve', indexes_and_claims.size, retrieved) + metrics&.record( + driver_name: driver.name, + payload_count: retrieved.size, + total_size_bytes: retrieved.sum { |payload| payload.to_proto.bytesize }, + start_time:, + end_time: + ) indexes_and_claims.zip(retrieved) { |(index, _claim), payload| result[index] = payload } end diff --git a/temporalio/lib/temporalio/internal/storage_operation_metrics.rb b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb new file mode 100644 index 00000000..f78f352e --- /dev/null +++ b/temporalio/lib/temporalio/internal/storage_operation_metrics.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'temporalio/internal/bridge/api' +require 'temporalio/internal/proto_utils' + +module Temporalio + module Internal + # Gathers what external storage moved during one store or retrieve pass, for core to report on the activation + # completion. + # + # @!visibility private + class StorageOperationMetrics + def initialize + @mutex = Mutex.new + @payload_count = 0 + @total_size_bytes = 0 + @driver_names = Set.new + @spans = [] + end + + # Record one driver call that moved payloads. + # + # @param driver_name [String] Driver that stored or retrieved the payloads. + # @param payload_count [Integer] Number of payloads moved. + # @param total_size_bytes [Integer] Encoded size of the payloads moved. + # @param start_time [Float] Monotonic clock reading taken before the driver call. + # @param end_time [Float] Monotonic clock reading taken after the driver call. + def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:) + @mutex.synchronize do + @driver_names << driver_name + @payload_count += payload_count + @total_size_bytes += total_size_bytes + @spans << [start_time, end_time] + end + end + + # @return [Bridge::Api::Common::ExternalStorageMetrics, nil] Metrics for the pass, or nil when it moved nothing, + # so the completion field stays unset. + def to_proto + payload_count, total_size_bytes, duration_seconds, driver_names = @mutex.synchronize do + return nil if @payload_count.zero? + + [@payload_count, @total_size_bytes, union_seconds, @driver_names.sort] + end + # Built outside the lock, since creating a protobuf message takes a process-wide lock that would otherwise also + # hold up record + Bridge::Api::Common::ExternalStorageMetrics.new( + payload_count:, + total_size_bytes:, + total_duration: ProtoUtils.seconds_to_duration(duration_seconds), + driver_names: + ) + end + + private + + def union_seconds + first, *rest = @spans.sort + return 0.0 unless first + + total = 0.0 + span_start, span_end = first + rest.each do |start_time, end_time| + if start_time > span_end + total += span_end - span_start + span_start = start_time + span_end = end_time + elsif end_time > span_end + span_end = end_time + end + end + total + (span_end - span_start) + end + end + end +end diff --git a/temporalio/lib/temporalio/internal/worker/workflow_worker.rb b/temporalio/lib/temporalio/internal/worker/workflow_worker.rb index 50ccce83..48a950c6 100644 --- a/temporalio/lib/temporalio/internal/worker/workflow_worker.rb +++ b/temporalio/lib/temporalio/internal/worker/workflow_worker.rb @@ -4,6 +4,7 @@ require 'temporalio/error' require 'temporalio/internal/bridge/api' require 'temporalio/internal/external_storage_runner' +require 'temporalio/internal/storage_operation_metrics' require 'temporalio/internal/worker/workflow_instance' require 'temporalio/scoped_logger' require 'temporalio/workflow' @@ -90,30 +91,22 @@ def initialize( 'storage and not using fibers' end - # If there is a payload codec or external storage, we need to build encoding and decoding visitors. Storage - # comes after the codec when encoding and before it when decoding. These may block, so they run in the - # background. - storage_runner = ExternalStorageRunner.new(external_storage, namespace:) + # If there is a payload codec or external storage, payloads are encoded and decoded by visitors built per + # pass, so each pass can measure its own storage operations. Storage comes after the codec when encoding and + # before it when decoding. These may block, so they run in the background. + @payload_codec = payload_codec + @external_storage = external_storage + @storage_runner = ExternalStorageRunner.new(external_storage, namespace:) @decode_in_background = !payload_codec.nil? || !external_storage.nil? - if @decode_in_background - @payload_encoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| - replace_payloads_on_visit(payload_or_payloads) do |payloads| - payloads = payload_codec.encode(payloads) if payload_codec - payloads = storage_runner.store(payloads.to_a) if external_storage - payloads - end - end - @payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| - replace_payloads_on_visit(payload_or_payloads) do |payloads| - payloads = storage_runner.retrieve(payloads.to_a) - payloads = payload_codec.decode(payloads) if payload_codec - payloads - end - end - else + # Download metrics by run ID, waiting for the completion of the activation that measured them. Core never has + # two activations of one run outstanding, so the run ID identifies the activation. + @pending_download_metrics = {} + @pending_download_metrics_mutex = Mutex.new + unless @decode_in_background # Without a codec or storage, decoding only rejects references that were never retrieved. That needs no # I/O, so it runs inline and existing workers need no thread pool. - @payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + storage_runner = @storage_runner + @payload_rejecting_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| storage_runner.retrieve( payload_or_payloads.is_a?(Api::Common::V1::Payload) ? [payload_or_payloads] : payload_or_payloads.to_a ) @@ -176,7 +169,7 @@ def handle_activation(runner:, activation:, decoded:) # Returns whether completion was submitted to the bridge # rubocop:disable-next Naming/PredicateMethod def handle_activation_complete(runner:, activation_completion:, encoded:, completion_complete_queue:) - if @payload_encoding_visitor && !encoded + if @decode_in_background && !encoded if Fiber.current_scheduler Fiber.schedule { encode_activation_completion(runner, activation_completion) } else @@ -203,7 +196,17 @@ def on_shutdown_complete # activation as failed. Otherwise the run waits forever for a completion that never comes. def decode_activation(runner, activation) - @payload_decoding_visitor.run(activation) + if @decode_in_background + metrics = nil + if @external_storage + metrics = StorageOperationMetrics.new + # Stored before the visit, so the completion reports what was retrieved even if decoding then fails + @pending_download_metrics_mutex.synchronize { @pending_download_metrics[activation.run_id] = metrics } + end + payload_decoding_visitor(metrics).run(activation) + else + @payload_rejecting_visitor&.run(activation) + end runner.apply_workflow_activation_decoded(workflow_worker: self, activation:) rescue Exception => e # rubocop:disable Lint/RescueException activation_completion = WorkflowInstance.new_completion_with_failure( @@ -217,18 +220,51 @@ def decode_activation(runner, activation) end def encode_activation_completion(runner, activation_completion) - @payload_encoding_visitor.run(activation_completion) - runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true) - rescue Exception => e # rubocop:disable Lint/RescueException - # Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads could - # fail the same way and retry forever - activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new( - run_id: activation_completion.run_id, - failed: Bridge::Api::WorkflowCompletion::Failure.new( - failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}") + download_metrics = @pending_download_metrics_mutex.synchronize do + @pending_download_metrics.delete(activation_completion.run_id) + end + upload_metrics = StorageOperationMetrics.new if @external_storage + begin + payload_encoding_visitor(upload_metrics).run(activation_completion) + rescue Exception => e # rubocop:disable Lint/RescueException + # Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads + # could fail the same way and retry forever + activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new( + run_id: activation_completion.run_id, + failed: Bridge::Api::WorkflowCompletion::Failure.new( + failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}") + ) ) - ) - runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: false) + end + # Reported even when a pass failed, since what it moved before failing still took time + activation_completion.payload_download_metrics = download_metrics&.to_proto + activation_completion.payload_upload_metrics = upload_metrics&.to_proto + runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true) + end + + def payload_encoding_visitor(metrics) + payload_codec = @payload_codec + external_storage = @external_storage + storage_runner = @storage_runner + Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + replace_payloads_on_visit(payload_or_payloads) do |payloads| + payloads = payload_codec.encode(payloads) if payload_codec + payloads = storage_runner.store(payloads.to_a, metrics:) if external_storage + payloads + end + end + end + + def payload_decoding_visitor(metrics) + payload_codec = @payload_codec + storage_runner = @storage_runner + Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads| + replace_payloads_on_visit(payload_or_payloads) do |payloads| + payloads = storage_runner.retrieve(payloads.to_a, metrics:) + payloads = payload_codec.decode(payloads) if payload_codec + payloads + end + end end def replace_payloads_on_visit(payload_or_payloads, &) diff --git a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi index 8e79afcb..295adb3c 100644 --- a/temporalio/rbi/temporalio/internal/external_storage_runner.rbi +++ b/temporalio/rbi/temporalio/internal/external_storage_runner.rbi @@ -4,19 +4,27 @@ class Temporalio::Internal::ExternalStorageRunner extend T::Sig sig do - params(storage: T.nilable(Temporalio::Converters::ExternalStorage), namespace: T.nilable(String)).void + params( + storage: T.nilable(Temporalio::Converters::ExternalStorage), + namespace: T.nilable(String), + clock: T.proc.returns(Float) + ).void end - def initialize(storage, namespace:); end + def initialize(storage, namespace:, clock: T.unsafe(nil)); end sig do - params(payloads: T::Array[Temporalio::Api::Common::V1::Payload]) - .returns(T::Array[Temporalio::Api::Common::V1::Payload]) + params( + payloads: T::Array[Temporalio::Api::Common::V1::Payload], + metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics) + ).returns(T::Array[Temporalio::Api::Common::V1::Payload]) end - def store(payloads); end + def store(payloads, metrics: nil); end sig do - params(payloads: T::Array[Temporalio::Api::Common::V1::Payload]) - .returns(T::Array[Temporalio::Api::Common::V1::Payload]) + params( + payloads: T::Array[Temporalio::Api::Common::V1::Payload], + metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics) + ).returns(T::Array[Temporalio::Api::Common::V1::Payload]) end - def retrieve(payloads); end + def retrieve(payloads, metrics: nil); end end diff --git a/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi b/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi new file mode 100644 index 00000000..68bfc267 --- /dev/null +++ b/temporalio/rbi/temporalio/internal/storage_operation_metrics.rbi @@ -0,0 +1,27 @@ +# typed: true + +class Temporalio::Internal::StorageOperationMetrics + extend T::Sig + + sig { void } + def initialize; end + + sig do + params( + driver_name: String, + payload_count: Integer, + total_size_bytes: Integer, + start_time: Float, + end_time: Float + ).void + end + def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:); end + + sig { returns(T.nilable(Temporalio::Internal::Bridge::Api::Common::ExternalStorageMetrics)) } + def to_proto; end + + private + + sig { returns(Float) } + def union_seconds; end +end diff --git a/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi b/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi index fbf1b985..3302afd3 100644 --- a/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi +++ b/temporalio/rbi/temporalio/internal/worker/workflow_worker.rbi @@ -92,6 +92,12 @@ class Temporalio::Internal::Worker::WorkflowWorker sig { params(runner: Temporalio::Internal::Worker::MultiRunner, activation_completion: Object).void } def encode_activation_completion(runner, activation_completion); end + sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) } + def payload_encoding_visitor(metrics); end + + sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) } + def payload_decoding_visitor(metrics); end + sig { params(payload_or_payloads: Object, block: T.proc.params(arg0: Object).returns(Object)).void } def replace_payloads_on_visit(payload_or_payloads, &block); end end diff --git a/temporalio/sig/temporalio/internal/external_storage_runner.rbs b/temporalio/sig/temporalio/internal/external_storage_runner.rbs index 9aa2f9f9..1c2707a9 100644 --- a/temporalio/sig/temporalio/internal/external_storage_runner.rbs +++ b/temporalio/sig/temporalio/internal/external_storage_runner.rbs @@ -3,11 +3,20 @@ module Temporalio class ExternalStorageRunner @storage: Converters::ExternalStorage? @namespace: String? + @clock: ^() -> Float - def initialize: (Converters::ExternalStorage? storage, namespace: String?) -> void + MONOTONIC_CLOCK: ^() -> Float - def store: (Array[Api::Common::V1::Payload] payloads) -> Array[Api::Common::V1::Payload] - def retrieve: (Array[Api::Common::V1::Payload] payloads) -> Array[Api::Common::V1::Payload] + def initialize: (Converters::ExternalStorage? storage, namespace: String?, ?clock: ^() -> Float) -> void + + def store: ( + Array[Api::Common::V1::Payload] payloads, + ?metrics: StorageOperationMetrics? + ) -> Array[Api::Common::V1::Payload] + def retrieve: ( + Array[Api::Common::V1::Payload] payloads, + ?metrics: StorageOperationMetrics? + ) -> Array[Api::Common::V1::Payload] private def check_result_count: ( Converters::StorageDriver driver, diff --git a/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs b/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs new file mode 100644 index 00000000..a09bffa2 --- /dev/null +++ b/temporalio/sig/temporalio/internal/storage_operation_metrics.rbs @@ -0,0 +1,25 @@ +module Temporalio + module Internal + class StorageOperationMetrics + @mutex: Mutex + @payload_count: Integer + @total_size_bytes: Integer + @driver_names: Set[String] + @spans: Array[[Float, Float]] + + def initialize: -> void + + def record: ( + driver_name: String, + payload_count: Integer, + total_size_bytes: Integer, + start_time: Float, + end_time: Float + ) -> void + + def to_proto: -> Bridge::Api::Common::ExternalStorageMetrics? + + private def union_seconds: -> Float + end + end +end diff --git a/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs b/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs index b584edb5..c434c02a 100644 --- a/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs +++ b/temporalio/sig/temporalio/internal/worker/workflow_worker.rbs @@ -50,6 +50,8 @@ module Temporalio def decode_activation: (MultiRunner runner, untyped activation) -> void def encode_activation_completion: (MultiRunner runner, untyped activation_completion) -> void + def payload_encoding_visitor: (StorageOperationMetrics? metrics) -> Api::PayloadVisitor + def payload_decoding_visitor: (StorageOperationMetrics? metrics) -> Api::PayloadVisitor def replace_payloads_on_visit: (untyped payload_or_payloads) { (untyped) -> Enumerable[untyped] } -> void class State diff --git a/temporalio/test/external_storage_runner_test.rb b/temporalio/test/external_storage_runner_test.rb index 3e9d56a8..686aff97 100644 --- a/temporalio/test/external_storage_runner_test.rb +++ b/temporalio/test/external_storage_runner_test.rb @@ -5,12 +5,44 @@ require 'temporalio/error' require 'temporalio/internal/external_storage_references' require 'temporalio/internal/external_storage_runner' +require 'temporalio/internal/storage_operation_metrics' require 'test' class ExternalStorageRunnerTest < Test Runner = Temporalio::Internal::ExternalStorageRunner References = Temporalio::Internal::ExternalStorageReferences + class FakeClock + attr_reader :now + + def initialize + @now = 0.0 + end + + def advance(seconds) + @now += seconds + end + end + + # Moves the fake clock forward during each call, so the time measured around a call is exactly the given seconds. + class ClockAdvancingDriver < InMemoryStorageDriver + def initialize(clock:, seconds:, name:) + super(name:) + @clock = clock + @seconds = seconds + end + + def store(context, payloads) + @clock.advance(@seconds) + super + end + + def retrieve(context, claims) + @clock.advance(@seconds) + super + end + end + def test_store_replaces_only_payloads_at_or_above_threshold driver = InMemoryStorageDriver.new small = payload('a') @@ -129,6 +161,51 @@ def test_without_storage_retrieve_raises_on_reference assert_raises(Temporalio::Error::ExternalStorageNotConfiguredError) { Runner.new(nil, namespace: nil).retrieve([reference]) } end + def test_store_and_retrieve_record_metrics_for_each_driver + clock = FakeClock.new + one = ClockAdvancingDriver.new(clock:, seconds: 2.0, name: 'one') + two = ClockAdvancingDriver.new(clock:, seconds: 3.0, name: 'two') + runner = Runner.new(Temporalio::Converters::ExternalStorage.new( + drivers: [two, one], + driver_selector: ->(_context, payload) { payload.data == '"a"' ? one : two }, + payload_size_threshold: 0 + ), namespace: nil, clock: -> { clock.now }) + originals = [payload('a'), payload('b'), payload('c')] + original_size = originals.sum { |p| p.to_proto.bytesize } + + store_metrics = Temporalio::Internal::StorageOperationMetrics.new + stored = runner.store(originals, metrics: store_metrics) + retrieve_metrics = Temporalio::Internal::StorageOperationMetrics.new + runner.retrieve(stored, metrics: retrieve_metrics) + + [store_metrics, retrieve_metrics].each do |metrics| + proto = metrics.to_proto or flunk('Expected metrics') + assert_equal 3, proto.payload_count + assert_equal original_size, proto.total_size_bytes + assert_equal %w[one two], proto.driver_names.to_a + assert_equal 5.0, Temporalio::Internal::ProtoUtils.duration_to_seconds(proto.total_duration) + end + end + + def test_metrics_are_unset_when_nothing_is_moved + runner = Runner.new(storage(InMemoryStorageDriver.new, payload_size_threshold: 1_000_000), namespace: nil) + metrics = Temporalio::Internal::StorageOperationMetrics.new + + runner.retrieve(runner.store([payload('a')], metrics:), metrics:) + + assert_nil metrics.to_proto + end + + def test_metrics_duration_counts_overlapping_calls_once + metrics = Temporalio::Internal::StorageOperationMetrics.new + metrics.record(driver_name: 'a', payload_count: 1, total_size_bytes: 1, start_time: 10.0, end_time: 12.0) + metrics.record(driver_name: 'b', payload_count: 1, total_size_bytes: 1, start_time: 11.0, end_time: 13.0) + metrics.record(driver_name: 'a', payload_count: 1, total_size_bytes: 1, start_time: 20.0, end_time: 20.5) + + duration = metrics.to_proto&.total_duration or flunk('Expected a duration') + assert_equal 3.5, Temporalio::Internal::ProtoUtils.duration_to_seconds(duration) + end + private def storage(driver, payload_size_threshold: 0) diff --git a/temporalio/test/external_storage_test.rb b/temporalio/test/external_storage_test.rb index 862b211e..b23f1087 100644 --- a/temporalio/test/external_storage_test.rb +++ b/temporalio/test/external_storage_test.rb @@ -121,7 +121,9 @@ def test_empty_driver_name_rejected def test_symbol_driver_name_rejected # A Symbol passes a bare empty? check but cannot be looked up by String or written to the proto. err = assert_raises(ArgumentError) do - Temporalio::Converters::ExternalStorage.new(drivers: [InMemoryStorageDriver.new(name: :mem)]) + Temporalio::Converters::ExternalStorage.new( + drivers: [InMemoryStorageDriver.new(name: :mem)] # steep:ignore ArgumentTypeMismatch + ) end assert_includes err.message, 'must be a non-empty String' end diff --git a/temporalio/test/in_memory_storage_driver.rb b/temporalio/test/in_memory_storage_driver.rb index 1aacef35..eaf5a32b 100644 --- a/temporalio/test/in_memory_storage_driver.rb +++ b/temporalio/test/in_memory_storage_driver.rb @@ -36,7 +36,7 @@ def retrieve_calls end def store(context, payloads) - @mutex.synchronize { @store_calls << [context, payloads.size] } + @mutex.synchronize { @store_calls << [context, payloads.count] } payloads.map do |payload| key = SecureRandom.uuid @mutex.synchronize { @stored[key] = payload.to_proto } @@ -45,7 +45,7 @@ def store(context, payloads) end def retrieve(_context, claims) - @mutex.synchronize { @retrieve_calls << claims.size } + @mutex.synchronize { @retrieve_calls << claims.count } claims.map do |claim| bytes = @mutex.synchronize { @stored.fetch(claim.claim_data.fetch('key')) } Temporalio::Api::Common::V1::Payload.decode(bytes) diff --git a/temporalio/test/sig/external_storage_runner_test.rbs b/temporalio/test/sig/external_storage_runner_test.rbs index 56bc96cd..30d25c62 100644 --- a/temporalio/test/sig/external_storage_runner_test.rbs +++ b/temporalio/test/sig/external_storage_runner_test.rbs @@ -1,4 +1,18 @@ class ExternalStorageRunnerTest < Test + class FakeClock + @now: Float + + attr_reader now: Float + def advance: (Float seconds) -> Float + end + + class ClockAdvancingDriver < InMemoryStorageDriver + @clock: FakeClock + @seconds: Float + + def initialize: (clock: FakeClock, seconds: Float, name: String) -> void + end + def storage: ( Temporalio::Converters::StorageDriver driver, ?payload_size_threshold: Integer diff --git a/temporalio/test/sig/in_memory_storage_driver.rbs b/temporalio/test/sig/in_memory_storage_driver.rbs new file mode 100644 index 00000000..9b116dbb --- /dev/null +++ b/temporalio/test/sig/in_memory_storage_driver.rbs @@ -0,0 +1,17 @@ +class InMemoryStorageDriver < Temporalio::Converters::StorageDriver + @name: String + @type: String + @mutex: Mutex + @stored: Hash[String, String] + @store_calls: Array[[Temporalio::Converters::StorageDriverStoreContext, Integer]] + @retrieve_calls: Array[Integer] + + attr_reader name: String + attr_reader type: String + + def initialize: (?name: String, ?type: String) -> void + + def stored: -> Hash[String, String] + def store_calls: -> Array[[Temporalio::Converters::StorageDriverStoreContext, Integer]] + def retrieve_calls: -> Array[Integer] +end diff --git a/temporalio/test/sig/worker_external_storage_test.rbs b/temporalio/test/sig/worker_external_storage_test.rbs index a089d6a1..88ff88dd 100644 --- a/temporalio/test/sig/worker_external_storage_test.rbs +++ b/temporalio/test/sig/worker_external_storage_test.rbs @@ -80,6 +80,9 @@ class WorkerExternalStorageTest < Test def change: (untyped input) -> untyped end + def run_echo_workflow_capturing_completions: ( + Temporalio::Converters::StorageDriver driver + ) -> Array[Temporalio::Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion] def assert_storage_failure_retried: (Temporalio::Converters::StorageDriver driver, String failure_message) -> void def client_with_storage: ( Temporalio::Converters::StorageDriver driver, diff --git a/temporalio/test/worker_external_storage_test.rb b/temporalio/test/worker_external_storage_test.rb index 94ad1e90..48ceae54 100644 --- a/temporalio/test/worker_external_storage_test.rb +++ b/temporalio/test/worker_external_storage_test.rb @@ -456,8 +456,58 @@ def test_worker_without_storage_fails_the_task_on_an_update_reference end end + def test_workflow_task_completions_report_storage_metrics + driver = InMemoryStorageDriver.new + completions = run_echo_workflow_capturing_completions(driver) + + # The first task retrieves the input and stores the result + first = completions.first or flunk('No completions') + stored_sizes = driver.stored.values.map(&:bytesize).sort + download = first.payload_download_metrics or flunk('No download metrics') + upload = first.payload_upload_metrics or flunk('No upload metrics') + assert_equal 1, download.payload_count + assert_equal 1, upload.payload_count + assert_equal stored_sizes, [download.total_size_bytes, upload.total_size_bytes].sort + assert_equal ['mem'], download.driver_names.to_a + assert_equal ['mem'], upload.driver_names.to_a + end + + def test_failed_store_still_reports_download_metrics + completions = run_echo_workflow_capturing_completions(FailOnceStorageDriver.new(fail_store_containing: 'done')) + + failed = completions.find(&:failed) or flunk('No failed completion') + download = failed.payload_download_metrics or flunk('No download metrics') + assert_equal 1, download.payload_count + assert_equal ['mem'], download.driver_names.to_a + assert_nil failed.payload_upload_metrics + end + private + def run_echo_workflow_capturing_completions(driver) + client = client_with_storage(driver) + worker = Temporalio::Worker.new( + client:, + task_queue: "tq-#{SecureRandom.uuid}", + workflows: [EchoWorkflow], + workflow_payload_codec_thread_pool: Temporalio::Worker::ThreadPool.default + ) + completions = Queue.new + bridge_worker = worker._bridge_worker + complete = bridge_worker.method(:async_complete_workflow_activation) + bridge_worker.define_singleton_method(:async_complete_workflow_activation) do |run_id, proto, queue| + completions << Temporalio::Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.decode(proto) + complete.call(run_id, proto, queue) + end + worker.run do + result = client.execute_workflow( + EchoWorkflow, 'start-input', id: "wf-#{SecureRandom.uuid}", task_queue: worker.task_queue + ) + assert_equal 'start-input done', result + end + Array.new(completions.size) { completions.pop } + end + def assert_storage_failure_retried(driver, failure_message) execute_workflow( EchoWorkflow, 'start-input',