Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions temporalio/lib/temporalio/internal/external_storage_runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
require 'temporalio/converters/storage_driver_store_context'
require 'temporalio/error'
require 'temporalio/internal/external_storage_references'
require 'temporalio/internal/storage_operation_metrics'

module Temporalio
module Internal
Expand All @@ -23,18 +24,24 @@ module Internal
#
# @!visibility private
class ExternalStorageRunner
MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
private_constant :MONOTONIC_CLOCK

# @param storage [Converters::ExternalStorage, nil] Configured storage. When nil, storing does nothing and
# retrieving raises if it finds a reference, so a reference is never mistaken for an ordinary payload.
# @param namespace [String, nil] Namespace of the client or worker storing through this runner, or nil when it is
# not known.
def initialize(storage, namespace:)
# @param clock [Proc] Returns monotonic seconds, read around each driver call to time it.
def initialize(storage, namespace:, clock: MONOTONIC_CLOCK)
@storage = storage
@namespace = namespace
@clock = clock
end

# @param payloads [Array<Api::Common::V1::Payload>] Codec-encoded payloads. Not mutated.
# @param metrics [StorageOperationMetrics, nil] Records each driver call that stores payloads.
# @return [Array<Api::Common::V1::Payload>] Payloads with every eligible one replaced by a reference.
def store(payloads)
def store(payloads, metrics: nil)
storage = @storage
return payloads unless storage

Expand Down Expand Up @@ -69,8 +76,17 @@ def store(payloads)
# Drivers get copies, since a driver may keep a payload after returning and the SDK modifies some payloads in
# place
copies = indexes_and_serialized.map { |_index, serialized| Api::Common::V1::Payload.decode(serialized) }
start_time = @clock.call
claims = driver.store(context, copies)
end_time = @clock.call
check_result_count(driver, 'store', indexes_and_serialized.size, claims)
metrics&.record(
driver_name: driver.name,
payload_count: claims.size,
total_size_bytes: indexes_and_serialized.sum { |_index, serialized| serialized.bytesize },
start_time:,
end_time:
)

indexes_and_serialized.zip(claims) do |(index, serialized), claim|
result[index] = ExternalStorageReferences.create_reference_payload(
Expand All @@ -82,9 +98,10 @@ def store(payloads)
end

# @param payloads [Array<Api::Common::V1::Payload>] Payloads that may contain references. Not mutated.
# @param metrics [StorageOperationMetrics, nil] Records each driver call that retrieves payloads.
# @return [Array<Api::Common::V1::Payload>] Payloads with every reference replaced by what it points at.
# @raise [Error::ExternalStorageNotConfiguredError] If there is a reference but no storage is configured.
def retrieve(payloads)
def retrieve(payloads, metrics: nil)
storage = @storage
# Driver => [[index, claim], ...] for each reference that driver stored. Every driver is looked up here, so an
# unregistered name fails before any request is made.
Expand All @@ -105,8 +122,17 @@ def retrieve(payloads)
context = Converters::StorageDriverRetrieveContext.new(
cancellation: Cancellation.new, limiter: UNBOUNDED_LIMITER
)
start_time = @clock.call
retrieved = driver.retrieve(context, indexes_and_claims.map { |_index, claim| claim })
end_time = @clock.call
check_result_count(driver, 'retrieve', indexes_and_claims.size, retrieved)
metrics&.record(
driver_name: driver.name,
payload_count: retrieved.size,
total_size_bytes: retrieved.sum { |payload| payload.to_proto.bytesize },
start_time:,
end_time:
)

indexes_and_claims.zip(retrieved) { |(index, _claim), payload| result[index] = payload }
end
Expand Down
76 changes: 76 additions & 0 deletions temporalio/lib/temporalio/internal/storage_operation_metrics.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# frozen_string_literal: true

require 'temporalio/internal/bridge/api'
require 'temporalio/internal/proto_utils'

module Temporalio
module Internal
# Gathers what external storage moved during one store or retrieve pass, for core to report on the activation
# completion.
#
# @!visibility private
class StorageOperationMetrics
def initialize
@mutex = Mutex.new
@payload_count = 0
@total_size_bytes = 0
@driver_names = Set.new
@spans = []
end

# Record one driver call that moved payloads.
#
# @param driver_name [String] Driver that stored or retrieved the payloads.
# @param payload_count [Integer] Number of payloads moved.
# @param total_size_bytes [Integer] Encoded size of the payloads moved.
# @param start_time [Float] Monotonic clock reading taken before the driver call.
# @param end_time [Float] Monotonic clock reading taken after the driver call.
def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:)
@mutex.synchronize do
@driver_names << driver_name
@payload_count += payload_count
@total_size_bytes += total_size_bytes
@spans << [start_time, end_time]
end
end

# @return [Bridge::Api::Common::ExternalStorageMetrics, nil] Metrics for the pass, or nil when it moved nothing,
# so the completion field stays unset.
def to_proto
payload_count, total_size_bytes, duration_seconds, driver_names = @mutex.synchronize do
return nil if @payload_count.zero?

[@payload_count, @total_size_bytes, union_seconds, @driver_names.sort]
end
# Built outside the lock, since creating a protobuf message takes a process-wide lock that would otherwise also
# hold up record
Bridge::Api::Common::ExternalStorageMetrics.new(
payload_count:,
total_size_bytes:,
total_duration: ProtoUtils.seconds_to_duration(duration_seconds),
driver_names:
)
end

private

def union_seconds
first, *rest = @spans.sort
return 0.0 unless first

total = 0.0
span_start, span_end = first
rest.each do |start_time, end_time|
if start_time > span_end
total += span_end - span_start
span_start = start_time
span_end = end_time
elsif end_time > span_end
span_end = end_time
end
end
total + (span_end - span_start)
end
end
end
end
104 changes: 70 additions & 34 deletions temporalio/lib/temporalio/internal/worker/workflow_worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
require 'temporalio/error'
require 'temporalio/internal/bridge/api'
require 'temporalio/internal/external_storage_runner'
require 'temporalio/internal/storage_operation_metrics'
require 'temporalio/internal/worker/workflow_instance'
require 'temporalio/scoped_logger'
require 'temporalio/workflow'
Expand Down Expand Up @@ -90,30 +91,22 @@ def initialize(
'storage and not using fibers'
end

# If there is a payload codec or external storage, we need to build encoding and decoding visitors. Storage
# comes after the codec when encoding and before it when decoding. These may block, so they run in the
# background.
storage_runner = ExternalStorageRunner.new(external_storage, namespace:)
# If there is a payload codec or external storage, payloads are encoded and decoded by visitors built per
# pass, so each pass can measure its own storage operations. Storage comes after the codec when encoding and
# before it when decoding. These may block, so they run in the background.
@payload_codec = payload_codec
@external_storage = external_storage
@storage_runner = ExternalStorageRunner.new(external_storage, namespace:)
@decode_in_background = !payload_codec.nil? || !external_storage.nil?
if @decode_in_background
@payload_encoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
replace_payloads_on_visit(payload_or_payloads) do |payloads|
payloads = payload_codec.encode(payloads) if payload_codec
payloads = storage_runner.store(payloads.to_a) if external_storage
payloads
end
end
@payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
replace_payloads_on_visit(payload_or_payloads) do |payloads|
payloads = storage_runner.retrieve(payloads.to_a)
payloads = payload_codec.decode(payloads) if payload_codec
payloads
end
end
else
# Download metrics by run ID, waiting for the completion of the activation that measured them. Core never has
# two activations of one run outstanding, so the run ID identifies the activation.
@pending_download_metrics = {}
@pending_download_metrics_mutex = Mutex.new
unless @decode_in_background
# Without a codec or storage, decoding only rejects references that were never retrieved. That needs no
# I/O, so it runs inline and existing workers need no thread pool.
@payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
storage_runner = @storage_runner
@payload_rejecting_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
storage_runner.retrieve(
payload_or_payloads.is_a?(Api::Common::V1::Payload) ? [payload_or_payloads] : payload_or_payloads.to_a
)
Expand Down Expand Up @@ -176,7 +169,7 @@ def handle_activation(runner:, activation:, decoded:)
# Returns whether completion was submitted to the bridge
# rubocop:disable-next Naming/PredicateMethod
def handle_activation_complete(runner:, activation_completion:, encoded:, completion_complete_queue:)
if @payload_encoding_visitor && !encoded
if @decode_in_background && !encoded
if Fiber.current_scheduler
Fiber.schedule { encode_activation_completion(runner, activation_completion) }
else
Expand All @@ -203,7 +196,17 @@ def on_shutdown_complete
# activation as failed. Otherwise the run waits forever for a completion that never comes.

def decode_activation(runner, activation)
@payload_decoding_visitor.run(activation)
if @decode_in_background
metrics = nil
if @external_storage
metrics = StorageOperationMetrics.new
# Stored before the visit, so the completion reports what was retrieved even if decoding then fails
@pending_download_metrics_mutex.synchronize { @pending_download_metrics[activation.run_id] = metrics }
end
payload_decoding_visitor(metrics).run(activation)
else
@payload_rejecting_visitor&.run(activation)
end
runner.apply_workflow_activation_decoded(workflow_worker: self, activation:)
rescue Exception => e # rubocop:disable Lint/RescueException
activation_completion = WorkflowInstance.new_completion_with_failure(
Expand All @@ -217,18 +220,51 @@ def decode_activation(runner, activation)
end

def encode_activation_completion(runner, activation_completion)
@payload_encoding_visitor.run(activation_completion)
runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true)
rescue Exception => e # rubocop:disable Lint/RescueException
# Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads could
# fail the same way and retry forever
activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new(
run_id: activation_completion.run_id,
failed: Bridge::Api::WorkflowCompletion::Failure.new(
failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}")
download_metrics = @pending_download_metrics_mutex.synchronize do
@pending_download_metrics.delete(activation_completion.run_id)
end
upload_metrics = StorageOperationMetrics.new if @external_storage
begin
payload_encoding_visitor(upload_metrics).run(activation_completion)
rescue Exception => e # rubocop:disable Lint/RescueException
# Only the message is sent, and it is marked encoded, because encoding a failure that carries payloads
# could fail the same way and retry forever
activation_completion = Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new(
run_id: activation_completion.run_id,
failed: Bridge::Api::WorkflowCompletion::Failure.new(
failure: Api::Failure::V1::Failure.new(message: "Failed encoding completion: #{e.message}")
)
)
)
runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: false)
end
# Reported even when a pass failed, since what it moved before failing still took time
activation_completion.payload_download_metrics = download_metrics&.to_proto
activation_completion.payload_upload_metrics = upload_metrics&.to_proto
runner.apply_workflow_activation_complete(workflow_worker: self, activation_completion:, encoded: true)
end

def payload_encoding_visitor(metrics)
payload_codec = @payload_codec
external_storage = @external_storage
storage_runner = @storage_runner
Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
replace_payloads_on_visit(payload_or_payloads) do |payloads|
payloads = payload_codec.encode(payloads) if payload_codec
payloads = storage_runner.store(payloads.to_a, metrics:) if external_storage
payloads
end
end
end

def payload_decoding_visitor(metrics)
payload_codec = @payload_codec
storage_runner = @storage_runner
Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
replace_payloads_on_visit(payload_or_payloads) do |payloads|
payloads = storage_runner.retrieve(payloads.to_a, metrics:)
payloads = payload_codec.decode(payloads) if payload_codec
payloads
end
end
end

def replace_payloads_on_visit(payload_or_payloads, &)
Expand Down
24 changes: 16 additions & 8 deletions temporalio/rbi/temporalio/internal/external_storage_runner.rbi
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,27 @@ class Temporalio::Internal::ExternalStorageRunner
extend T::Sig

sig do
params(storage: T.nilable(Temporalio::Converters::ExternalStorage), namespace: T.nilable(String)).void
params(
storage: T.nilable(Temporalio::Converters::ExternalStorage),
namespace: T.nilable(String),
clock: T.proc.returns(Float)
).void
end
def initialize(storage, namespace:); end
def initialize(storage, namespace:, clock: T.unsafe(nil)); end

sig do
params(payloads: T::Array[Temporalio::Api::Common::V1::Payload])
.returns(T::Array[Temporalio::Api::Common::V1::Payload])
params(
payloads: T::Array[Temporalio::Api::Common::V1::Payload],
metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)
).returns(T::Array[Temporalio::Api::Common::V1::Payload])
end
def store(payloads); end
def store(payloads, metrics: nil); end

sig do
params(payloads: T::Array[Temporalio::Api::Common::V1::Payload])
.returns(T::Array[Temporalio::Api::Common::V1::Payload])
params(
payloads: T::Array[Temporalio::Api::Common::V1::Payload],
metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)
).returns(T::Array[Temporalio::Api::Common::V1::Payload])
end
def retrieve(payloads); end
def retrieve(payloads, metrics: nil); end
end
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# typed: true

class Temporalio::Internal::StorageOperationMetrics
extend T::Sig

sig { void }
def initialize; end

sig do
params(
driver_name: String,
payload_count: Integer,
total_size_bytes: Integer,
start_time: Float,
end_time: Float
).void
end
def record(driver_name:, payload_count:, total_size_bytes:, start_time:, end_time:); end

sig { returns(T.nilable(Temporalio::Internal::Bridge::Api::Common::ExternalStorageMetrics)) }
def to_proto; end

private

sig { returns(Float) }
def union_seconds; end
end
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ class Temporalio::Internal::Worker::WorkflowWorker
sig { params(runner: Temporalio::Internal::Worker::MultiRunner, activation_completion: Object).void }
def encode_activation_completion(runner, activation_completion); end

sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) }
def payload_encoding_visitor(metrics); end

sig { params(metrics: T.nilable(Temporalio::Internal::StorageOperationMetrics)).returns(Temporalio::Api::PayloadVisitor) }
def payload_decoding_visitor(metrics); end

sig { params(payload_or_payloads: Object, block: T.proc.params(arg0: Object).returns(Object)).void }
def replace_payloads_on_visit(payload_or_payloads, &block); end
end
Expand Down
Loading
Loading