Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/added/decoding-cloakroom-ticket.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added experimental support for reading external storage reference payloads that are encoded with the configured payload codec and marked with `__temporal_external_storage_reference` metadata. This release does not write them yet. A later release will, once workers have had time to upgrade.
7 changes: 6 additions & 1 deletion temporalio/lib/temporalio/converters/data_converter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ def to_payloads(values, hints: nil)
# @param hint [Object, nil] Hint, if any, to assist conversion.
# @return [Object] Decoded and converted Ruby value.
def from_payload(payload, hint: nil)
if external_storage
payload = Internal::ExternalStorageReferences.decode_references([payload], payload_codec:).first
end
payload = @external_storage_runner.retrieve([payload]).first
payload = payload_codec.decode([payload]).first if payload_codec
payload_converter.from_payload(payload, hint:)
Expand All @@ -105,7 +108,9 @@ def from_payload(payload, hint: nil)
def from_payloads(payloads, hints: nil)
return [] unless payloads && !payloads.payloads.empty?

payloads = Api::Common::V1::Payloads.new(payloads: @external_storage_runner.retrieve(payloads.payloads.to_a))
payloads = payloads.payloads.to_a
payloads = Internal::ExternalStorageReferences.decode_references(payloads, payload_codec:) if external_storage
payloads = Api::Common::V1::Payloads.new(payloads: @external_storage_runner.retrieve(payloads))
if payload_codec && !payloads.payloads.empty?
payloads = Api::Common::V1::Payloads.new(payloads: payload_codec.decode(payloads.payloads))
end
Expand Down
16 changes: 16 additions & 0 deletions temporalio/lib/temporalio/error.rb
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,22 @@ def initialize(
end
end

# Error that occurs when a payload marked as an encoded external storage reference does not decode to a reference.
# This usually means the payload codec configured to read it is not the one that wrote it.
#
# @note WARNING: This API is experimental and may change in the future.
#
# @!visibility private
class ExternalStorageReferenceDecodeError < Error
# @!visibility private
def initialize(
message = '[TMPRL1106] Encountered an encoded external storage reference that could not be decoded. ' \
'Check that the payload codec used to write it is configured.'
)
super
end
end

# Error raised by a client for a general RPC failure.
class RPCError < Error
# @return [Code] Status code for the error.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

require 'temporalio/api'
require 'temporalio/api/sdk/v1/external_storage'
require 'temporalio/error'

module Temporalio
module Internal
Expand All @@ -14,6 +15,7 @@ module Internal
module ExternalStorageReferences
ENCODING = 'json/protobuf'
MESSAGE_TYPE = 'temporal.api.sdk.v1.ExternalStorageReference'
MARKER_KEY = '__temporal_external_storage_reference'

class << self
# @param payload [Api::Common::V1::Payload] Payload to check.
Expand All @@ -38,6 +40,37 @@ def parse_reference(payload)
)
end

# @param payload [Api::Common::V1::Payload] Payload to check.
# @return [Boolean] Whether the payload is marked as a reference that may have been encoded by a payload codec.
def marked_reference?(payload)
payload.metadata[MARKER_KEY] == 'true'
end

# Replace each marked reference with the plain reference it encodes, so it can be retrieved.
#
# @param payloads [Array<Api::Common::V1::Payload>] Payloads that may contain marked references. Not mutated.
# @param payload_codec [Converters::PayloadCodec, nil] Codec that encoded the references. Without one, payloads
# are returned as is and the storage runner rejects a marked payload that is not a plain reference.
# @return [Array<Api::Common::V1::Payload>] Payloads with every marked reference decoded.
# @raise [Error::ExternalStorageReferenceDecodeError] If a marked reference does not decode to a reference.
def decode_references(payloads, payload_codec:)
return payloads unless payload_codec

payloads.map do |payload|
Comment thread
cconstable marked this conversation as resolved.
Resolved
next payload unless marked_reference?(payload)

# The payload is decoded as it appears in history, marker included, since that is what a codec server
# receives too.
reference = payload_codec.decode([payload]).first
raise Error::ExternalStorageReferenceDecodeError unless reference && reference?(reference)

if reference.external_payloads.empty?
reference.external_payloads.push(*payload.external_payloads.map(&:dup))
end
reference
end
end

# @param driver_name [String] Name of the driver that stored the payload.
# @param claim_data [Hash<String, String>] Driver data identifying the stored payload.
# @param original_size_bytes [Integer] Encoded size of the payload that was offloaded.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ def retrieve(payloads)
claims_by_driver = Hash.new { |hash, driver| hash[driver] = [] }
payloads.each_with_index do |payload, index|
reference = ExternalStorageReferences.parse_reference(payload)
next unless reference
next unless reference || ExternalStorageReferences.marked_reference?(payload)
raise Error::ExternalStorageNotConfiguredError unless storage
raise Error::ExternalStorageReferenceDecodeError unless reference

driver = storage.driver(reference.driver_name)
raise Error, "No external storage driver registered with name '#{reference.driver_name}'" unless driver
Expand Down
11 changes: 9 additions & 2 deletions temporalio/lib/temporalio/internal/worker/activity_worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
require 'temporalio/cancellation'
require 'temporalio/converters/raw_value'
require 'temporalio/internal/bridge/api'
require 'temporalio/internal/external_storage_references'
require 'temporalio/internal/external_storage_runner'
require 'temporalio/internal/proto_utils'
require 'temporalio/scoped_logger'
Expand Down Expand Up @@ -193,8 +194,11 @@ def execute_activity(task_token, defn, start)
raw_heartbeat_details: begin
payloads = start.heartbeat_details.to_ary
data_converter = @worker.options.client.data_converter
payloads = ExternalStorageRunner.new(data_converter.external_storage, namespace: nil).retrieve(payloads)
codec = data_converter.payload_codec
if data_converter.external_storage
payloads = ExternalStorageReferences.decode_references(payloads, payload_codec: codec)
end
payloads = ExternalStorageRunner.new(data_converter.external_storage, namespace: nil).retrieve(payloads)
payloads = codec.decode(payloads) if codec
payloads.map { |p| Temporalio::Converters::RawValue.new(p) }
end,
Expand All @@ -218,9 +222,12 @@ def execute_activity(task_token, defn, start)
args: if defn.raw_args
payloads = start.input.to_ary
data_converter = @worker.options.client.data_converter
codec = data_converter.payload_codec
if data_converter.external_storage
payloads = ExternalStorageReferences.decode_references(payloads, payload_codec: codec)
end
payloads = ExternalStorageRunner.new(data_converter.external_storage, namespace: nil)
.retrieve(payloads)
codec = data_converter.payload_codec
payloads = codec.decode(payloads) if codec
payloads.map { |p| Temporalio::Converters::RawValue.new(p) }
else
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
require 'temporalio/api/payload_visitor'
require 'temporalio/error'
require 'temporalio/internal/bridge/api'
require 'temporalio/internal/external_storage_references'
require 'temporalio/internal/external_storage_runner'
require 'temporalio/internal/worker/workflow_instance'
require 'temporalio/scoped_logger'
Expand Down Expand Up @@ -105,7 +106,9 @@ def initialize(
end
@payload_decoding_visitor = Api::PayloadVisitor.new(skip_search_attributes: true) do |payload_or_payloads|
replace_payloads_on_visit(payload_or_payloads) do |payloads|
payloads = storage_runner.retrieve(payloads.to_a)
payloads = payloads.to_a
payloads = ExternalStorageReferences.decode_references(payloads, payload_codec:) if external_storage
payloads = storage_runner.retrieve(payloads)
payloads = payload_codec.decode(payloads) if payload_codec
payloads
end
Expand Down
7 changes: 7 additions & 0 deletions temporalio/rbi/temporalio/error.rbi
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,13 @@ class Temporalio::Error::ExternalStorageNotConfiguredError < ::Temporalio::Error
def initialize(message = T.unsafe(nil)); end
end

class Temporalio::Error::ExternalStorageReferenceDecodeError < ::Temporalio::Error
extend T::Sig

sig { params(message: String).void }
def initialize(message = T.unsafe(nil)); end
end

class Temporalio::Error::RPCError < ::Temporalio::Error
sig { params(message: String, code: Integer, raw_grpc_status: T.nilable(Object)).void }
def initialize(message, code:, raw_grpc_status:); end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module Temporalio::Internal::ExternalStorageReferences

ENCODING = T.let(T.unsafe(nil), String)
MESSAGE_TYPE = T.let(T.unsafe(nil), String)
MARKER_KEY = T.let(T.unsafe(nil), String)

sig { params(payload: Temporalio::Api::Common::V1::Payload).returns(T::Boolean) }
def self.reference?(payload); end
Expand All @@ -15,6 +16,17 @@ module Temporalio::Internal::ExternalStorageReferences
end
def self.parse_reference(payload); end

sig { params(payload: Temporalio::Api::Common::V1::Payload).returns(T::Boolean) }
def self.marked_reference?(payload); end

sig do
params(
payloads: T::Array[Temporalio::Api::Common::V1::Payload],
payload_codec: T.nilable(Temporalio::Converters::PayloadCodec)
).returns(T::Array[Temporalio::Api::Common::V1::Payload])
end
def self.decode_references(payloads, payload_codec:); end

sig do
params(
driver_name: String,
Expand Down
4 changes: 4 additions & 0 deletions temporalio/sig/temporalio/error.rbs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ module Temporalio
def initialize: (?String message) -> void
end

class ExternalStorageReferenceDecodeError < Error
def initialize: (?String message) -> void
end

class RPCError < Error
attr_reader code: Code::enum

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@ module Temporalio
module ExternalStorageReferences
ENCODING: String
MESSAGE_TYPE: String
MARKER_KEY: String

def self.reference?: (Api::Common::V1::Payload payload) -> bool
def self.marked_reference?: (Api::Common::V1::Payload payload) -> bool
def self.decode_references: (
Array[Api::Common::V1::Payload] payloads,
payload_codec: Converters::PayloadCodec?
) -> Array[Api::Common::V1::Payload]
def self.parse_reference: (
Api::Common::V1::Payload payload
) -> Api::Sdk::V1::ExternalStorageReference?
Expand Down
2 changes: 2 additions & 0 deletions temporalio/test/base64_codec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ def encode(payloads)

def decode(payloads)
payloads.map do |p|
next p unless p.metadata['encoding'] == 'test/base64'

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This automatically weakens the strict nature of this codec for all tests. Consider subclassing or adding a optional parameter to allow passthrough when encoding doesn't match with a default of false.


Temporalio::Api::Common::V1::Payload.decode(
Base64.strict_decode64(p.data)
)
Expand Down
21 changes: 21 additions & 0 deletions temporalio/test/external_storage_runner_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,27 @@ def test_without_storage_retrieve_raises_on_reference
assert_raises(Temporalio::Error::ExternalStorageNotConfiguredError) { Runner.new(nil, namespace: nil).retrieve([reference]) }
end

def test_without_storage_retrieve_raises_on_marked_reference
# Opaque to the runner, as a codec-encoded reference is
marked = Temporalio::Api::Common::V1::Payload.new(
metadata: { 'encoding' => 'binary/encrypted', References::MARKER_KEY => 'true' }, data: 'encrypted'
)

assert_raises(Temporalio::Error::ExternalStorageNotConfiguredError) { Runner.new(nil, namespace: nil).retrieve([marked]) }
end

def test_retrieve_raises_on_marked_payload_that_is_not_a_reference
# A reference encoded by a codec this side does not have
marked = Temporalio::Api::Common::V1::Payload.new(
metadata: { 'encoding' => 'binary/encrypted', References::MARKER_KEY => 'true' }, data: 'encrypted'
)

err = assert_raises(Temporalio::Error::ExternalStorageReferenceDecodeError) do
Runner.new(storage(InMemoryStorageDriver.new), namespace: nil).retrieve([marked])
end
assert_includes err.message, 'TMPRL1106'
end

private

def storage(driver, payload_size_threshold: 0)
Expand Down
72 changes: 72 additions & 0 deletions temporalio/test/external_storage_test.rb
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
# frozen_string_literal: true

require 'base64'
require 'base64_codec'
require 'in_memory_storage_driver'
require 'recording_storage_driver_limiter'
require 'securerandom'
require 'temporalio/cancellation'
require 'temporalio/converters/data_converter'
require 'temporalio/converters/external_storage'
require 'temporalio/converters/storage_driver_activity_info'
require 'temporalio/converters/storage_driver_claim'
Expand All @@ -14,6 +16,7 @@
require 'temporalio/converters/storage_driver_workflow_info'
require 'temporalio/error'
require 'temporalio/internal/external_storage_references'
require 'temporalio/internal/external_storage_runner'
require 'test'

class ExternalStorageTest < Test
Expand Down Expand Up @@ -285,8 +288,77 @@ def test_not_configured_error_names_its_code
assert_includes err.message, 'TMPRL1105'
end

def test_decode_references_decodes_marked_reference_and_keeps_its_size
Comment thread
cconstable marked this conversation as resolved.
Resolved
reference = References.create_reference_payload(
driver_name: 'mem', claim_data: { 'key' => 'k' }, original_size_bytes: 385
)
ordinary = payload('"ordinary"')

decoded = References.decode_references(
[encoded_reference(reference), ordinary], payload_codec: Base64Codec.new
)

assert_equal reference, decoded[0]
assert_same ordinary, decoded[1]
end

def test_decode_references_raises_when_marked_payload_decodes_to_something_else
assert_raises(Temporalio::Error::ExternalStorageReferenceDecodeError) do
References.decode_references(
[encoded_reference(payload('"not a reference"'))], payload_codec: Base64Codec.new
)
end
end

def test_decode_references_keeps_marked_reference_the_codec_passes_through
reference = References.create_reference_payload(
driver_name: 'mem', claim_data: { 'key' => 'k' }, original_size_bytes: 385
)
reference.metadata[References::MARKER_KEY] = 'true'
ordinary = payload('"ordinary"')

decoded = References.decode_references(
[reference, ordinary], payload_codec: Base64Codec.new
)

assert_equal reference, decoded[0]
assert_same ordinary, decoded[1]
end

def test_decode_references_without_codec_returns_payloads_unchanged
encoded = encoded_reference(
References.create_reference_payload(driver_name: 'mem', claim_data: {}, original_size_bytes: 1)
)

assert_same encoded, References.decode_references([encoded], payload_codec: nil).first
end

def test_data_converter_reads_encoded_reference
driver = InMemoryStorageDriver.new
storage = Temporalio::Converters::ExternalStorage.new(drivers: [driver], payload_size_threshold: 0)
codec = Base64Codec.new
stored = Temporalio::Converters::DataConverter.new(payload_codec: codec).to_payload('stored-value')
reference = Temporalio::Internal::ExternalStorageRunner.new(storage, namespace: nil).store([stored]).first
converter = Temporalio::Converters::DataConverter.new(payload_codec: codec, external_storage: storage)

assert_equal 'stored-value', converter.from_payload(encoded_reference(reference))
assert_equal ['stored-value'], converter.from_payloads(
Temporalio::Api::Common::V1::Payloads.new(payloads: [encoded_reference(reference)])
)
end

private

# Encodes the payload without its size and marks it, so decoding must restore the size from the outer payload
def encoded_reference(reference)
without_size = Temporalio::Api::Common::V1::Payload.decode(reference.to_proto)
without_size.external_payloads.clear
encoded = Base64Codec.new.encode([without_size]).first
encoded.metadata[References::MARKER_KEY] = 'true'
encoded.external_payloads.push(*reference.external_payloads.map(&:dup))
encoded
end

def cancellation
Temporalio::Cancellation.new
end
Expand Down
1 change: 1 addition & 0 deletions temporalio/test/sig/external_storage_test.rbs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
class ExternalStorageTest < Test
def encoded_reference: (Temporalio::Api::Common::V1::Payload reference) -> Temporalio::Api::Common::V1::Payload
def cancellation: -> Temporalio::Cancellation
def limiter: -> Temporalio::Converters::StorageDriverLimiter
def select_context: -> Temporalio::Converters::StorageDriverSelectContext
Expand Down
12 changes: 12 additions & 0 deletions temporalio/test/sig/worker_external_storage_test.rbs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ class WorkerExternalStorageTest < Test
def execute: -> String
end

class EncodedHeartbeatActivity < Temporalio::Activity::Definition
@plain_client: Temporalio::Client
@encoded_detail: Temporalio::Api::Common::V1::Payload

def initialize: (Temporalio::Client plain_client, Temporalio::Api::Common::V1::Payload encoded_detail) -> void
def execute: () -> untyped
end

class AsyncCompletionActivity < Temporalio::Activity::Definition
@task_token: Queue

Expand Down Expand Up @@ -85,6 +93,10 @@ class WorkerExternalStorageTest < Test
Temporalio::Converters::StorageDriver driver,
?payload_codec: Temporalio::Converters::PayloadCodec?
) -> Temporalio::Client
def encoded_reference: (
Temporalio::Converters::StorageDriver driver,
String value
) -> Temporalio::Api::Common::V1::Payload
def first_attrs: (Enumerable[untyped] events, Symbol attrs_method) -> untyped
def assert_reference: (Temporalio::Api::Common::V1::Payload payload) -> void
end
Loading
Loading