Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ strands-agents-evals = [
"strands-agents-evals>=1.0.3,<2.0.0"
]
simulation = [
"jinja2>=3.1.0",
"jinja2>=3.1.6",
"strands-agents-evals>=1.0.3,<2.0.0",
]
datasets = [
Expand Down
3 changes: 2 additions & 1 deletion src/bedrock_agentcore/evaluation/runner/dataset_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ class SimulationConfig(BaseModel):
default model when None.
system_prompt_template: Jinja2 system prompt template for the actor.
Must contain an ``{{ actor_profile }}`` placeholder. When None, the built-in
``structured_user_simulator.j2`` template is used.
``structured_user_simulator.j2`` template is used. The template is rendered
in an isolated Jinja2 environment.
input_type: Pydantic model class describing the agent's expected input.
When set, ``input`` values in SimulatedScenario are validated into
this type for the first agent call. For subsequent turns the actor is
Expand Down
46 changes: 40 additions & 6 deletions src/bedrock_agentcore/evaluation/runner/prompts/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,28 @@
:func:`render_template_string` for caller-supplied template strings.
"""

import re
from pathlib import Path
from typing import Any

_PROMPTS_DIR = Path(__file__).parent
_environment = None
_string_environment = None

_JINJA2_IMPORT_ERROR = (
"jinja2 is required for SimulatedScenario execution. Install it with: pip install 'bedrock-agentcore[simulation]'"
)
_MIN_STRING_TEMPLATE_JINJA2_VERSION = (3, 1, 6)


def _get_environment():
"""Return the environment for built-in, trusted ``.j2`` files shipped with the SDK."""
global _environment
if _environment is None:
try:
from jinja2 import Environment, FileSystemLoader
except ImportError as e:
raise ImportError(
"jinja2 is required for SimulatedScenario execution. "
"Install it with: pip install 'bedrock-agentcore[simulation]'"
) from e
raise ImportError(_JINJA2_IMPORT_ERROR) from e
_environment = Environment( # nosec B701 - templates render plain text/JSON, not HTML
loader=FileSystemLoader(_PROMPTS_DIR),
autoescape=False,
Expand All @@ -30,6 +35,32 @@ def _get_environment():
return _environment


def _get_string_environment():
"""Return an isolated environment for caller-supplied template strings.

``ImmutableSandboxedEnvironment`` keeps input values unchanged, while omitting a
loader keeps template strings independent from the bundled template files.
"""
global _string_environment
if _string_environment is None:
try:
import jinja2
from jinja2.sandbox import ImmutableSandboxedEnvironment
except ImportError as e:
raise ImportError(_JINJA2_IMPORT_ERROR) from e
version = tuple(int(part) for part in re.findall(r"\d+", jinja2.__version__)[:3])
if version < _MIN_STRING_TEMPLATE_JINJA2_VERSION:
raise ImportError(
f"jinja2>=3.1.6 is required to render system_prompt_template "
f"(found {jinja2.__version__}). Install it with: pip install 'bedrock-agentcore[simulation]'"
)
_string_environment = ImmutableSandboxedEnvironment( # nosec B701 - templates render plain text/JSON
autoescape=False,
keep_trailing_newline=True,
)
return _string_environment


def render_template_file(name: str, **kwargs: Any) -> str:
"""Render a built-in ``.j2`` template file by name.

Expand All @@ -44,13 +75,16 @@ def render_template_file(name: str, **kwargs: Any) -> str:


def render_template_string(template_str: str, **kwargs: Any) -> str:
"""Render a caller-supplied Jinja2 template string.
"""Render a caller-supplied Jinja2 template string in a sandbox.

Args:
template_str: A Jinja2 template string (use ``{{ variable }}`` syntax).
**kwargs: Variables substituted into the template.

Returns:
The rendered template string.

Raises:
jinja2.exceptions.SecurityError: If the template attempts an unsafe operation.
"""
return _get_environment().from_string(template_str).render(**kwargs)
return _get_string_environment().from_string(template_str).render(**kwargs)
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,40 @@ def test_render_template_string_handles_json_curly_braces(self):
result = render_template_string(template, schema=schema)
assert schema in result

def test_render_template_string_supports_control_flow(self):
template = "{% if scenario_description %}Scenario: {{ scenario_description }}{% endif %}"
template += "|{{ actor_profile.goal }}"
result = render_template_string(template, scenario_description="pizza", actor_profile={"goal": "order"})
assert result == "Scenario: pizza|order"

def test_render_template_string_restricts_private_attribute_access(self):
from jinja2.exceptions import SecurityError

with pytest.raises(SecurityError):
render_template_string("{{ self.__init__.__globals__ }}")

def test_render_template_string_requires_supported_jinja2(self, monkeypatch):
import jinja2

import bedrock_agentcore.evaluation.runner.prompts as prompts

monkeypatch.setattr(prompts, "_string_environment", None)
monkeypatch.setattr(jinja2, "__version__", "3.1.5")
with pytest.raises(ImportError, match=r"jinja2>=3\.1\.6"):
render_template_string("{{ x }}", x=1)

def test_render_template_string_blocks_mutation_of_inputs(self):
from jinja2.exceptions import SecurityError

profile = {"traits": ["calm"]}
with pytest.raises(SecurityError):
render_template_string("{{ actor_profile.traits.append('focused') }}", actor_profile=profile)
assert profile == {"traits": ["calm"]}

def test_render_template_string_cannot_include_files(self):
with pytest.raises(TypeError):
render_template_string("{% include 'structured_user_simulator.j2' %}")

def test_render_system_prompt_output_type_injected(self, mock_strands):
"""output_type schema must appear in the rendered prompt."""
_, _, mock_profile_cls = mock_strands
Expand Down
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading