diff --git a/.github/workflows/architecture-pages.yml b/.github/workflows/architecture-pages.yml new file mode 100644 index 000000000..b7b02f65e --- /dev/null +++ b/.github/workflows/architecture-pages.yml @@ -0,0 +1,124 @@ +name: Architecture GitHub Pages + +on: + push: + branches: [dev, vrdmr-docs-architecture-pages] + paths: + - "workers/**" + - "runtimes/**" + - "eng/docs/**" + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/architecture-pages.yml" + pull_request: + branches: [dev] + paths: + - "workers/**" + - "runtimes/**" + - "eng/docs/**" + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/architecture-pages.yml" + repository_dispatch: + types: [sibling-repo-updated] + schedule: + # Daily cross-repo sync with python-library, python-extensions, and language-worker-protobuf + - cron: "0 8 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: "pages-architecture-${{ github.ref }}" + cancel-in-progress: true + +jobs: + build: + name: Build & Validate Architecture Site + runs-on: ubuntu-latest + steps: + - name: Checkout python-worker + uses: actions/checkout@v4 + + - name: Checkout azure-functions-host (gRPC channel metadata) + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-host + path: _sibling_repos/functions-host + fetch-depth: 1 + sparse-checkout: src/WebJobs.Script.Grpc + + - name: Checkout azure-functions-python-library + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-python-library + path: _sibling_repos/python-library + fetch-depth: 1 + + - name: Checkout azure-functions-python-extensions + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-python-extensions + path: _sibling_repos/python-extensions + fetch-depth: 1 + + - name: Checkout azure-functions-language-worker-protobuf + uses: actions/checkout@v4 + with: + repository: Azure/azure-functions-language-worker-protobuf + path: _sibling_repos/language-worker-protobuf + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: "pip" + cache-dependency-path: eng/docs/requirements.txt + + - name: Install MkDocs Material + run: python3 -m pip install -r eng/docs/requirements.txt + + - name: Run architecture extractor & Markdown sync unit tests + run: python3 -m unittest discover -s eng/docs/tests -v + + - name: Extract multi-repo architecture graph & check Markdown blocks + run: python3 eng/docs/extract_arch.py --sibling-root _sibling_repos --check-markdown + + - name: Build MkDocs Material documentation + run: mkdocs build --strict + + - name: Upload preview artifact + uses: actions/upload-artifact@v4 + with: + name: architecture-site + path: ./site + + - name: Upload GitHub Pages artifact + if: github.event_name != 'pull_request' + uses: actions/upload-pages-artifact@v3 + with: + path: ./site + + deploy: + name: Deploy to GitHub Pages + if: github.event_name != 'pull_request' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Configure Pages + uses: actions/configure-pages@v5 + with: + enablement: true + continue-on-error: true + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index ccab2a249..b19305502 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,8 @@ tests/**/extensions.csproj __blobstorage__/* __queuestorage__/* __azurite* + +# Generated architecture site artifacts +docs/assets/architecture-graph.json +docs/assets/architecture-data.js +_sibling_repos/ diff --git a/docs/architecture/dependency-isolation.md b/docs/architecture/dependency-isolation.md new file mode 100644 index 000000000..436ae8a45 --- /dev/null +++ b/docs/architecture/dependency-isolation.md @@ -0,0 +1,47 @@ +# Dependency Isolation & Placeholder Specialization + +In Linux Consumption and Flex Consumption, warm standby ("placeholder") containers boot the worker process **before** any customer function app or `.python_packages` directory is mounted. When a container is specialized to a customer site via `FunctionEnvironmentReloadRequest` (or during `WorkerInitRequest` on Dedicated/Premium plans), the worker must promote customer dependencies ahead of bundled worker packages. + +--- + +## `sys.path` Search Priority After Specialization + +Across both `proxy_worker` (`3.13–3.14`) and `r2p2` (`3.15+`), specialized `sys.path` ordering guarantees customer-pinned packages take precedence: + +```text +1. Customer packages (/.python_packages/lib/site-packages) +2. Bundled worker runtime (workers_dir — fallback azure_functions_runtime v2/v1) +3. Customer app directory (function_app_directory — function_app.py & user modules) +``` + +--- + +## Evolution of Isolation Across Worker Generations + +### Python ≤ 3.14 (`DependencyManager`) +In `workers/azure_functions_worker/utils/dependency.py` and `workers/proxy_worker/utils/dependency.py`: + +1. **Cold Placeholder Boot (`use_worker_dependencies`)**: Forces worker dependencies first on `sys.path` so `grpcio` and `google.protobuf` C-extensions load from the worker's verified bundle rather than unverified paths. +2. **Specialization (`prioritize_customer_dependencies`)**: Reorders `sys.path` so customer `.python_packages` come first and clears previously imported worker copies of customer-overridable libraries. +3. **Structural Limitation**: Because `google.protobuf` uses a process-global C descriptor pool (`upb` / C++ extension), loading `protobuf` in the worker process at boot could still clash with incompatible `protobuf` versions pinned by customer apps. + +### Python 3.15+ R2P2 (`bridge.py` + Namespace-Safe Eviction) +R2P2 eliminates transport collisions completely and hardens `sys.modules` eviction in [`workers/r2p2/bridge/bridge.py`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/bridge/bridge.py): + +1. **Zero Transport Pollution**: Rust (`tonic` + `prost`) owns gRPC and Protobuf on the native side, while `protos_adapter.py` is pure stdlib Python. Neither `grpcio` nor `google.protobuf` is ever imported into `sys.modules`. +2. **PEP 420 Namespace-Preserving Module Eviction (`_clear_modules_from_path`)**: + - **Retains shared namespace parents**: Namespace packages without `__file__` (such as `azure` and `azurefunctions`) have no module initializer to re-run and dynamically follow `sys.path`; keeping them preserves already-attached customer child packages. + - **Evicts file-backed worker packages & descendants**: Any module whose `__file__` resolves under `workers_dir` is evicted together with all cached submodule descendants. + - **Detaches stale parent attributes**: When evicting `parent.child` from a retained `parent` namespace, `parent.__dict__.pop("child", None)` removes the stale attribute so `from parent import child` cleanly re-imports from the customer's `.python_packages`. + - **Protected bridge modules**: `bridge`, `protos_adapter`, and active runtime prefixes remain protected across specialization, followed by `importlib.invalidate_caches()`. + +```mermaid +flowchart LR + Reload["FunctionEnvironmentReloadRequest"] --> Clear["_clear_modules_from_path(workers_dir)"] + Clear --> NS{"Module has __file__
under workers_dir?"} + NS -->|"No (PEP 420 namespace:
azure, azurefunctions)"| Keep["Keep namespace parent in sys.modules
(search path tracks new sys.path)"] + NS -->|"Yes (file-backed worker pkg)"| Evict["Evict module + cached descendants
& pop stale attr on retained parent"] + Keep --> Reorder["_prioritize_customer_dependencies(app_dir)"] + Evict --> Reorder + Reorder --> Invalidate["importlib.invalidate_caches()
+ _select_runtime(app_dir)"] +``` diff --git a/docs/architecture/ecosystem.md b/docs/architecture/ecosystem.md new file mode 100644 index 000000000..0f9b85556 --- /dev/null +++ b/docs/architecture/ecosystem.md @@ -0,0 +1,109 @@ +# 5-Repository Ecosystem Architecture + +The Azure Functions Python stack is cleanly partitioned across **five GitHub repositories**, separating the `.NET` Script Host runtime, the language-neutral gRPC wire contract, the Python/Rust worker shells & runtimes, the customer-facing SDK, and optional framework/SDK extensions. + +
+ +--- + +## Repository Boundaries & Cross-Repo Topology + + + +| Repository | PyPI Packages | Architectural Role | +| --- | --- | --- | +| [`Azure/azure-functions-host`](https://github.com/Azure/azure-functions-host) | — | .NET Functions Script Host runtime: event/trigger listeners, WorkerChannel gRPC server, Consumption specialization, scale probes, HTTP v2 proxy, and AppInsights/Kusto telemetry. | +| [`Azure/azure-functions-language-worker-protobuf`](https://github.com/Azure/azure-functions-language-worker-protobuf) | — | Canonical gRPC wire contract (`FunctionRpc.proto`, `settlement.proto`, `ClaimsIdentityRpc.proto`) shared between `azure-functions-host` and all language workers. | +| [`Azure/azure-functions-python-worker`](https://github.com/Azure/azure-functions-python-worker) | `azure-functions-runtime`, `azure-functions-runtime-v1` | Worker transports (monolith, proxy_worker, R2P2 Rust) and v1/v2 execution runtimes. | +| [`Azure/azure-functions-python-library`](https://github.com/Azure/azure-functions-python-library) | `azure-functions` | Customer-facing Python SDK: V2 @app decorators, Blueprints, Datum converters, Trigger/Binding types, Out[T], Context. | +| [`Azure/azure-functions-python-extensions`](https://github.com/Azure/azure-functions-python-extensions) | `azurefunctions-extensions-base`, `azurefunctions-extensions-http-fastapi`, `azurefunctions-extensions-bindings-blob`, `azurefunctions-extensions-bindings-servicebus`, `azurefunctions-agents-extensions-*` | Extension contracts: WebApp HTTP v2 streaming, SdkType deferred bindings, Settlement gRPC client, and Custom/Agent Runtime hooks. | + +### Full 5-Repository Topology + +```mermaid +flowchart TB + subgraph ce_host["Azure/azure-functions-host (.NET Script Host)"] + ne_host_chan["Script Host & WorkerChannel
(functions-host)"] + ne_host_duplex["ServerDuplexChannel & ShMem
(functions-host)"] + end + subgraph ce_proto["Azure/azure-functions-language-worker-protobuf (Contract)"] + ne_funcrpc["FunctionRpc.proto
(worker-protobuf)"] + ne_settle["settlement.proto & ClaimsIdentity
(worker-protobuf)"] + end + subgraph ce_worker["Azure/azure-functions-python-worker"] + subgraph ce_w_shell["Worker Transport Shells (workers/)"] + ne_shells["Transports: Monolith / Proxy / R2P2
(python-worker)"] + end + subgraph ce_w_rt["Execution Runtimes (runtimes/v1 & v2)"] + ne_runtimes["azure-functions-runtime (v2 & v1)
(python-worker)"] + end + end + subgraph ce_lib["Azure/azure-functions-python-library"] + ne_lib_dec["Decorators & Blueprints
(python-library)"] + ne_lib_datum["meta.py Datum & Converters
(python-library)"] + end + subgraph ce_ext["Azure/azure-functions-python-extensions"] + ne_ext_base["extensions-base (WebApp, SdkType, Runtime)
(python-extensions)"] + ne_ext_pkgs["FastAPI, SDK Bindings & Agent Runtimes
(python-extensions)"] + end + ne_host_chan -->|"dispatch / RpcLog sink"| ne_host_duplex + ne_host_duplex -->|"EventStream server"| ne_funcrpc + ne_host_chan -.->|"Settlement server"| ne_settle + ne_funcrpc -->|"prost / grpcio client"| ne_shells + ne_shells -->|"WorkerRequest / PyO3 FFI"| ne_runtimes + ne_runtimes -->|"index_function_app()"| ne_lib_dec + ne_runtimes -->|"Datum <-> Binding decode"| ne_lib_datum + ne_runtimes -->|"http_v2 / deferred SdkType"| ne_ext_base + ne_shells -.->|"entry_points(azurefunctions.runtimes)"| ne_ext_base + ne_settle -.->|"grpcClientType settlement"| ne_ext_base + ne_ext_base -->|"metaclass registration"| ne_ext_pkgs +``` + +### End-to-End 5-Repo Request Lifecycle + +```mermaid +sequenceDiagram + participant host as functions-host + participant proto as worker-protobuf + participant shell as python-worker (Shell) + participant rt as python-worker (Runtime) + participant lib as library & extensions + host->>proto: WorkerChannel serializes event onto ServerDuplexChannel + proto->>shell: FunctionRpc.proto StreamingMessage frame over HTTP/2 + shell->>rt: Dispatch WorkerInit / Invocation across seam + rt->>lib: Decode Datum (python-library) or SdkType/WebApp (python-extensions) + lib-->>host: InvocationResponse / RpcLog / Settlement back to Host +``` + +### Key Cross-Repository Components + +| Component | Repository | Source Anchor | Responsibility | +| --- | --- | --- | --- | +| **Script Host & WorkerChannel** | `functions-host` | [`src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs`](https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs) | .NET Script Host orchestrating triggers, worker process lifecycle, placeholder specialization, status probes, and RpcLog telemetry. | +| **ServerDuplexChannel & ShMem** | `functions-host` | [`src/WebJobs.Script.Grpc/Channel/ServerDuplexChannel.cs`](https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.Grpc/Channel/ServerDuplexChannel.cs) | Host-side bidirectional gRPC `EventStream` endpoint and POSIX/Windows shared-memory data transfer. | +| **FunctionRpc.proto** | `worker-protobuf` | [`src/proto/FunctionRpc.proto`](https://github.com/Azure/azure-functions-language-worker-protobuf/blob/dev/src/proto/FunctionRpc.proto) | Canonical bidirectional gRPC protocol contract shared between `azure-functions-host` and all out-of-proc language workers. | +| **settlement.proto & ClaimsIdentity** | `worker-protobuf` | [`src/proto/servicebus/settlement.proto`](https://github.com/Azure/azure-functions-language-worker-protobuf/blob/dev/src/proto/servicebus/settlement.proto) | Protobuf service schema enabling Python extensions to settle ServiceBus messages directly against the Host. | +| **Transports: Monolith / Proxy / R2P2** | `python-worker` | [`workers/r2p2/src/main.rs:L231`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/src/main.rs#L231) | Version-selected worker shell managing Host gRPC stream, logging, and sys.path dependency isolation. | +| **azure-functions-runtime (v2 & v1)** | `python-worker` | [`runtimes/v2/azure_functions_runtime/handle_event.py:L69`](https://github.com/Azure/azure-functions-python-worker/blob/dev/runtimes/v2/azure_functions_runtime/handle_event.py#L69) | PyPI-published execution engines that index apps, decode/encode bindings, and run sync/async handlers. | +| **Decorators & Blueprints** | `python-library` | [`azure/functions/__init__.py`](https://github.com/Azure/azure-functions-python-library/blob/dev/azure/functions/__init__.py) | Defines V2 Python programming model (`@app.route`, `@app.queue_trigger`, `Blueprint`, MCP & Connector triggers). | +| **meta.py Datum & Converters** | `python-library` | [`azure/functions/meta.py`](https://github.com/Azure/azure-functions-python-library/blob/dev/azure/functions/meta.py) | Protobuf-agnostic `Datum` value container and trigger/binding converters (`HttpRequest`, `QueueMessage`, `Blob`). | +| **extensions-base (WebApp, SdkType, Runtime)** | `python-extensions` | [`azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py`](https://github.com/Azure/azure-functions-python-extensions/blob/dev/azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py) | Metaclass registries bridging HTTP v2 ASGI servers, SDK-type deferred bindings, and custom Agent Runtimes. | +| **FastAPI, SDK Bindings & Agent Runtimes** | `python-extensions` | [`azurefunctions-extensions-http-fastapi/README.md`](https://github.com/Azure/azure-functions-python-extensions/blob/dev/azurefunctions-extensions-http-fastapi/README.md) | Concrete extension packages for FastAPI HTTP streaming, Blob/Cosmos/EventHub/ServiceBus SDK types, and Agents. | + + + +--- + +## How Contracts Connect the 5 Repositories + +1. **`azure-functions-host` ↔ `azure-functions-language-worker-protobuf` ↔ `azure-functions-python-worker`**: + - `WorkerChannel.cs` and `ServerDuplexChannel.cs` in `azure-functions-host` host the gRPC `FunctionRpc.EventStream` server defined by `FunctionRpc.proto`. + - Neither `azure-functions-host` nor `FunctionRpc.proto` needed protocol changes when Python moved from the `≤ 3.12` monolith to `3.13` `proxy_worker` or `3.15+` Rust `r2p2`. +2. **`azure-functions-python-worker` ↔ `azure-functions-python-library`**: + - `azure-functions` (`python-library`) defines `@app.*` decorators, `Blueprint`s, `Out[T]`, `Context`, and the protobuf-free `Datum(value, type)` intermediate representation (`azure/functions/meta.py`). + - R2P2's `native_invocation.py` constructs `Datum` objects directly from Rust `PyO3` tuples—meaning standard triggers and bindings execute with **zero Python protobuf dependencies**. +3. **`azure-functions-python-worker` ↔ `azure-functions-python-extensions`**: + - `azurefunctions-extensions-base` defines three metaclass extension seams: + - `WebApp` (`web.py`) — local loopback ASGI/WSGI HTTP v2 streaming (`azurefunctions-extensions-http-fastapi`). + - `SdkType` (`sdkType.py`) — deferred bindings hydrating rich Azure SDK clients (`BlobClient`, `ContainerClient`, Service Bus `ServiceBusReceivedMessage` + `settlement.proto` gRPC client). + - `RuntimeTrackerMeta` (`runtime.py`) — custom execution engines discovered via `entry_points(group='azurefunctions.runtimes')` when `PYTHON_ENABLE_AGENT_RUNTIME=1`. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md new file mode 100644 index 000000000..1bfd908b0 --- /dev/null +++ b/docs/architecture/overview.md @@ -0,0 +1,53 @@ +# Three-Generation Evolution Overview + +Historically, the Azure Functions Python worker shipped as a single monolithic package (`azure_functions_worker`) coupling gRPC transport, protobuf serialization, dependency isolation, indexing, bindings, and execution. Over Python `3.13` and `3.15`, the architecture evolved in two deliberate steps: + +```mermaid +flowchart TB + subgraph E1["Python <= 3.12 (Monolith)"] + H1["Functions Host (.NET)"] <-->|"gRPC (grpcio + protobuf)"| M1["azure_functions_worker
Transport + Indexing + Bindings + Executor"] + M1 --> U1["Customer Function App"] + end + + subgraph E2["Python 3.13 - 3.14 (Proxy + Runtime Split)"] + H2["Functions Host (.NET)"] <-->|"gRPC (grpcio + protobuf)"| P2["proxy_worker
Thin Transport Shell"] + P2 -->|"In-Proc WorkerRequest
+ injected protos"| R2["azure-functions-runtime (v2 / v1 / Agent)
Independently Shipped on PyPI"] + R2 --> U2["Customer Function App"] + end + + subgraph E3["Python 3.15+ (R2P2 — Rust to Python Proxy)"] + H3["Functions Host (.NET)"] <-->|"Same gRPC Wire
(Rust tonic + prost)"| P3["r2p2 Native Binary
Zero Python grpcio / protobuf"] + P3 -->|"PyO3 FFI (GIL):
Hot Datum Tuples / Control Dicts"| R3["Embedded CPython 3.15
native_invocation.py + runtimes (v2 / v1)"] + R3 --> U3["Customer Function App"] + end +``` + +--- + +## Side-by-Side Architectural Comparison + +| Architectural Dimension | Python ≤ 3.12 (Monolith) | Python 3.13 – 3.14 (Proxy Split) | Python 3.15+ (R2P2 Rust Worker) | +| --- | --- | --- | --- | +| **Source Path** | `workers/azure_functions_worker` | `workers/proxy_worker` + `runtimes/{v1,v2}` | `workers/r2p2` + `runtimes/{v1,v2}` | +| **Process & Launcher** | `prodV4/worker.py` → CPython | `proxyV4/worker.py` → CPython | Compiled `r2p2` Rust binary embedding `libpython3.15` (GIL) | +| **gRPC & Wire Codec** | Python `grpcio` + `google.protobuf` | Python `grpcio` + worker `protobuf` + `uvloop` | Native Rust `tonic` + `prost` + `tokio` (**0 Python protobuf**) | +| **Transport ↔ Runtime Seam** | None (direct internal imports) | In-process async `WorkerRequest` + injected `protos` | `PyO3` FFI: `invoke_native` `Datum` tuples (hot) + `protos_adapter` dicts (control) | +| **Runtime Release Cadence** | Coupled to Host worker image releases | Decoupled on PyPI (`azure-functions-runtime`) | Decoupled on PyPI (`azure-functions-runtime`) | +| **Day-1 New Python Support** | Blocked waiting for `grpcio` / `protobuf` C-extension wheels | Blocked waiting for `grpcio` / `protobuf` C-extension wheels | **Unblocked** — Rust owns wire; Python runtime is pure Python | +| **Customer Protobuf Collisions** | Vulnerable (`google.protobuf` C descriptor pool shared) | Mitigated via isolation, still shares C descriptor pool | **Eliminated permanently** — worker loads zero `google.protobuf` | +| **`WorkerStatusRequest` Probe** | Handled on Python `asyncio` loop (contends on GIL) | Short-circuited in `proxy_worker` (still contends on GIL) | **Zero-GIL pure Rust reply** (`src/bridge.rs`) | +| **V2 Function Indexing Phase** | Lazy in `FunctionsMetadataRequest` by default (`PYTHON_ENABLE_INIT_INDEXING` opt-in) | Eager during `WorkerInitRequest` / `EnvironmentReload` | Eager during `WorkerInitRequest` / `EnvironmentReload` | + +--- + +## Why Each Transition Happened + +### 1. Why Split `proxy_worker` and `runtimes/` in Python 3.13? +In `≤ 3.12`, shipping any binding fix, OpenTelemetry enhancement, or HTTP v2 streaming improvement required rebuilding and rolling out the entire worker bundle across Azure Functions Host images for every Python version. Splitting `proxy_worker` (thin, rarely changing transport) from `azure-functions-runtime` (PyPI-published execution engine) decoupled their release cadences and unlocked third-party/custom **Agent Runtimes** (`PYTHON_ENABLE_AGENT_RUNTIME=1`). + +### 2. Why Move the Proxy Shell to Rust (`R2P2`) in Python 3.15? +Even with the 3.13 split, `proxy_worker` remained written in Python and linked `grpcio` and `google.protobuf` C-extensions: + +1. **New Python release bottleneck**: `grpcio` and `protobuf` C-extension wheels routinely lag new CPython releases, blocking day-one Python 3.15+ support on Azure Functions. +2. **Descriptor-pool collisions**: Sharing an interpreter between worker `protobuf` and customer-pinned `protobuf` packages caused recurring packaging friction. +3. **Transport overhead**: Replacing `grpcio` + Python protobuf marshaling with native `tonic` + `prost` + a direct `PyO3` `Datum`-tuple FFI seam (`native_invocation.py`) eliminates Python-side wire serialization overhead on the standard GIL build while keeping `runtimes/v1`, `runtimes/v2`, and customer code 100% Python. diff --git a/docs/architecture/python-312.md b/docs/architecture/python-312.md new file mode 100644 index 000000000..c42dd994e --- /dev/null +++ b/docs/architecture/python-312.md @@ -0,0 +1,679 @@ +# Python ≤ 3.12 Architecture — Monolithic Worker + +For **Python 3.7 through 3.12**, the worker ships as a single monolithic Python package under [`workers/azure_functions_worker`](https://github.com/Azure/azure-functions-python-worker/tree/dev/workers/azure_functions_worker), launched by [`workers/python/prodV4/worker.py`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/python/prodV4/worker.py). + +Transport (`grpcio` + `google.protobuf`), `DependencyManager`, function indexing (`loader.py`), binding serialization (`bindings/`), shared-memory transfer (`SharedMemoryManager`), HTTP v2 streaming (`http_v2.py`), worker extensions (`extension.py`), and the synchronous `ThreadPoolExecutor` (`_sync_call_tp`) all reside inside one coupled module tree. + +
+ +--- + +## Architecture & Component Reference + + + +### Full Static Topology (Mermaid) + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"ServerDuplexChannel"| n312_proto + n312_proto -->|"EventStream (grpcio)"| n312_grpc + n312_boot -->|"boot + DependencyManager"| n312_disp + n312_grpc -->|"call_soon_threadsafe / Queue"| n312_disp + n312_disp -->|"index & load"| n312_load + n312_disp -->|"TypedData <-> args"| n312_bind + n312_load -->|"inspect @app / function.json"| n312_sdk + n312_bind -->|"Datum decode + invoke"| n312_sdk + n312_bind -.->|"WebApp / SdkType"| n312_ext +``` + +### Component & Source Reference + +| Component | Repository | Source Anchor | Responsibility | +| --- | --- | --- | --- | +| **Script Host (WorkerChannel)** | `functions-host` | [`src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs`](https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs) | .NET Functions Host runtime spawning `prodV4/worker.py`, driving triggers, scale probes, and telemetry. | +| **FunctionRpc.proto Contract** | `worker-protobuf` | [`src/proto/FunctionRpc.proto`](https://github.com/Azure/azure-functions-language-worker-protobuf/blob/dev/src/proto/FunctionRpc.proto) | Language-neutral gRPC `.proto` contract between `azure-functions-host` and `azure_functions_worker`. | +| **prodV4/worker.py & main.py** | `python-worker` | [`workers/azure_functions_worker/main.py:L70`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/main.py#L70) | Sets worker-first sys.path so bundled `grpcio`/`protobuf` import cleanly, then starts asyncio loop. | +| **gRPC Pump (__poll_grpc)** | `python-worker` | [`workers/azure_functions_worker/dispatcher.py:L155`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/dispatcher.py#L155) | Background OS thread reading `stub.EventStream` and posting tasks via `call_soon_threadsafe`. | +| **Monolithic Dispatcher** | `python-worker` | [`workers/azure_functions_worker/dispatcher.py:L90`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/dispatcher.py#L90) | Central class owning message routing, logging switch, OTel/AppInsights setup, and `_sync_call_tp`. | +| **DependencyManager** | `python-worker` | [`workers/azure_functions_worker/utils/dependency.py:L25`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/utils/dependency.py#L25) | Manages worker-first placeholder boot vs customer `.python_packages` promotion on Init/Specialization. | +| **Loader & ExtensionManager** | `python-worker` | [`workers/azure_functions_worker/loader.py:L249`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/loader.py#L249) | Indexes V2 `@app` decorators or V1 `function.json` files and invokes `AppExtensionBase` hooks. | +| **Bindings, SharedMem & http_v2** | `python-worker` | [`workers/azure_functions_worker/http_v2.py:L261`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/azure_functions_worker/http_v2.py#L261) | Converts `protos.TypedData` <-> `Datum`, manages shared-memory rings, and coordinates HTTP v2 streaming. | +| **azure-functions SDK & User App** | `python-library` | [`azure/functions/meta.py`](https://github.com/Azure/azure-functions-python-library/blob/dev/azure/functions/meta.py) | Customer decorators, trigger/binding models (`HttpRequest`, `QueueMessage`), and `Datum` converters. | +| **azurefunctions-extensions-*** | `python-extensions` | [`azurefunctions-extensions-base/azurefunctions/extensions/base/sdkType.py`](https://github.com/Azure/azure-functions-python-extensions/blob/dev/azurefunctions-extensions-base/azurefunctions/extensions/base/sdkType.py) | Optional HTTP v2 ASGI streaming (`WebApp`) and deferred SDK bindings (`BlobClient`, etc.). | + + + +--- + +## The 7 Worker Lifecycle Phases (Python ≤ 3.12) + +Each Host ↔ Worker lifecycle phase exercises a distinct subset of components and interactions in `azure_functions_worker`. + + + +### 1. Worker Initialization Flow + + +--- + +- **Wire Protocol:** `StartStream → WorkerInitRequest → WorkerInitResponse` + +--- + +- **Execution Scope:** All SKUs (Cold Start) + + +--- + +Host spawns `prodV4/worker.py`, which boots `azure_functions_worker` with worker dependencies isolated first (`grpcio` + `protobuf`). Once `EventStream` connects and sends `StartStream`, `WorkerInitRequest` negotiates capabilities (`SHARED_MEMORY`, `RAW_HTTP_BODY_BYTES`, `WORKER_STATUS`), initializes OTel/AppInsights, promotes customer `.python_packages` (on Dedicated/EP), loads the binding registry, and optionally pre-indexes if `PYTHON_ENABLE_INIT_INDEXING=1`. + + +--- + +> **Bypassed / Idle Note:** In Linux Consumption placeholder mode, customer packages are NOT mounted yet during Init (deferred to Phase 2 Specialization). + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant boot as prodV4 + main.py + participant grpc as gRPC Thread + participant disp as Monolith Dispatcher + participant dep as DependencyManager + host->>boot: Spawn python prodV4/worker.py --host --port --workerId + boot->>grpc: Dispatcher.connect() opens insecure grpcio channel + grpc-->>host: Send StartStream(worker_id) + detach console logger + host->>disp: WorkerInitRequest(capabilities, function_app_directory) + disp->>dep: should_load_cx_dependencies() -> prioritize_customer_deps() + disp->>disp: Configure OTel/AppInsights + bindings.load_binding_registry() + disp-->>host: WorkerInitResponse(capabilities, worker_metadata) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. spawn prodV4/worker.py"| n312_boot + n312_boot -->|"2. use_worker_deps + connect()"| n312_grpc + n312_grpc -->|"3. StartStream(worker_id)"| n312_proto + n312_proto -->|"4. WorkerInitRequest"| n312_grpc + n312_grpc -->|"5. call_soon_threadsafe"| n312_disp + n312_disp -->|"6. prioritize_customer_deps()"| n312_dep + n312_disp -->|"7. load_binding_registry()"| n312_bind +``` + +--- + +### 2. Worker Specialization Flow (Consumption Placeholder) + + +--- + +- **Wire Protocol:** `FunctionEnvironmentReloadRequest → FunctionEnvironmentReloadResponse` + +--- + +- **Execution Scope:** Linux / Flex Consumption Only (Placeholder Mode) + + +--- + +Warm standby containers boot before any customer app is assigned. When specialized, the Host mounts the customer package and sends `FunctionEnvironmentReloadRequest` carrying `function_app_directory` and `environment_variables`. The monolith clears `os.environ`, calls `time.tzset()`, clears `sys.path_importer_cache`, promotes customer `.python_packages`, recreates `_sync_call_tp` (`PYTHON_THREADPOOL_THREAD_COUNT`), and changes `cwd`. + + +--- + +> **Bypassed / Idle Note:** Never sent on Dedicated/Premium cold boots or worker restarts (those go straight through WorkerInit). + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant disp as Monolith Dispatcher + participant dep as DependencyManager + participant sdk as Customer Packages + host->>disp: FunctionEnvironmentReloadRequest(app_dir, env_vars) + disp->>disp: os.environ.clear() + apply env_vars + time.tzset() + disp->>dep: sys.path_importer_cache.clear() + prioritize_customer_deps() + disp->>disp: bindings.load_binding_registry() + os.chdir(app_dir) + disp-->>host: FunctionEnvironmentReloadResponse(capabilities) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. FunctionEnvironmentReloadRequest"| n312_grpc + n312_grpc -->|"2. _dispatch_grpc_request"| n312_disp + n312_disp -->|"3. reload os.environ + sys.path"| n312_dep + n312_dep -.->|"4. reload customer azure.functions"| n312_sdk + n312_disp -->|"5. optional init indexing + http_v2"| n312_load + n312_disp -->|"6. recreate _sync_call_tp + bindings"| n312_bind +``` + +--- + +### 3. Worker Metadata & Indexing Flow + + +--- + +- **Wire Protocol:** `FunctionsMetadataRequest → FunctionMetadataResponse` + +--- + +- **Execution Scope:** V2 Worker-Indexed Apps (Returns Fallback Flag for V1) + + +--- + +Host asks the worker whether it owns function discovery. In `≤ 3.12`, `load_function_metadata()` checks if `function_app.py` exists: for **V2**, it imports `function_app.py`, walks `@app` decorators and `Blueprint`s (`python-library`) plus `WebApp`/`SdkType` hooks (`python-extensions`), and returns `RawFunctionMetadata` (`use_default_metadata_indexing=False`). For **V1**, it returns `use_default_metadata_indexing=True` so the Host scans `function.json` directories itself. + + +--- + +> **Bypassed / Idle Note:** For V1 apps (`function.json`), no customer Python code is imported during Metadata — Host falls back to disk scanning. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant disp as Monolith Dispatcher + participant load as loader.py + participant sdk as python-library / App + host->>disp: FunctionsMetadataRequest(function_app_directory) + disp->>load: Check os.path.exists(function_app.py) + load->>sdk: [V2] index_function_app() -> import function_app.py + load-->>disp: Register FunctionInfo in _functions + build RawFunctionMetadata + disp-->>host: FunctionMetadataResponse(results, use_default_metadata_indexing) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. FunctionsMetadataRequest"| n312_grpc + n312_grpc -->|"2. _handle__functions_metadata_request"| n312_disp + n312_disp -->|"3. load_function_metadata()"| n312_load + n312_load -->|"4a. V2: import function_app & @app"| n312_sdk + n312_load -.->|"4b. inspect WebApp / SdkType"| n312_ext +``` + +--- + +### 4. Worker Functions Load Flow (V1 Primary) + + +--- + +- **Wire Protocol:** `FunctionLoadRequest → FunctionLoadResponse` + +--- + +- **Execution Scope:** V1 Programming Model (`function.json`); V2 Multi-Worker / Ack + + +--- + +After the Host parses `function.json` files for a **V1** app, it sends one `FunctionLoadRequest` per function with `script_file`, `entry_point`, and `bindings`. `loader.load_function()` imports the target Python module (`__init__.py`), validates parameter type annotations against `bindings/`, and stores `FunctionInfo` in `self._functions`. (For V2 single-worker, `_functions` already holds the function from Phase 3, so it fast-acks; for 2nd+ workers in multi-proc mode, it triggers lazy V2 indexing). + + +--- + +> **Bypassed / Idle Note:** For standard V2 apps, user scripts were already loaded in Phase 3 Metadata, making FunctionLoad a fast registry check. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant disp as Monolith Dispatcher + participant load as loader.py + participant app as V1 __init__.py + host->>disp: FunctionLoadRequest(function_id, RpcFunctionMetadata) + disp->>load: loader.load_function(name, dir, script_file, entry_point) + load->>app: importlib.import_module() -> resolve main() callable + disp->>disp: _functions.add_function(function_id, func, metadata) + disp-->>host: FunctionLoadResponse(function_id, Success) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. FunctionLoadRequest(metadata)"| n312_grpc + n312_grpc -->|"2. _handle__function_load_request"| n312_disp + n312_disp -->|"3. loader.load_function(script_file)"| n312_load + n312_load -->|"4. importlib import __init__.py"| n312_sdk + n312_disp -->|"5. validate binding types & add_function"| n312_bind +``` + +--- + +### 5. Worker Invocation Flow + + +--- + +- **Wire Protocol:** `InvocationRequest → InvocationResponse` + +--- + +- **Execution Scope:** Every Function Trigger Execution + + +--- + +Host sends `InvocationRequest` with `input_data` (`TypedData` or POSIX/Win shared-memory map) and `trigger_metadata`. `Dispatcher` decodes bindings via `python-library` (`Datum`) or `python-extensions` (`SdkType` / `http_v2`), sets W3C OpenTelemetry context, executes `async def` on the event loop or `def` on `_sync_call_tp`, and marshals `Out[T]` + `$return` back to protobuf. + + +--- + +> **Bypassed / Idle Note:** `prodV4` boot and `loader.py` are idle during steady-state invocations. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant disp as Monolith Dispatcher + participant bind as Bindings / SharedMem + participant app as SDK & User Handler + host->>disp: InvocationRequest(invocation_id, function_id, input_data) + disp->>bind: from_incoming_proto() + get_context() + OTel traceparent + bind->>app: Execute async coroutine or run_in_executor(_sync_call_tp) + app-->>disp: to_outgoing_proto($return, Out) + sys.stdout.flush() + disp-->>host: Enqueue StreamingMessage(invocation_response) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. InvocationRequest"| n312_grpc + n312_grpc -->|"2. call_soon_threadsafe"| n312_disp + n312_disp -->|"3. from_incoming_proto / ShMem"| n312_bind + n312_bind -.->|"4a. http_v2 / SdkType"| n312_ext + n312_bind -->|"4b. await / _sync_call_tp"| n312_sdk + n312_disp -->|"5. _grpc_resp_queue.put"| n312_grpc +``` + +--- + +### 6. Worker Status Flow (Host Health & Scale Probe) + + +--- + +- **Wire Protocol:** `WorkerStatusRequest → WorkerStatusResponse` + +--- + +- **Execution Scope:** Host Scale Controller & Worker Health Monitoring + + +--- + +The Host periodically sends `WorkerStatusRequest` to measure worker round-trip responsiveness for out-of-proc scaling decisions. In `≤ 3.12`, `_handle__worker_status_request` suppresses all logging and returns an empty `WorkerStatusResponse` immediately—**however**, it still hop-schedules onto the single Python `asyncio` event loop and acquires the GIL. + + +--- + +> **Bypassed / Idle Note:** Loader, Bindings, SDK, Extensions, and User Code are completely untouched — pure transport/loop health probe. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Host Scale Controller + participant grpc as gRPC Thread + participant disp as Dispatcher (asyncio) + host->>grpc: WorkerStatusRequest(request_id) + grpc->>disp: call_soon_threadsafe(_handle__worker_status_request) + disp-->>host: WorkerStatusResponse() via _grpc_resp_queue +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_host -->|"WorkerChannel"| n312_proto + n312_proto -->|"1. WorkerStatusRequest"| n312_grpc + n312_grpc -->|"2. call_soon_threadsafe (GIL + loop)"| n312_disp + n312_disp -->|"3. WorkerStatusResponse (no logs)"| n312_grpc + n312_grpc -->|"4. EventStream reply"| n312_proto +``` + +--- + +### 7. Worker Logs Flow (Worker → Host Reverse Stream) + + +--- + +- **Wire Protocol:** `Python logging.LogRecord → StreamingMessage(RpcLog) → Host` + +--- + +- **Execution Scope:** Reverse Telemetry Stream (System & Customer Logs) + + +--- + +Before `EventStream` connects, logs print to stdout/stderr with prefix `LanguageWorkerConsoleLog`. Once connected, `disable_console_logging()` detaches console output and attaches `AsyncLoggingHandler` to the root logger. Every `logging.info/error` from customer code (`User` category) or worker (`System` category) is stamped with `invocation_id` (from task/thread-local) and pushed **back to the Host** on `_grpc_resp_queue` as `RpcLog` messages for Application Insights & Kusto. + + +--- + +> **Bypassed / Idle Note:** Originates inside User Code or Worker internals and flows RIGHT → LEFT back to the Host without a Host request. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant app as User Code / Worker + participant disp as AsyncLoggingHandler + participant grpc as _grpc_resp_queue + participant host as Host (AppInsights / Kusto) + app->>disp: logging.info('Order processed') / logger.info(...) + disp->>disp: Classify System vs User + resolve invocation_id + disp->>grpc: put_nowait(StreamingMessage(rpc_log=RpcLog(...))) + grpc-->>host: Stream RpcLog frame over EventStream to Host +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c312_host["Functions Host (.NET — azure-functions-host)"] + n312_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c312_wire["Wire Contract (worker-protobuf)"] + n312_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c312_worker["Monolithic Worker Process (workers/azure_functions_worker)"] + subgraph c312_trans["gRPC Transport & Isolation"] + n312_boot["prodV4/worker.py & main.py
(python-worker)"] + n312_grpc["gRPC Pump (__poll_grpc)
(python-worker)"] + n312_disp["Monolithic Dispatcher
(python-worker)"] + n312_dep["DependencyManager
(python-worker)"] + end + subgraph c312_eng["In-Tree Indexing, Bindings & Executor"] + n312_load["Loader & ExtensionManager
(python-worker)"] + n312_bind["Bindings, SharedMem & http_v2
(python-worker)"] + end + end + subgraph c312_lib["Azure/azure-functions-python-library"] + n312_sdk["azure-functions SDK & User App
(python-library)"] + end + subgraph c312_ext["Azure/azure-functions-python-extensions"] + n312_ext["azurefunctions-extensions-*
(python-extensions)"] + end + n312_proto -->|"AppInsights / Kusto"| n312_host + n312_sdk -->|"1. logging.info() -> AsyncLoggingHandler"| n312_disp + n312_disp -->|"2. stamp invocation_id -> _grpc_resp_queue"| n312_grpc + n312_grpc -->|"3. StreamingMessage(rpc_log) → Host"| n312_proto +``` + + diff --git a/docs/architecture/python-313-314.md b/docs/architecture/python-313-314.md new file mode 100644 index 000000000..e029831ff --- /dev/null +++ b/docs/architecture/python-313-314.md @@ -0,0 +1,666 @@ +# Python 3.13 – 3.14 Architecture — Split Proxy Worker & Runtimes + +Starting in **Python 3.13**, the monolithic worker was split into two independently versioned layers: + +1. **Proxy Worker (`workers/proxy_worker`, launched via `workers/python/proxyV4/worker.py`)**: A thin, stable transport shell that owns the `grpcio` `EventStream`, `uvloop` policy, console-to-`RpcLog` logging switch, 4-tier `invocation_id` correlation, `WorkerStatusRequest` fast-path, and `DependencyManager` `sys.path` isolation. +2. **Execution Runtimes (`runtimes/v2` → `azure-functions-runtime` and `runtimes/v1` → `azure-functions-runtime-v1`)**: Independently PyPI-published execution engines invoked in-process across a narrow 5-verb `WorkerRequest` coroutine contract. + +> **Why inject `protos` at `WorkerInitRequest`?** +> `proxy_worker` passes `properties={"protos": protos, "host": self._host}` on `worker_init_request` and `function_environment_reload_request`. The runtime caches `protos` in a module global, decoupling PyPI runtime releases from specific `protobuf` C-extension wheel builds. + +
+ +--- + +## Runtime Selection Priority (`reload_library_worker`) + +During `WorkerInitRequest` (and again on `FunctionEnvironmentReloadRequest`), `Dispatcher.reload_library_worker(directory)` selects the execution engine in strict priority order: + +1. **Custom / Agent Runtime (`PYTHON_ENABLE_AGENT_RUNTIME=1`)**: Discovers `importlib.metadata.entry_points(group="azurefunctions.runtimes")` and loads the registered package via `azurefunctions.extensions.base.RuntimeTrackerMeta`. +2. **V2 Decorator Runtime (`azure_functions_runtime`)**: Selected when `function_app.py` (or `PYTHON_SCRIPT_FILE_NAME`) exists in the app directory. Indexes functions eagerly during `WorkerInit` / `EnvironmentReload`. +3. **V1 Classic Runtime (`azure_functions_runtime_v1`)**: Selected when no V2 script file exists; relies on Host `function.json` scanning followed by per-function `FunctionLoadRequest`. + +--- + +## Architecture & Component Reference + + + +### Full Static Topology (Mermaid) + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"ServerDuplexChannel"| n313_proto + n313_proto -->|"EventStream (grpcio)"| n313_grpc + n313_grpc -->|"ContextEnabledTask / Queue"| n313_router + n313_router -->|"WorkerRequest (V2)"| n313_rtv2 + n313_router -.->|"WorkerRequest (V1)"| n313_rtv1 + n313_router -.->|"Agent Runtime EntryPoint"| n313_ext + n313_rtv2 -->|"WebApp HTTP-v2 / SdkType"| n313_ext + n313_rtv2 -->|"Datum decode + invoke"| n313_lib + n313_rtv1 -.->|"Datum decode + invoke"| n313_lib +``` + +### Component & Source Reference + +| Component | Repository | Source Anchor | Responsibility | +| --- | --- | --- | --- | +| **Script Host (WorkerChannel)** | `functions-host` | [`src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs`](https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs) | .NET Functions Host runtime spawning `proxyV4/worker.py` (`python -m proxy_worker`) for Python 3.13/3.14. | +| **FunctionRpc.proto Contract** | `worker-protobuf` | [`src/proto/FunctionRpc.proto`](https://github.com/Azure/azure-functions-language-worker-protobuf/blob/dev/src/proto/FunctionRpc.proto) | Canonical gRPC wire schema compiled into `proxy_worker/protos` and injected into PyPI runtimes. | +| **start_worker + gRPC Pump** | `python-worker` | [`workers/proxy_worker/dispatcher.py:L278`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/proxy_worker/dispatcher.py#L278) | Locks worker deps first, enables `uvloop`, opens `grpcio` stream, and switches console -> `RpcLog`. | +| **Proxy Router & WorkerRequest** | `python-worker` | [`workers/proxy_worker/dispatcher.py:L326`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/proxy_worker/dispatcher.py#L326) | Promotes customer deps, dynamically selects V2 / V1 / Agent runtime, and passes `WorkerRequest` + `protos`. | +| **Runtime V2 (azure_functions_runtime)** | `python-worker` | [`runtimes/v2/azure_functions_runtime/handle_event.py:L69`](https://github.com/Azure/azure-functions-python-worker/blob/dev/runtimes/v2/azure_functions_runtime/handle_event.py#L69) | PyPI V2 engine: decorator indexing, `Datum` bindings, HTTP v2 streaming, OpenTelemetry, threadpool. | +| **Runtime V1 (azure_functions_runtime_v1)** | `python-worker` | [`runtimes/v1/azure_functions_runtime_v1/handle_event.py:L53`](https://github.com/Azure/azure-functions-python-worker/blob/dev/runtimes/v1/azure_functions_runtime_v1/handle_event.py#L53) | PyPI V1 engine supporting classic `function.json` apps across the exact same 5-verb contract. | +| **azure-functions SDK & User App** | `python-library` | [`azure/functions/__init__.py`](https://github.com/Azure/azure-functions-python-library/blob/dev/azure/functions/__init__.py) | Customer `function_app.py` + `azure-functions` SDK types invoked on asyncio loop or threadpool. | +| **Agent Runtimes, WebApp & SdkType** | `python-extensions` | [`azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py`](https://github.com/Azure/azure-functions-python-extensions/blob/dev/azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py) | Custom `azurefunctions.runtimes` entry points (`PYTHON_ENABLE_AGENT_RUNTIME`), FastAPI HTTP v2, and SDK bindings. | + + + +--- + +## The 7 Worker Lifecycle Phases (Python 3.13 – 3.14) + + + +### 1. Worker Initialization Flow (Runtime Selection + V2 Default Indexing) + + +--- + +- **Wire Protocol:** `StartStream → WorkerInitRequest → WorkerRequest(protos) → WorkerInitResponse` + +--- + +- **Execution Scope:** All SKUs (Cold Start) + + +--- + +`proxyV4/worker.py` starts `proxy_worker` with `uvloop` and worker-first deps (`grpcio`/`protobuf`). On `WorkerInitRequest`, `proxy_worker` promotes customer deps, runs **`reload_library_worker(dir)`** (resolving `PYTHON_ENABLE_AGENT_RUNTIME` `entry_points` → `function_app.py` V2 → V1), starts the runtime threadpool, and passes `WorkerRequest(properties={'protos': protos, 'host': host})`. **Key change in 3.13+**: Runtime V2 now **indexes `function_app.py` and starts `http_v2` during `WorkerInit`**! + + +--- + +> **Bypassed / Idle Note:** Only ONE runtime engine (Agent, V2, or V1) is imported and bound to `_library_worker`. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant proxy as proxy_worker + participant ext as python-extensions + participant rt as Runtime (v2 / v1) + participant lib as python-library / App + host->>proxy: Spawn proxyV4 -> start_worker(uvloop) + StartStream + host->>proxy: WorkerInitRequest(function_app_directory) + proxy->>ext: reload_library_worker(): check AGENT_RUNTIME entry_points + proxy->>rt: start_threadpool_executor() + worker_init_request(WorkerRequest) + rt->>lib: [V2] Cache protos, load_binding_registry, index_function_app() + rt-->>host: WorkerInitResponse -> wrapped in StreamingMessage +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. StartStream + WorkerInitRequest"| n313_grpc + n313_grpc -->|"2. call_soon_threadsafe"| n313_router + n313_router -.->|"3a. check Agent Runtime entry_points"| n313_ext + n313_router -->|"3b. V2: WorkerRequest(protos, host)"| n313_rtv2 + n313_router -.->|"3c. V1: WorkerRequest(protos, host)"| n313_rtv1 + n313_rtv2 -->|"4. V2 indexes function_app.py in Init!"| n313_lib +``` + +--- + +### 2. Worker Specialization Flow (Consumption Runtime Swap) + + +--- + +- **Wire Protocol:** `FunctionEnvironmentReloadRequest → reload_library_worker() → ReloadResponse` + +--- + +- **Execution Scope:** Linux / Flex Consumption Only (Placeholder Mode) + + +--- + +During placeholder boot, the worker defaulted to bundled `azure_functions_runtime`. On `FunctionEnvironmentReloadRequest`, `proxy_worker` promotes the newly mounted customer `.python_packages`, **re-runs `reload_library_worker(directory)`** (so a V1 app switches to `azure_functions_runtime_v1`, an Agent app loads its custom runtime, or a customer-pinned PyPI `azure-functions-runtime` takes over!), re-injects `protos`, reloads `os.environ` + `tzset()`, and indexes the specialized app. + + +--- + +> **Bypassed / Idle Note:** Only occurs when specializing warm Consumption standby containers. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant proxy as proxy_worker + participant rt as Selected Runtime + participant lib as Customer Env & App + host->>proxy: FunctionEnvironmentReloadRequest(app_dir, env_vars) + proxy->>proxy: DependencyManager.prioritize_customer_dependencies(dir) + proxy->>rt: reload_library_worker(dir) + start_threadpool_executor() + rt->>lib: function_environment_reload_request(WorkerRequest(protos)) + rt-->>host: FunctionEnvironmentReloadResponse(capabilities) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. FunctionEnvironmentReloadRequest"| n313_grpc + n313_grpc -->|"2. prioritize_customer_deps(dir)"| n313_router + n313_router -.->|"3a. re-evaluate Agent Runtime"| n313_ext + n313_router -->|"3b. reload V2 + inject protos"| n313_rtv2 + n313_router -.->|"3c. or switch to V1!"| n313_rtv1 + n313_rtv2 -->|"4. reset_host_config + env + index"| n313_lib +``` + +--- + +### 3. Worker Metadata Flow (Cached V2 Result vs V1 Host Fallback) + + +--- + +- **Wire Protocol:** `FunctionsMetadataRequest → WorkerRequest → FunctionMetadataResponse` + +--- + +- **Execution Scope:** V2 Returns Cached Metadata; V1 Delegates Indexing to Host + + +--- + +`proxy_worker` forwards `WorkerRequest(name='WorkerMetadataRequest')` (no `protos` needed — already cached in runtime!). In **Runtime V2**, `functions_metadata_request` immediately returns the `_metadata_result` indexed during Init/Specialization (`use_default_metadata_indexing=False`). In **Runtime V1**, it returns `use_default_metadata_indexing=True, results=None` so the Host reads `function.json` files. + + +--- + +> **Bypassed / Idle Note:** V2 already indexed `function_app.py` during Init/Specialization, so Phase 3 is an O(1) cached return! + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant proxy as proxy_worker + participant rtv2 as Runtime V2 + participant rtv1 as Runtime V1 + host->>proxy: FunctionsMetadataRequest(function_app_directory) + proxy->>rtv2: [If V2] await rtv2.functions_metadata_request(req) + proxy->>rtv1: [If V1] await rtv1.functions_metadata_request(req) + proxy-->>host: StreamingMessage(function_metadata_response) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. FunctionsMetadataRequest"| n313_grpc + n313_grpc -->|"2. _handle__functions_metadata_request"| n313_router + n313_router -->|"3a. V2: return cached _metadata_result"| n313_rtv2 + n313_router -.->|"3b. V1: use_default_indexing=True"| n313_rtv1 +``` + +--- + +### 4. Worker Functions Load Flow (V1 Script Loading vs V2 No-Op) + + +--- + +- **Wire Protocol:** `FunctionLoadRequest → WorkerRequest → FunctionLoadResponse` + +--- + +- **Execution Scope:** Active Import for V1 (`runtimes/v1`); Immediate No-Op Ack for V2 + + +--- + +The split makes V1 vs V2 load behavior explicit: **`runtimes/v1/handle_event.py`** implements real `load_function()` + `_functions.add_function(function_id, func, metadata, protos)` to import each `function.json` script; **`runtimes/v2/handle_event.py`** (`L160`) is a 6-line immediate `FunctionLoadResponse(Success)` because V2 functions were already loaded during indexing! + + +--- + +> **Bypassed / Idle Note:** In Runtime V2, `function_load_request` is a pure no-op acknowledgment. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant proxy as proxy_worker + participant rtv1 as Runtime V1 + participant app as V1 __init__.py + host->>proxy: FunctionLoadRequest(function_id, metadata) + proxy->>rtv1: [V1] await rtv1.function_load_request(WorkerRequest) + rtv1->>app: loader.load_function() imports script_file & entry_point + rtv1-->>host: FunctionLoadResponse(function_id, Success) +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. FunctionLoadRequest"| n313_grpc + n313_grpc -->|"2. WorkerRequest"| n313_router + n313_router -->|"3a. V1: load_function + add_function"| n313_rtv1 + n313_rtv1 -->|"4. V1 imports __init__.py & SDK hints"| n313_lib + n313_router -.->|"3b. V2: immediate Success ack"| n313_rtv2 +``` + +--- + +### 5. Worker Invocation Flow + + +--- + +- **Wire Protocol:** `InvocationRequest → WorkerRequest → Runtime v2/v1 → InvocationResponse` + +--- + +- **Execution Scope:** Every Function Trigger Execution + + +--- + +`proxy_worker` stamps the `ContextEnabledTask` with `invocation_id` and awaits `_library_worker.invocation_request(WorkerRequest)`. The runtime decodes inputs using its cached `protos`, coordinates `http_v2` streaming or `SdkType` bindings (`python-extensions`), executes async on `uvloop` or sync on the runtime's `ThreadPoolExecutor`, and returns `InvocationResponse`. + + +--- + +> **Bypassed / Idle Note:** Runtime discovery and `protos` injection do NOT run per-invocation — cached from Phase 1/2. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant proxy as proxy_worker + participant rt as Runtime Engine + participant app as SDK / Extensions / App + host->>proxy: InvocationRequest(invocation_id, function_id) + proxy->>rt: await _library_worker.invocation_request(WorkerRequest) + rt->>app: from_incoming_proto -> http_v2/SdkType -> run handler + app-->>host: to_outgoing_proto -> InvocationResponse -> _grpc_resp_queue +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. InvocationRequest"| n313_grpc + n313_grpc -->|"2. ContextEnabledTask"| n313_router + n313_router -->|"3. WorkerRequest (no protos copy)"| n313_rtv2 + n313_rtv2 -.->|"4a. http_v2 / SdkType / Settlement"| n313_ext + n313_rtv2 -->|"4b. execute_async / run_sync_func"| n313_lib +``` + +--- + +### 6. Worker Status Flow (Proxy Short-Circuit) + + +--- + +- **Wire Protocol:** `WorkerStatusRequest → proxy_worker → WorkerStatusResponse` + +--- + +- **Execution Scope:** Host Scale Controller Health Check (Runtime Bypassed!) + + +--- + +In `3.13 – 3.14`, `_handle__worker_status_request` is the **only** message handler in `proxy_worker/dispatcher.py` that **does NOT delegate to `_library_worker`**! It short-circuits inside `proxy_worker` and returns `WorkerStatusResponse` immediately—though it still runs on the shared CPython interpreter's `uvloop` and GIL. + + +--- + +> **Bypassed / Idle Note:** Runtime V2, Runtime V1, SDK, Extensions, and Customer App are 100% bypassed! + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Host Scale Controller + participant grpc as Proxy gRPC Thread + participant proxy as Proxy Dispatcher + host->>grpc: WorkerStatusRequest() + grpc->>proxy: _handle__worker_status_request (short-circuit) + proxy-->>host: WorkerStatusResponse() +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_host -->|"WorkerChannel"| n313_proto + n313_proto -->|"1. WorkerStatusRequest"| n313_grpc + n313_grpc -->|"2. proxy short-circuit (still GIL)"| n313_router + n313_router -->|"3. WorkerStatusResponse"| n313_grpc + n313_grpc -->|"4. EventStream reply"| n313_proto +``` + +--- + +### 7. Worker Logs Flow (4-Tier Invocation-ID Correlation → Host) + + +--- + +- **Wire Protocol:** `LogRecord → 4-Tier ID Resolver → StreamingMessage(RpcLog) → Host` + +--- + +- **Execution Scope:** Reverse Telemetry Stream (Worker / Runtime / Customer → Host) + + +--- + +Because execution lives in a separate PyPI runtime package (or custom Agent Runtime), `proxy_worker`'s `AsyncLoggingHandler` resolves `invocation_id` across **4 fallback tiers**: 1) global tracker (for custom runtimes without `cv`), 2) `ContextEnabledTask` attribute, 3) `_library_worker.invocation_id_cv` (`ContextVar`), and 4) `_thread_invocation_registry`—then streams `RpcLog` frames back to the Host. + + +--- + +> **Bypassed / Idle Note:** Flows RIGHT → LEFT from User App / Runtime loggers through proxy_worker's AsyncLoggingHandler to the Host. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant app as User Code / Thread + participant rt as Runtime (invocation_id_cv) + participant proxy as Proxy AsyncLoggingHandler + participant host as Host (AppInsights / Kusto) + app->>proxy: logging.info(...) from async task, threadpool, or user thread + proxy->>rt: get_current_invocation_id() 4-tier lookup + proxy-->>host: Tag System vs User -> protos.RpcLog -> EventStream +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c313_host["Functions Host (.NET — azure-functions-host)"] + n313_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c313_wire["Wire Contract (worker-protobuf)"] + n313_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c313_worker["Python Worker Process (proxyV4/worker.py)"] + subgraph c313_proxy["Layer 1: proxy_worker (Transport Shell)"] + n313_grpc["start_worker + gRPC Pump
(python-worker)"] + n313_router["Proxy Router & WorkerRequest
(python-worker)"] + end + subgraph c313_rt["Layer 2: Pluggable Runtime Engine (runtimes/)"] + n313_rtv2["Runtime V2 (azure_functions_runtime)
(python-worker)"] + n313_rtv1["Runtime V1 (azure_functions_runtime_v1)
(python-worker)"] + end + end + subgraph c313_lib["Azure/azure-functions-python-library"] + n313_lib["azure-functions SDK & User App
(python-library)"] + end + subgraph c313_ext["Azure/azure-functions-python-extensions"] + n313_ext["Agent Runtimes, WebApp & SdkType
(python-extensions)"] + end + n313_proto -->|"AppInsights / Kusto"| n313_host + n313_lib -->|"1. user logging.info() (cv / TLS set)"| n313_rtv2 + n313_rtv2 -->|"2. root AsyncLoggingHandler + 4-tier ID"| n313_router + n313_router -->|"3. protos.RpcLog -> _grpc_resp_queue"| n313_grpc + n313_grpc -->|"4. StreamingMessage(rpc_log) → Host"| n313_proto +``` + + diff --git a/docs/architecture/python-315-r2p2.md b/docs/architecture/python-315-r2p2.md new file mode 100644 index 000000000..c01bfd0f8 --- /dev/null +++ b/docs/architecture/python-315-r2p2.md @@ -0,0 +1,725 @@ +# Python 3.15+ Architecture — R2P2 (Rust to Python Proxy) + +For **Python 3.15+**, **R2P2** ([`workers/r2p2`](https://github.com/Azure/azure-functions-python-worker/tree/dev/workers/r2p2)) replaces the Python `proxy_worker` with a compiled native **Rust** binary that embeds CPython (`libpython3.15`, **GIL** build) in-process via **`PyO3`**, while keeping function discovery, bindings, and user execution in the existing Python runtime libraries (`runtimes/v2` and `runtimes/v1`). + +
+ +--- + +## Key Architectural Innovations in R2P2 + +### 1. Zero Python `grpcio` or `google.protobuf` +Rust owns every byte on the Host wire using **`tonic`** (HTTP/2 gRPC + `BytesCodec`), **`prost`** (generated from vendored `FunctionRpc.proto` in `build.rs`), and multi-threaded **`tokio`**. Nothing in the embedded Python interpreter imports `grpc` or `google.protobuf`, eliminating wheel-lag blockers for new Python versions and ending customer `protobuf` descriptor-pool collisions. + +### 2. Dual FFI Execution Paths (Hot Native Seam vs Control Adapter) +After indexing, `main.rs` memoizes `needs_control_path(function_id)` once per function: + +- **Hot Native Path (`needs_control_path == false`)**: Standard triggers and bindings (`HttpRequest`, `HttpResponse`, `QueueMessage`, `TimerRequest`, `Out[T]`) are decoded by `prost` in Rust, converted by `src/convert.rs` into compact `(type_tag, primitive_or_dict)` **PyO3 datum tuples**, and passed straight into `azure_functions_runtime.native_invocation` with zero Python protobuf marshaling. +- **Control & Extensions Path (`needs_control_path == true` or lifecycle messages)**: `WorkerInit`, `EnvironmentReload`, `FunctionsMetadata`, `FunctionLoad`, and functions using `python-extensions` (`http_v2` ASGI/FastAPI streaming or `SdkType` deferred bindings) convert `prost` structs to `PyDict`s (`src/control.rs`) and run the standard `handle_event.py` coroutines against `workers/r2p2/bridge/protos_adapter.py` (a pure-Python `protos` stand-in). + +### 3. Zero-GIL Scale Probes & Lock-Free `LogSink` +- **`WorkerStatusRequest`**: Matched directly in `src/bridge.rs` and answered in pure Rust without calling `Python::attach` or acquiring the GIL. +- **Reverse `RpcLog` Telemetry**: Python's `_RpcLogHandler` resolves `invocation_id` from `_rt.invocation_id_cv` or `_rt_tls` (`context.thread_local_storage` for customer-spawned threads) and calls `#[pyclass] LogSink.emit_log(dict)`, which `prost`-encodes and enqueues onto a single lock-free `tokio::sync::mpsc::UnboundedSender` without blocking GIL threads. + +--- + +## Architecture & Component Reference + + + +### Full Static Topology (Mermaid) + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"ServerDuplexChannel"| n315_proto + n315_proto -->|"EventStream (tonic + BytesCodec)"| n315_tonic + n315_tonic -->|"tokio::spawn + mpsc"| n315_router + n315_router -->|"Hot Path: bridge::invoke()"| n315_hot_ffi + n315_router -->|"Control / Extensions Path"| n315_ctrl_ffi + n315_hot_ffi -->|"invoke_native(datum tuples)"| n315_native_py + n315_ctrl_ffi -->|"handle_control(PyDict)"| n315_bridge_py + n315_bridge_py -->|"WorkerRequest + protos_adapter"| n315_rt + n315_native_py -->|"Datum decode + invoke"| n315_lib + n315_rt -.->|"WebApp stream / SdkType"| n315_ext + n315_rt -.->|"index @app decorators"| n315_lib +``` + +### Component & Source Reference + +| Component | Repository | Source Anchor | Responsibility | +| --- | --- | --- | --- | +| **Script Host (WorkerChannel)** | `functions-host` | [`src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs`](https://github.com/Azure/azure-functions-host/blob/dev/src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs) | .NET Functions Host runtime spawning the compiled `r2p2` binary with zero Host-side protocol changes. | +| **FunctionRpc.proto Contract** | `worker-protobuf` | [`src/proto/FunctionRpc.proto`](https://github.com/Azure/azure-functions-language-worker-protobuf/blob/dev/src/proto/FunctionRpc.proto) | Shared Host ↔ Worker gRPC contract compiled at Rust build time into native `prost` structs (`pb.rs`). | +| **Tokio + Tonic EventStream** | `python-worker` | [`workers/r2p2/src/main.rs:L231`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/src/main.rs#L231) | Multi-thread tokio pump dialing Host (`TCP_NODELAY`), spawning concurrent tasks, and draining `mpsc`. | +| **Prost Router & Status Fast-Path** | `python-worker` | [`workers/r2p2/src/main.rs:L191`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/src/main.rs#L191) | Answers `WorkerStatus` in pure Rust (0 GIL) and splits Hot vs Control invocations via `control_path_cache`. | +| **Hot FFI: bridge::invoke + convert.rs** | `python-worker` | [`workers/r2p2/src/bridge.rs:L246`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/src/bridge.rs#L246) | Translates prost `TypedData` directly into `(tag, value)` Python tuples across `Python::attach`. | +| **Control FFI: bridge::handle + LogSink** | `python-worker` | [`workers/r2p2/src/bridge.rs:L111`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/src/bridge.rs#L111) | Converts Init/Load/Reload & HTTP-v2/deferred requests to `PyDict`s; exposes non-blocking Rust `LogSink`. | +| **bridge.py & protos_adapter.py** | `python-worker` | [`workers/r2p2/bridge/bridge.py:L329`](https://github.com/Azure/azure-functions-python-worker/blob/dev/workers/r2p2/bridge/bridge.py#L329) | Pure-Python `protos` stand-in + `_clear_modules_from_path` namespace-safe customer dependency promotion. | +| **native_invocation.py (v2 & v1)** | `python-worker` | [`runtimes/v2/azure_functions_runtime/native_invocation.py:L87`](https://github.com/Azure/azure-functions-python-worker/blob/dev/runtimes/v2/azure_functions_runtime/native_invocation.py#L87) | Decodes Rust datum tuples directly into `python-library` bindings using `_InvocShim` and runs user code. | +| **Runtime handle_event + http_v2** | `python-worker` | [`runtimes/v2/azure_functions_runtime/handle_event.py:L69`](https://github.com/Azure/azure-functions-python-worker/blob/dev/runtimes/v2/azure_functions_runtime/handle_event.py#L69) | Handles `WorkerInit`, `FunctionLoad`, `EnvironmentReload`, and fallback invocations for Extensions. | +| **azure-functions (Datum + Decorators)** | `python-library` | [`azure/functions/meta.py`](https://github.com/Azure/azure-functions-python-library/blob/dev/azure/functions/meta.py) | Standard V2/V1 triggers & bindings (`HttpRequest`, `QueueMessage`, `TimerRequest`, `Out[T]`) + User App. | +| **extensions (HTTP-v2 FastAPI & SdkType)** | `python-extensions` | [`azurefunctions-extensions-base/azurefunctions/extensions/base/web.py`](https://github.com/Azure/azure-functions-python-extensions/blob/dev/azurefunctions-extensions-base/azurefunctions/extensions/base/web.py) | `WebApp` ASGI streaming (`http-fastapi`) and `SdkType` deferred bindings (`BlobClient`, `ServiceBus`). | + + + +--- + +## The 7 Worker Lifecycle Phases (Python 3.15+ R2P2) + + + +### 1. Worker Initialization Flow (Embed CPython + Protobuf-Free Init) + + +--- + +- **Wire Protocol:** `bridge::configure → StartStream (prost) → WorkerInitRequest → protos_adapter` + +--- + +- **Execution Scope:** All SKUs (Cold Start) + + +--- + +Host spawns compiled Rust binary `r2p2`. Before dialing gRPC, `main.rs` creates a single lock-free `tokio::mpsc::unbounded_channel` and calls `bridge::configure()` via PyO3 (embedding CPython, starting `bridge-asyncio` loop thread, and attaching `_RpcLogHandler` to Rust's `LogSink`). Rust opens `tonic` `EventStream`, sends `prost`-encoded `StartStream`, and routes `WorkerInitRequest` as a `PyDict` through `bridge.py` + pure-Python `protos_adapter.py` into Runtime V2/V1. + + +--- + +> **Bypassed / Idle Note:** Hot FFI (`convert.rs` / `native_invocation.py`) is not used during control-plane initialization. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant rust as R2P2 (tonic+prost) + participant ctrl as bridge.rs (PyO3) + participant bpy as bridge.py + adapter + participant rt as Runtime v2 / v1 + rust->>bpy: Create mpsc + bridge::configure(LogSink) + rust-->>host: Dial tonic EventStream + send prost StartStream(worker_id) + host->>ctrl: WorkerInitRequest -> prost decode -> worker_init_req_to_py + ctrl->>bpy: bridge.handle_control('worker_init_request', dict) + bpy->>rt: worker_init_request(_WorkerRequest(protos=protos_adapter)) + rt-->>host: protos_adapter.to_dict() -> py_to_worker_init_response -> prost +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_tonic -->|"1. bridge::configure(LogSink)"| n315_ctrl_ffi + n315_tonic -->|"2. tonic connect + StartStream (prost)"| n315_proto + n315_proto -->|"3. WorkerInitRequest bytes"| n315_router + n315_router -->|"4. worker_init_req_to_py(PyDict)"| n315_ctrl_ffi + n315_ctrl_ffi -->|"5. handle_control + select v2/v1"| n315_bridge_py + n315_bridge_py -->|"6. WorkerRequest(protos=protos_adapter)"| n315_rt + n315_rt -->|"7. V2 indexes @app decorators"| n315_lib +``` + +--- + +### 2. Worker Specialization Flow (Namespace-Safe Eviction + Reload) + + +--- + +- **Wire Protocol:** `FunctionEnvironmentReloadRequest → _clear_modules_from_path → ReloadResponse` + +--- + +- **Execution Scope:** Linux / Flex Consumption Only (Placeholder Mode) + + +--- + +When specializing a warm R2P2 standby process, `bridge.py` runs **`_clear_modules_from_path(_workers_dir)`**: file-backed worker modules & cached descendants are evicted and stale child attributes are detached from retained parents, while **PEP 420 namespace parents (`azure`, `azurefunctions`) without `__file__` are preserved** so customer SDK/extension children stay attached! Plus zero `grpcio`/`protobuf` C-extensions ever existed in `sys.modules` to collide with customer pins. + + +--- + +> **Bypassed / Idle Note:** Hot invocation seam is cleared (`_control_path_cache.clear()`) and re-bound to the newly selected runtime. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant ctrl as bridge.rs (prost/PyO3) + participant bpy as bridge.py Dep Engine + participant rt as Runtime + Customer Pkgs + host->>ctrl: FunctionEnvironmentReloadRequest(app_dir, env_vars) + ctrl->>bpy: _clear_modules_from_path(_workers_dir) + invalidate_caches() + bpy->>bpy: _prioritize_customer_dependencies(dir) + _select_runtime(dir) + bpy->>rt: function_environment_reload_request(protos=protos_adapter) + rt-->>host: py_to_env_reload_response -> prost StreamingMessage +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_proto -->|"1. FunctionEnvironmentReloadRequest"| n315_router + n315_router -->|"2. env_reload_req_to_py(PyDict)"| n315_ctrl_ffi + n315_ctrl_ffi -->|"3. _clear_modules_from_path (namespace-safe)"| n315_bridge_py + n315_bridge_py -.->|"4a. promote customer .python_packages"| n315_lib + n315_bridge_py -.->|"4b. keep azurefunctions.* namespaces"| n315_ext + n315_bridge_py -->|"5. reload runtime + env_vars + index"| n315_rt +``` + +--- + +### 3. Worker Metadata & Indexing Flow + + +--- + +- **Wire Protocol:** `FunctionsMetadataRequest → control.rs → protos_adapter → prost` + +--- + +- **Execution Scope:** V2 Returns Indexed Decorator Metadata; V1 Signals Host Indexing + + +--- + +Rust `bridge::handle` decodes `FunctionsMetadataRequest` and calls `bridge.handle_control('functions_metadata_request', {})`. Runtime V2 returns its indexed `RawFunctionMetadata` list wrapped in `protos_adapter` objects (`.to_dict()`), which `control::py_to_function_metadata_response` serializes into native `prost` wire bytes. Runtime V1 returns `use_default_metadata_indexing=True`. + + +--- + +> **Bypassed / Idle Note:** Hot native tuple FFI (`bridge::invoke`) is unused during metadata discovery. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant ctrl as bridge.rs + control.rs + participant bpy as bridge.py + adapter + participant rt as Runtime v2 / v1 + host->>ctrl: FunctionsMetadataRequest bytes + ctrl->>bpy: handle_control('functions_metadata_request', {}) + bpy->>rt: _run_coro(_rt.functions_metadata_request(req)) + rt-->>host: to_dict() -> py_to_function_metadata_response -> prost +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_proto -->|"1. FunctionsMetadataRequest"| n315_router + n315_router -->|"2. bridge::handle()"| n315_ctrl_ffi + n315_ctrl_ffi -->|"3. handle_control(dict)"| n315_bridge_py + n315_bridge_py -->|"4. functions_metadata_request()"| n315_rt +``` + +--- + +### 4. Worker Functions Load Flow (V1 `_metadata_ns` Import vs V2 Ack) + + +--- + +- **Wire Protocol:** `FunctionLoadRequest → _metadata_ns(SimpleNamespace) → FunctionLoadResponse` + +--- + +- **Execution Scope:** V1 `function.json` Script Loading (`azure_functions_runtime_v1`); V2 Immediate Ack + + +--- + +For **V1** apps, `control::function_load_req_to_py` decodes `RpcFunctionMetadata` & `BindingInfo` into a `PyDict`, and `bridge._metadata_ns()` wraps them in `SimpleNamespace` objects matching protobuf attribute access so `azure_functions_runtime_v1` imports `__init__.py` without `google.protobuf`! For **V2** apps, `function_load_request` immediately returns `Success`. + + +--- + +> **Bypassed / Idle Note:** Hot FFI stays idle; V2 apps return immediate Success while V1 apps import their `__init__.py` entrypoints here. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant ctrl as bridge.rs + control.rs + participant bpy as bridge.py (_metadata_ns) + participant rt as Runtime V1 (or V2) + participant app as V1 __init__.py + host->>ctrl: FunctionLoadRequest(function_id, metadata) + ctrl->>bpy: handle_control('function_load_request', req_dict) + bpy->>rt: await _rt.function_load_request(_WorkerRequest) + rt->>app: [V1 Only] Import script_file & register FunctionInfo + rt-->>host: py_to_function_load_response -> prost bytes +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_proto -->|"1. FunctionLoadRequest"| n315_router + n315_router -->|"2. function_load_req_to_py(dict)"| n315_ctrl_ffi + n315_ctrl_ffi -->|"3. _metadata_ns + _binding_ns"| n315_bridge_py + n315_bridge_py -->|"4. V1 load_function() / V2 ack"| n315_rt + n315_rt -.->|"5. V1 imports __init__.py"| n315_lib +``` + +--- + +### 5. Worker Invocation Flow (Dual Path: Native Datum Tuples vs Extensions) + + +--- + +- **Wire Protocol:** `InvocationRequest → [Hot: bridge::invoke | Ext: bridge::handle] → InvocationResponse` + +--- + +- **Execution Scope:** Every Function Trigger Execution + + +--- + +`main.rs` spawns a concurrent task per request and checks memoized `needs_control_path(function_id)`: **(A) Hot Path** (standard V2/V1 `python-library` bindings) converts `prost` `TypedData` directly into PyO3 `(tag, val)` datum tuples (`convert.rs`) and calls `native_invocation.py`—zero Python protobuf! **(B) Control Path** (`python-extensions` `http_v2` FastAPI streaming or `SdkType` deferred bindings) routes via `control.rs` `PyDict` + `protos_adapter.py` to `handle_event.py`. + + +--- + +> **Bypassed / Idle Note:** Standard functions completely bypass `control.rs`, `protos_adapter.py`, and `handle_event.py`! + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Functions Host + participant rust as Tokio + Router + participant hot as Hot FFI (convert.rs) + participant native as native_invocation.py + participant app as SDK / Extensions / App + host->>rust: StreamingMessage(InvocationRequest) -> tokio::spawn + rust->>hot: [Hot Path] spawn_blocking -> bridge::invoke(&req) + hot->>native: Python::attach -> invoke_native(fn_id, inputs, meta) + native->>app: Decode python-library Datum + run sync/async handler + app-->>host: tuple_to_typed_data -> prost encode -> mpsc +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_proto -->|"1. InvocationRequest (BytesCodec)"| n315_tonic + n315_tonic -->|"2. tokio::spawn + control_path_cache"| n315_router + n315_router -->|"3a. Hot: bridge::invoke()"| n315_hot_ffi + n315_hot_ffi -->|"4a. invoke_native(datum tuples)"| n315_native_py + n315_native_py -->|"5a. Datum decode + sync/async fn"| n315_lib + n315_router -.->|"3b. Ext: HTTP-v2 / SdkType"| n315_ctrl_ffi + n315_ctrl_ffi -.->|"4b. handle_invocation_control"| n315_bridge_py + n315_bridge_py -.->|"5b. handle_event + http_v2"| n315_rt + n315_rt -.->|"6b. stream WebApp / SdkType"| n315_ext +``` + +--- + +### 6. Worker Status Flow (Zero-GIL Native Rust Fast-Path) + + +--- + +- **Wire Protocol:** `WorkerStatusRequest → Rust prost → WorkerStatusResponse (0 Python / 0 GIL)` + +--- + +- **Execution Scope:** Host Scale Controller Health Check (Entire CPython Interpreter Bypassed!) + + +--- + +Major architectural win in R2P2: when `bridge::handle` (`src/bridge.rs:L114`) matches `Content::WorkerStatusRequest`, it builds `WorkerStatusResponse` in **pure Rust** and pushes the bytes onto `mpsc` **without calling `Python::attach` and without acquiring the GIL**! Even if customer Python threads are 100% CPU-saturated holding the GIL, scale probes answer in microseconds. + + +--- + +> **Bypassed / Idle Note:** PyO3 GIL boundary, Bridge, Runtime Engine, SDK, Extensions, and User Code are 100% untouched! + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant host as Host Scale Controller + participant tonic as Tokio + Tonic + participant router as Rust Router (0 GIL) + host->>tonic: StreamingMessage(WorkerStatusRequest) + tonic->>router: Match Some(Content::WorkerStatusRequest(_)) in Rust + router-->>host: prost encode WorkerStatusResponse -> mpsc tx +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_host -->|"WorkerChannel"| n315_proto + n315_proto -->|"1. WorkerStatusRequest bytes"| n315_tonic + n315_tonic -->|"2. prost match WorkerStatusRequest"| n315_router + n315_router -->|"3. WorkerStatusResponse (ZERO GIL!)"| n315_tonic + n315_tonic -->|"4. EventStream reply"| n315_proto +``` + +--- + +### 7. Worker Logs Flow (Non-Blocking PyO3 LogSink → Lock-Free Rust mpsc) + + +--- + +- **Wire Protocol:** `logging.LogRecord → _RpcLogHandler → #[pyclass] LogSink → prost → Host` + +--- + +- **Execution Scope:** Reverse Telemetry Stream (Customer & System Logs → Host) + + +--- + +Startup logs emit `LanguageWorkerConsoleLog` via `src/log.rs`. Once `bridge.configure()` runs, `_RpcLogHandler` captures all Python root logs, resolves `invocation_id` from **1) `_rt.invocation_id_cv`** (async & threadpool handlers) or **2) `_rt_tls` (`context.thread_local_storage.invocation_id`)** (user-spawned `threading.Thread`s!), and calls `LogSink.emit_log(dict)`. Rust `prost`-encodes `RpcLog` and pushes onto the lock-free `UnboundedSender` without awaiting or deadlocking the GIL. + + +--- + +> **Bypassed / Idle Note:** Flows RIGHT → LEFT from User Code / Runtime into PyO3 `LogSink` and straight onto the outbound tonic stream. + + +--- + +#### Sequence Diagram + + +--- + +```mermaid +sequenceDiagram + participant app as User Code / User Thread + participant bpy as bridge.py (_RpcLogHandler) + participant sink as Rust #[pyclass] LogSink + participant tonic as mpsc + Tonic + participant host as Host (AppInsights / Kusto) + app->>bpy: logging.info(...) from handler or custom threading.Thread + bpy->>bpy: Resolve _current_invocation_id() + System vs User category + bpy->>sink: sink.emit_log({level, message, category, invocation_id}) + sink->>tonic: py_log_to_streaming_message -> tx.send(Bytes) + tonic-->>host: Drain UnboundedReceiverStream onto EventStream +``` + + +--- + +#### Phase Compound Topology + + +--- + +```mermaid +flowchart TB + subgraph c315_host["Functions Host (.NET — azure-functions-host)"] + n315_host["Script Host (WorkerChannel)
(functions-host)"] + end + subgraph c315_wire["Wire Contract (worker-protobuf)"] + n315_proto["FunctionRpc.proto Contract
(worker-protobuf)"] + end + subgraph c315_proc["R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)"] + subgraph c315_rust["Layer 1: Native Rust Shell (workers/r2p2/src)"] + n315_tonic["Tokio + Tonic EventStream
(python-worker)"] + n315_router["Prost Router & Status Fast-Path
(python-worker)"] + end + subgraph c315_ffi["PyO3 FFI Thin Waist (GIL)"] + n315_hot_ffi["Hot FFI: bridge::invoke + convert.rs
(python-worker)"] + n315_ctrl_ffi["Control FFI: bridge::handle + LogSink
(python-worker)"] + end + subgraph c315_py["Layer 2: Embedded Python Bridge & Runtime"] + n315_bridge_py["bridge.py & protos_adapter.py
(python-worker)"] + n315_native_py["native_invocation.py (v2 & v1)
(python-worker)"] + n315_rt["Runtime handle_event + http_v2
(python-worker)"] + end + end + subgraph c315_lib["Azure/azure-functions-python-library"] + n315_lib["azure-functions (Datum + Decorators)
(python-library)"] + end + subgraph c315_ext["Azure/azure-functions-python-extensions"] + n315_ext["extensions (HTTP-v2 FastAPI & SdkType)
(python-extensions)"] + end + n315_proto -->|"AppInsights / Kusto"| n315_host + n315_lib -->|"1. user logging.info() (cv / _rt_tls)"| n315_native_py + n315_native_py -->|"2. _RpcLogHandler + _current_invocation_id()"| n315_bridge_py + n315_bridge_py -->|"3. LogSink.emit_log(flat dict)"| n315_ctrl_ffi + n315_ctrl_ffi -->|"4. prost encode -> UnboundedSender"| n315_tonic + n315_tonic -->|"5. StreamingMessage(RpcLog) → Host"| n315_proto +``` + + diff --git a/docs/assets/atlas-embed.js b/docs/assets/atlas-embed.js new file mode 100644 index 000000000..c7a8b53d1 --- /dev/null +++ b/docs/assets/atlas-embed.js @@ -0,0 +1,768 @@ +/** + * Embeddable Interactive Architecture Atlas for MkDocs Material pages. + * + * Mounts into any `
` + * while preserving static Markdown & Mermaid documentation below it on the page. + */ +(function () { + const ZOOM_MIN = 0.12; + const ZOOM_MAX = 1.8; + const ZOOM_STEP = 0.12; + + const EDGE_PALETTE = { + "grpc-wire": { color: "#38bdf8", label: "gRPC Wire (FunctionRpc)" }, + "thread-queue": { color: "#a78bfa", label: "Thread / Async Channel" }, + "inproc-contract": { color: "#34d399", label: "In-Proc Python Contract" }, + "pyo3-hot": { color: "#fb923c", label: "PyO3 Hot Tuple FFI" }, + "pyo3-control": { color: "#f472b6", label: "PyO3 Control / Extensions" }, + "syspath-mutation": { color: "#fbbf24", label: "Import / EntryPoint" }, + "exec-dispatch": { color: "#60a5fa", label: "Handler Execution" }, + }; + + const CONTAINER_TONES = { + proto: { stroke: "#fbbf24", fill: "rgba(120, 53, 15, 0.14)", head: "rgba(146, 64, 14, 0.32)", text: "#fde68a" }, + host: { stroke: "#f87171", fill: "rgba(127, 29, 29, 0.14)", head: "rgba(153, 27, 27, 0.32)", text: "#fecaca" }, + python: { stroke: "#64748b", fill: "rgba(15, 23, 42, 0.50)", head: "rgba(30, 41, 59, 0.80)", text: "#e2e8f0" }, + "rust-proc": { stroke: "#fb923c", fill: "rgba(67, 20, 7, 0.22)", head: "rgba(124, 45, 18, 0.42)", text: "#fed7aa" }, + rust: { stroke: "#fb923c", fill: "rgba(124, 45, 18, 0.16)", head: "rgba(154, 52, 18, 0.36)", text: "#ffedd5" }, + transport: { stroke: "#a78bfa", fill: "rgba(76, 29, 149, 0.15)", head: "rgba(91, 33, 182, 0.32)", text: "#ddd6fe" }, + bridge: { stroke: "#f472b6", fill: "rgba(131, 24, 67, 0.16)", head: "rgba(157, 23, 77, 0.34)", text: "#fbcfe8" }, + runtime: { stroke: "#34d399", fill: "rgba(6, 78, 59, 0.16)", head: "rgba(6, 95, 70, 0.34)", text: "#a7f3d0" }, + lib: { stroke: "#34d399", fill: "rgba(6, 78, 59, 0.12)", head: "rgba(4, 120, 87, 0.28)", text: "#6ee7b7" }, + ext: { stroke: "#c084fc", fill: "rgba(88, 28, 135, 0.16)", head: "rgba(107, 33, 168, 0.34)", text: "#e9d5ff" }, + }; + + function esc(str) { + return String(str ?? "") + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); + } + + function mdInline(str) { + return esc(str) + .replace(/\*\*([^*]+)\*\*/g, "$1") + .replace(/`([^`]+)`/g, "$1"); + } + + let activeDisposers = []; + + function mountWidget(hostEl, graph, mountIdx) { + const eraId = hostEl.getAttribute("data-era") || "py315_r2p2"; + const era = graph.eras.find((e) => e.id === eraId); + if (!era) return () => {}; + + const hasPhases = Boolean(era.phases && era.phases.length); + const defaultPhase = hostEl.getAttribute("data-phase") || (hasPhases ? "init" : "all"); + + const initialOrientation = (hostEl.clientWidth || window.innerWidth) < 520 ? "TB" : "LR"; + + const st = { + mode: "topology", // topology | sequence | compare_phase + orientation: initialOrientation, + phaseId: defaultPhase, + stepIdx: -1, + selectedNodeId: null, + selectedEdgeId: null, + zoom: 0.75, + panX: 16, + panY: 16, + fullscreen: false, + needsFit: true, + }; + + // Optional URL hash sync (`#phase=worker_logs&mode=sequence`) without breaking heading anchors + const rawHash = (window.location.hash || "").replace(/^#/, ""); + if (rawHash.startsWith("phase=")) { + const params = new URLSearchParams(rawHash); + if (params.get("phase")) st.phaseId = params.get("phase"); + if (params.get("mode")) st.mode = params.get("mode"); + } + + let stageAbort = null; + const globalAbort = new AbortController(); + + function activePhase() { + if (!hasPhases || st.phaseId === "all") return null; + return era.phases.find((p) => p.id === st.phaseId) || era.phases[0] || null; + } + + const initPh = activePhase(); + st.selectedNodeId = initPh?.activeNodes?.[0] || era.nodes[0]?.id || null; + + function fitToStage() { + const ph = activePhase(); + const layout = window.ArchLayoutEngine.layoutCompoundGraph( + era, + st.orientation, + ph ? ph.topologyEdges : null + ); + const stageEl = hostEl.querySelector(".aa-stage"); + const availW = Math.max(360, (stageEl ? stageEl.clientWidth : 780) - 28); + const availH = Math.max(280, (stageEl ? stageEl.clientHeight : 460) - 58); + const scale = Math.min(1.0, availW / layout.width, availH / layout.height); + st.zoom = Math.max(ZOOM_MIN, Math.min(ZOOM_MAX, Number(scale.toFixed(3)))); + st.panX = Math.round(((stageEl ? stageEl.clientWidth : availW) - layout.width * st.zoom) / 2); + st.panY = Math.max(10, Math.round(((stageEl ? stageEl.clientHeight : availH) - layout.height * st.zoom) / 2 - 6)); + const vp = hostEl.querySelector(".aa-viewport-g"); + if (vp) vp.setAttribute("transform", `translate(${st.panX}, ${st.panY}) scale(${st.zoom})`); + } + + function captureFocus() { + const a = document.activeElement; + if (!a || !hostEl.contains(a)) return null; + for (const attr of ["data-node", "data-edge", "data-seq-step", "data-phase", "data-mode", "data-act"]) { + if (a.hasAttribute && a.hasAttribute(attr)) { + return { attr, val: a.getAttribute(attr) }; + } + } + if (a.hasAttribute && a.hasAttribute("data-boundary-node")) { + return { attr: "data-node", val: a.getAttribute("data-boundary-node") }; + } + return null; + } + + function restoreFocus(desc) { + if (!desc) return; + const target = hostEl.querySelector(`[${desc.attr}="${CSS.escape(desc.val)}"]`); + if (target && typeof target.focus === "function") { + target.focus({ preventScroll: true }); + } + } + + function renderTopology(ph, activeStep) { + const layout = window.ArchLayoutEngine.layoutCompoundGraph( + era, + st.orientation, + ph ? ph.topologyEdges : null + ); + const phaseNodes = ph ? new Set(ph.activeNodes) : null; + const stepNodes = activeStep ? new Set(activeStep.activeNodes) : null; + const stepEdges = activeStep ? new Set(activeStep.activeEdges) : null; + const pfx = `m${mountIdx}`; + + const markers = Object.entries(EDGE_PALETTE) + .map( + ([k, v]) => ` + + + + ` + ) + .join(""); + + const containersSvg = layout.containers + .map((c) => { + const t = CONTAINER_TONES[c.tone] || CONTAINER_TONES.python; + const r = c.isNested ? 9 : 13; + const rawTitle = c.label.toUpperCase(); + const maxChars = Math.max(16, Math.floor((c.w - 22) / 6.3)); + const displayTitle = rawTitle.length > maxChars ? rawTitle.slice(0, maxChars - 1) + "…" : rawTitle; + return ` + + ${esc(c.label)} + + + ${esc(displayTitle)} + + `; + }) + .join(""); + + const edgesSvg = layout.edges + .map((e) => { + const pal = EDGE_PALETTE[e.kind] || EDGE_PALETTE["inproc-contract"]; + const isStepAct = stepEdges ? stepEdges.has(e.id) : false; + const isPhaseFlow = Boolean(ph) && !stepEdges; + const isSel = st.selectedEdgeId === e.id; + const isDim = stepEdges && !isStepAct; + const strokeW = isStepAct || isSel ? 3.3 : isPhaseFlow ? 2.3 : 1.9; + const labelW = Math.max(78, e.label.length * 6.3 + 16); + const pathId = `${pfx}-ep-${e.id}`; + + return ` + + + + ${ + isStepAct + ? ` + + + + ` + : "" + } + + + + ${esc(e.label)} + + + + `; + }) + .join(""); + + const nodesSvg = layout.nodes + .map((p) => { + const n = p.node; + const repo = graph.repos[n.repo] || graph.repos["python-worker"]; + const isBypassed = phaseNodes ? !phaseNodes.has(n.id) : false; + const isStepAct = stepNodes ? stepNodes.has(n.id) : false; + const isDim = isBypassed || (stepNodes && !isStepAct); + const isSel = st.selectedNodeId === n.id; + const borderCol = isStepAct ? "#fb923c" : isSel ? "#38bdf8" : !isBypassed && phaseNodes ? "#3b527e" : "#24324d"; + const shortFile = n.file.split("/").slice(-2).join("/"); + const lineTag = n.line > 1 ? `:${n.line}` : ""; + + return ` + + + + + ${esc(repo.shortName)} + ${esc(isBypassed ? "BYPASSED" : n.badge.slice(0, 17))} + + ${esc(n.label.length > 29 ? n.label.slice(0, 28) + "…" : n.label)} + + ${esc((shortFile + lineTag).slice(0, 29))} + ${ + n.extractedSymbols && n.extractedSymbols.length + ? `${n.extractedSymbols.length} AST` + : "" + } + + `; + }) + .join(""); + + return ` +
+ + ${markers} + + ${containersSvg} + ${nodesSvg} + ${edgesSvg} + + +
+ + + + +
+
+ ${Object.values(EDGE_PALETTE) + .slice(0, 5) + .map( + (p) => ` + + ${esc(p.label)} + ` + ) + .join("")} +
+
+ `; + } + + function renderInspector(ph) { + const edgePool = ph ? ph.topologyEdges : era.edges; + const selEdge = st.selectedEdgeId ? edgePool.find((e) => e.id === st.selectedEdgeId) : null; + if (selEdge) { + const srcNode = era.nodes.find((n) => n.id === selEdge.source); + const tgtNode = era.nodes.find((n) => n.id === selEdge.target); + const pal = EDGE_PALETTE[selEdge.kind] || EDGE_PALETTE["inproc-contract"]; + return ` + + `; + } + + const node = era.nodes.find((n) => n.id === st.selectedNodeId) || era.nodes[0]; + if (!node) return ``; + const repo = graph.repos[node.repo] || graph.repos["python-worker"]; + const inbound = edgePool.filter((e) => e.target === node.id); + const outbound = edgePool.filter((e) => e.source === node.id); + const isBypassed = ph ? !ph.activeNodes.includes(node.id) : false; + + return ` + + `; + } + + function renderSequence(flow) { + if (!flow) return `
No sequence flow defined.
`; + const colW = 182; + const padX = 110; + const startY = 88; + const rowH = 64; + const width = Math.max(720, padX * 2 + (flow.actors.length - 1) * colW); + const height = startY + flow.steps.length * rowH + 36; + const actorX = new Map(flow.actors.map((a, i) => [a.id, padX + i * colW])); + const pfx = `sq${mountIdx}`; + + return ` +
+ + + + + + + + + + + + + ${flow.actors + .map((a) => { + const x = actorX.get(a.id); + return ` + + + ${esc(a.label)} + `; + }) + .join("")} + ${flow.steps + .map((s, idx) => { + const x1 = actorX.get(s.from); + const x2 = actorX.get(s.to); + const y = startY + idx * rowH; + const isAct = st.stepIdx === idx; + const stroke = isAct ? "#fb923c" : s.isReturn ? "#34d399" : "#38bdf8"; + const mk = isAct ? `url(#${pfx}-act)` : s.isReturn ? `url(#${pfx}-ret)` : `url(#${pfx}-req)`; + const mid = x1 === x2 ? x1 + 60 : (x1 + x2) / 2; + return ` + + + + ${idx + 1} + + ${esc(s.message)} + ${esc(s.payload)} + + `; + }) + .join("")} + +
+ `; + } + + function renderComparePhase() { + const targetId = st.phaseId === "all" ? "init" : st.phaseId; + const workerEras = graph.eras.filter((e) => e.phases && e.phases.length); + return ` +
+
+ ${workerEras + .map((e) => { + const p = e.phases.find((x) => x.id === targetId); + if (!p) return ""; + return ` +
+
${esc(e.shortTitle)}
+
${esc(p.proto)}
+
${mdInline(p.summary)}
+
⚡ ${mdInline(p.bypassedNote)}
+
+ `; + }) + .join("")} +
+
+ `; + } + + function render() { + if (stageAbort) { + stageAbort.abort(); + stageAbort = null; + } + const focusDesc = captureFocus(); + const ph = activePhase(); + const seqSource = ph || era.flows[0]; + const stepList = seqSource ? seqSource.steps : []; + const activeStep = + seqSource && st.stepIdx >= 0 && st.stepIdx < stepList.length + ? stepList[st.stepIdx] + : null; + + hostEl.innerHTML = ` +
+
+ +
+
+ + + ${hasPhases ? `` : ""} +
+ +
+
+ + ${ + hasPhases + ? ` +
+ Phase: + + ${(graph.phaseCatalog || []) + .map( + (p) => ` + ` + ) + .join("")} +
` + : "" + } + + ${ + seqSource && stepList.length + ? ` +
+
+
${esc(ph ? ph.title : seqSource.title)}
+ ${ph ? `
⚡ ${mdInline(ph.bypassedNote)}
` : ""} +
+
+ ${ + st.stepIdx >= 0 + ? ` + + Hop ${st.stepIdx + 1} / ${stepList.length} + + ` + : `` + } +
+
` + : "" + } + + ${ + activeStep + ? `
+ Hop ${st.stepIdx + 1}: ${esc(activeStep.message)} + [${esc(activeStep.payload)}] +
` + : "" + } + + ${ + st.mode === "topology" + ? `
${renderTopology(ph, activeStep)}${renderInspector(ph)}
` + : st.mode === "sequence" + ? renderSequence(seqSource) + : renderComparePhase() + } +
+ `; + + if (st.needsFit && st.mode === "topology") { + st.needsFit = false; + fitToStage(); + } + bindWidgetEvents(); + restoreFocus(focusDesc); + } + + function onKeyActivate(el, fn) { + el.addEventListener("keydown", (ev) => { + if (ev.key === "Enter" || ev.key === " ") { + ev.preventDefault(); + ev.stopPropagation(); + fn(ev); + } + }); + } + + function bindWidgetEvents() { + hostEl.querySelectorAll("[data-mode]").forEach((el) => { + el.addEventListener("click", () => { + st.mode = el.getAttribute("data-mode"); + if (st.mode === "topology") st.needsFit = true; + render(); + }); + }); + + hostEl.querySelectorAll("[data-phase]").forEach((el) => { + el.addEventListener("click", () => { + st.phaseId = el.getAttribute("data-phase"); + st.stepIdx = -1; + st.selectedEdgeId = null; + const ph = activePhase(); + if (ph && ph.activeNodes?.length) st.selectedNodeId = ph.activeNodes[0]; + st.needsFit = true; + render(); + }); + }); + + hostEl.querySelectorAll("[data-act]").forEach((el) => { + el.addEventListener("click", () => { + const act = el.getAttribute("data-act"); + if (act === "step-start") st.stepIdx = 0; + else if (act === "step-prev") st.stepIdx = Math.max(0, st.stepIdx - 1); + else if (act === "step-next") st.stepIdx += 1; + else if (act === "step-reset") st.stepIdx = -1; + else if (act === "zoom-in") st.zoom = Math.min(ZOOM_MAX, Math.max(ZOOM_MIN, +(st.zoom + ZOOM_STEP).toFixed(2))); + else if (act === "zoom-out") st.zoom = Math.max(ZOOM_MIN, Math.min(ZOOM_MAX, +(st.zoom - ZOOM_STEP).toFixed(2))); + else if (act === "zoom-fit") fitToStage(); + else if (act === "orient") { + st.orientation = st.orientation === "LR" ? "TB" : "LR"; + st.needsFit = true; + } else if (act === "fullscreen") { + st.fullscreen = !st.fullscreen; + document.body.style.overflow = st.fullscreen ? "hidden" : ""; + st.needsFit = true; + } + render(); + }); + }); + + hostEl.querySelectorAll("[data-node]").forEach((el) => { + const fn = (ev) => { + ev.stopPropagation(); + st.selectedNodeId = el.getAttribute("data-node"); + st.selectedEdgeId = null; + render(); + }; + el.addEventListener("click", fn); + onKeyActivate(el, fn); + }); + + hostEl.querySelectorAll("[data-boundary-node]").forEach((el) => { + el.addEventListener("click", (ev) => { + ev.stopPropagation(); + st.selectedNodeId = el.getAttribute("data-boundary-node"); + st.selectedEdgeId = null; + render(); + }); + }); + + hostEl.querySelectorAll("[data-edge]").forEach((el) => { + const fn = (ev) => { + ev.stopPropagation(); + st.selectedEdgeId = el.getAttribute("data-edge"); + render(); + }; + el.addEventListener("click", fn); + onKeyActivate(el, fn); + }); + + hostEl.querySelectorAll("[data-seq-step]").forEach((el) => { + const fn = () => { + st.stepIdx = Number(el.getAttribute("data-seq-step")); + render(); + }; + el.addEventListener("click", fn); + onKeyActivate(el, fn); + }); + + const stage = hostEl.querySelector(".aa-stage"); + const vp = hostEl.querySelector(".aa-viewport-g"); + if (stage && vp) { + stageAbort = new AbortController(); + const { signal } = stageAbort; + let activePointerId = null; + let sx = 0, sy = 0, ox = st.panX, oy = st.panY; + + const endPan = (ev) => { + if (activePointerId === null || (ev && ev.pointerId !== activePointerId)) return; + if (stage.hasPointerCapture && stage.hasPointerCapture(activePointerId)) { + try { stage.releasePointerCapture(activePointerId); } catch (_) {} + } + activePointerId = null; + }; + + stage.addEventListener("pointerdown", (ev) => { + if (ev.pointerType === "mouse" && ev.button !== 0) return; + if (ev.target.closest("button") || ev.target.closest(".svg-node") || ev.target.closest(".svg-edge")) return; + activePointerId = ev.pointerId; + sx = ev.clientX; + sy = ev.clientY; + ox = st.panX; + oy = st.panY; + if (stage.setPointerCapture) { + try { stage.setPointerCapture(ev.pointerId); } catch (_) {} + } + }); + + window.addEventListener( + "pointermove", + (ev) => { + if (activePointerId === null || ev.pointerId !== activePointerId) return; + st.panX = ox + (ev.clientX - sx); + st.panY = oy + (ev.clientY - sy); + vp.setAttribute("transform", `translate(${st.panX}, ${st.panY}) scale(${st.zoom})`); + }, + { signal } + ); + window.addEventListener("pointerup", endPan, { signal }); + window.addEventListener("pointercancel", endPan, { signal }); + stage.addEventListener("lostpointercapture", endPan); + stage.addEventListener( + "wheel", + (ev) => { + ev.preventDefault(); + const factor = ev.deltaY < 0 ? 1.07 : 0.93; + st.zoom = Math.max(ZOOM_MIN, Math.min(ZOOM_MAX, +(st.zoom * factor).toFixed(3))); + vp.setAttribute("transform", `translate(${st.panX}, ${st.panY}) scale(${st.zoom})`); + }, + { passive: false } + ); + } + } + + window.addEventListener( + "keydown", + (ev) => { + if (ev.key === "Escape" && st.fullscreen) { + st.fullscreen = false; + document.body.style.overflow = ""; + st.needsFit = true; + render(); + } + }, + { signal: globalAbort.signal } + ); + + render(); + + return () => { + if (stageAbort) stageAbort.abort(); + globalAbort.abort(); + document.body.style.overflow = ""; + }; + } + + function initAllMounts() { + activeDisposers.forEach((fn) => fn()); + activeDisposers = []; + const graph = window.__ARCH_GRAPH__; + if (!graph || !window.ArchLayoutEngine) return; + document.querySelectorAll(".arch-atlas-mount").forEach((el, idx) => { + activeDisposers.push(mountWidget(el, graph, idx)); + }); + } + + if (window.document$ && typeof window.document$.subscribe === "function") { + window.document$.subscribe(() => initAllMounts()); + } else if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", initAllMounts); + } else { + initAllMounts(); + } +})(); diff --git a/docs/assets/atlas.css b/docs/assets/atlas.css new file mode 100644 index 000000000..92a86491b --- /dev/null +++ b/docs/assets/atlas.css @@ -0,0 +1,429 @@ +/* MkDocs Material Custom Theme Accents + Scoped Embedded Architecture Atlas */ +[data-md-color-scheme="slate"] { + --md-default-bg-color: #0a101d; + --md-typeset-a-color: #38bdf8; +} + +.md-header { + background-color: #0d1628; + border-bottom: 1px solid #1e2d4a; +} + +.md-tabs { + background-color: #09101f; + border-bottom: 1px solid #1e2d4a; +} + +.md-header__button.md-logo img, +.md-header__button.md-logo svg { + width: 1.45rem; + height: 1.45rem; +} + +/* Widen content grid slightly so embedded architecture diagrams breathe */ +.md-grid { + max-width: 86rem; +} + +.arch-atlas-widget { + --aa-bg: #070b14; + --aa-elev: #0d1424; + --aa-card: #111b2e; + --aa-hover: #17243d; + --aa-border: #1e2d4a; + --aa-border-strong: #314870; + --aa-text: #f1f5f9; + --aa-muted: #94a3b8; + --aa-dim: #64748b; + --aa-accent: #38bdf8; + --aa-mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + + margin: 1.1rem 0 1.8rem; + border: 1px solid var(--aa-border-strong); + border-radius: 12px; + overflow: hidden; + background: var(--aa-bg); + color: var(--aa-text); + box-shadow: 0 10px 28px rgba(0, 0, 0, 0.35); + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, Roboto, sans-serif; +} + +.arch-atlas-widget.is-fullscreen { + position: fixed; + inset: 0; + margin: 0; + border-radius: 0; + z-index: 9999; + display: flex; + flex-direction: column; +} + +.aa-header { + display: flex; + justify-content: space-between; + align-items: center; + gap: 10px; + padding: 9px 14px; + background: var(--aa-elev); + border-bottom: 1px solid var(--aa-border); + flex-wrap: wrap; +} + +.aa-title-group { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +.aa-badge { + font-size: 10.5px; + font-family: var(--aa-mono); + padding: 2px 8px; + border-radius: 99px; + border: 1px solid rgba(56, 189, 248, 0.4); + background: rgba(56, 189, 248, 0.12); + color: #7dd3fc !important; + text-decoration: none !important; +} + +.aa-mode-tabs { + display: flex; + gap: 4px; + background: var(--aa-bg); + padding: 3px; + border-radius: 8px; + border: 1px solid var(--aa-border); + flex-wrap: wrap; +} + +.aa-tab { + background: transparent; + border: none; + color: var(--aa-muted); + font-size: 11.5px; + font-weight: 600; + padding: 5px 10px; + border-radius: 6px; + cursor: pointer; +} + +.aa-tab.active { + background: var(--aa-card); + color: var(--aa-text); + box-shadow: inset 0 0 0 1px var(--aa-border-strong); +} + +.aa-phase-bar { + display: flex; + align-items: center; + gap: 6px; + padding: 7px 14px; + background: rgba(9, 15, 29, 0.96); + border-bottom: 1px solid var(--aa-border); + overflow-x: auto; +} + +.aa-phase-pill { + background: var(--aa-card); + border: 1px solid var(--aa-border); + color: var(--aa-muted); + border-radius: 7px; + padding: 4px 9px; + font-size: 11px; + font-weight: 650; + cursor: pointer; + white-space: nowrap; + display: inline-flex; + align-items: center; + gap: 5px; +} + +.aa-phase-pill.active { + background: rgba(56, 189, 248, 0.16); + border-color: var(--aa-accent); + color: #f8fafc; +} + +.aa-phase-num { + display: inline-grid; + place-items: center; + width: 16px; + height: 16px; + border-radius: 4px; + font-size: 9.5px; + font-family: var(--aa-mono); + background: rgba(255, 255, 255, 0.08); + color: #cbd5e1; +} + +.aa-phase-pill.active .aa-phase-num { + background: var(--aa-accent); + color: #060a12; + font-weight: 800; +} + +.aa-context-strip { + padding: 8px 14px; + background: rgba(17, 27, 46, 0.78); + border-bottom: 1px solid var(--aa-border); + display: flex; + justify-content: space-between; + align-items: center; + gap: 12px; + flex-wrap: wrap; +} + +.aa-bypassed { + font-size: 10.5px; + color: #fbbf24; + font-family: var(--aa-mono); + margin-top: 3px; +} + +.aa-btn { + background: var(--aa-card); + color: var(--aa-text); + border: 1px solid var(--aa-border); + border-radius: 6px; + padding: 4px 9px; + font-size: 11px; + font-weight: 600; + cursor: pointer; +} + +.aa-btn:hover:not(:disabled) { + border-color: var(--aa-accent); + background: var(--aa-hover); +} + +.aa-btn:disabled { + opacity: 0.4; + cursor: default; +} + +.aa-btn.primary { + background: rgba(251, 146, 60, 0.16); + border-color: rgba(251, 146, 60, 0.55); + color: #fed7aa; +} + +.aa-step-banner { + padding: 5px 14px; + background: rgba(251, 146, 60, 0.11); + border-bottom: 1px solid rgba(251, 146, 60, 0.35); + font-size: 11.5px; + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} + +.aa-split { + display: flex; + flex-direction: column; +} + +.arch-atlas-widget.is-fullscreen .aa-split { + flex: 1; + min-height: 0; +} + +.aa-stage { + position: relative; + height: 540px; + min-height: 540px; + overflow: hidden; + background-color: #060a12; + background-image: radial-gradient(rgba(148, 163, 184, 0.14) 1px, transparent 1px); + background-size: 22px 22px; + cursor: grab; + touch-action: none; +} + +.arch-atlas-widget.is-fullscreen .aa-stage { + flex: 1; + height: auto; + min-height: 360px; +} + +.md-typeset .arch-atlas-widget .aa-stage > svg, +.arch-atlas-widget .aa-stage > svg { + position: absolute; + inset: 0; + display: block; + width: 100% !important; + height: 100% !important; + max-width: none !important; + max-height: none !important; +} + +.md-typeset .arch-atlas-widget .aa-scroll-pane > svg, +.arch-atlas-widget .aa-scroll-pane > svg { + max-width: none !important; + height: auto !important; +} + +.aa-stage:active { + cursor: grabbing; +} + +.aa-zoom-hud { + position: absolute; + top: 10px; + right: 10px; + display: flex; + gap: 5px; + background: rgba(13, 20, 36, 0.92); + border: 1px solid var(--aa-border); + padding: 4px; + border-radius: 7px; + z-index: 5; +} + +.aa-legend-hud { + position: absolute; + bottom: 10px; + left: 10px; + display: flex; + gap: 10px; + flex-wrap: wrap; + background: rgba(13, 20, 36, 0.92); + border: 1px solid var(--aa-border); + padding: 5px 10px; + border-radius: 7px; + font-size: 10px; + color: var(--aa-muted); + z-index: 5; +} + +.aa-inspector { + background: var(--aa-elev); + border-top: 1px solid var(--aa-border); + padding: 12px 14px; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 10px; + align-items: start; + max-height: 230px; + overflow-y: auto; +} + +.aa-card { + background: var(--aa-card); + border: 1px solid var(--aa-border); + border-radius: 8px; + padding: 10px 12px; +} + +.aa-kicker { + font-size: 9.5px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.07em; + color: var(--aa-dim); + margin-bottom: 4px; +} + +.aa-sym-list { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 6px; +} + +.aa-sym-item { + display: flex; + justify-content: space-between; + align-items: center; + gap: 6px; + width: 100%; + text-align: left; + cursor: pointer; + padding: 4px 7px; + border-radius: 5px; + background: rgba(7, 11, 20, 0.65); + border: 1px solid var(--aa-border); + font-family: var(--aa-mono); + font-size: 10px; + color: #e2e8f0 !important; + text-decoration: none !important; +} + +.aa-sym-item:hover { + border-color: var(--aa-accent); +} + +.aa-gh-link { + display: flex; + justify-content: space-between; + align-items: center; + gap: 8px; + margin-top: 8px; + padding: 6px 9px; + background: rgba(56, 189, 248, 0.1); + border: 1px solid rgba(56, 189, 248, 0.38); + border-radius: 6px; + color: #7dd3fc !important; + text-decoration: none !important; + font-family: var(--aa-mono); + font-size: 10.5px; + word-break: break-all; +} + +.aa-gh-link > span:last-child { + white-space: nowrap; + flex-shrink: 0; +} + +.aa-scroll-pane { + padding: 16px; + max-height: 520px; + overflow: auto; + background: #060a12; +} + +.arch-atlas-widget.is-fullscreen .aa-scroll-pane { + flex: 1; + max-height: none; +} + +.aa-compare-grid { + display: grid; + grid-template-columns: repeat(3, minmax(220px, 1fr)); + gap: 12px; +} + +@media (max-width: 960px) { + .aa-stage { + height: 420px; + min-height: 420px; + } + .aa-inspector { + grid-template-columns: 1fr; + max-height: none; + } + .aa-compare-grid { + grid-template-columns: 1fr; + } +} + +.svg-node:focus-visible, +.svg-edge:focus-visible, +[data-seq-step]:focus-visible, +.arch-atlas-widget button:focus-visible, +.arch-atlas-widget a:focus-visible { + outline: 2px solid var(--aa-accent); + outline-offset: 2px; +} + +@keyframes aaDashFlow { + to { + stroke-dashoffset: -28; + } +} + +.aa-edge-animated { + stroke-dasharray: 8 6; + animation: aaDashFlow 0.85s linear infinite; +} diff --git a/docs/assets/function-apps.svg b/docs/assets/function-apps.svg new file mode 100644 index 000000000..00ec83860 --- /dev/null +++ b/docs/assets/function-apps.svg @@ -0,0 +1 @@ +Icon-compute-29 diff --git a/docs/assets/layoutEngine.js b/docs/assets/layoutEngine.js new file mode 100644 index 000000000..ca78b2f71 --- /dev/null +++ b/docs/assets/layoutEngine.js @@ -0,0 +1,433 @@ +/** + * Deterministic 2D Tiered Compound Architecture Layout Engine. + * + * Balances horizontal and vertical aspect ratio by: + * 1. Grouping peer root containers into topological tiers (stacking independent + * repos like `python-library` and `python-extensions` in the same column). + * 2. Using orthogonal intra-container stacking (vertical leaf stacks inside + * left-to-right swimlanes) to prevent 1D horizontal sprawl. + * 3. Allocating ports per card face (top/bottom/left/right) with non-overlapping corridors. + */ +(function () { + const NODE_W = 228; + const NODE_H = 86; + + function computeRanks(nodes, edges) { + const ids = nodes.map((n) => n.id); + const incoming = new Map(ids.map((id) => [id, []])); + for (const e of edges) { + if (incoming.has(e.target) && e.source !== e.target) { + incoming.get(e.target).push(e.source); + } + } + const memo = new Map(); + const visiting = new Set(); + + function rankOf(id) { + if (memo.has(id)) return memo.get(id); + if (visiting.has(id)) return 0; + visiting.add(id); + let r = 0; + for (const pred of incoming.get(id) || []) { + r = Math.max(r, rankOf(pred) + 1); + } + visiting.delete(id); + memo.set(id, r); + return r; + } + + for (const id of ids) rankOf(id); + return memo; + } + + function shiftSubtree(box, dx, dy, subById, nodePositions) { + box.x += dx; + box.y += dy; + for (const sid of box.subIds || []) { + const sb = subById.get(sid); + if (sb) { + sb.x += dx; + sb.y += dy; + } + } + for (const lid of box.leafIds || []) { + const np = nodePositions.get(lid); + if (np) { + np.x += dx; + np.y += dy; + } + } + } + + function layoutCompoundGraph(era, orientation, edgesOverride) { + const isLR = orientation !== "TB"; + const activeEdgeList = edgesOverride || era.edges; + const containerById = new Map(era.containers.map((c) => [c.id, c])); + const childContainers = new Map(); + for (const c of era.containers) { + const pid = c.parentId || "__root__"; + if (!childContainers.has(pid)) childContainers.set(pid, []); + childContainers.get(pid).push(c); + } + + const leavesByContainer = new Map(); + for (const n of era.nodes) { + if (!leavesByContainer.has(n.containerId)) leavesByContainer.set(n.containerId, []); + leavesByContainer.get(n.containerId).push(n); + } + + const ranks = computeRanks(era.nodes, era.edges); + const nodePositions = new Map(); + const containerBoxes = []; + + function descendantLeaves(cid) { + const own = [...(leavesByContainer.get(cid) || [])]; + for (const sub of childContainers.get(cid) || []) { + own.push(...descendantLeaves(sub.id)); + } + return own; + } + + function rootOfContainer(cid) { + let cur = containerById.get(cid); + while (cur && cur.parentId) { + cur = containerById.get(cur.parentId); + } + return cur ? cur.id : cid; + } + + const nodeRootMap = new Map(era.nodes.map((n) => [n.id, rootOfContainer(n.containerId)])); + const rawRoots = [...(childContainers.get("__root__") || [])]; + + // Compute root-container topological tier from cross-root edges when c.tier is omitted + const rootPreds = new Map(rawRoots.map((r) => [r.id, new Set()])); + for (const e of era.edges) { + const sr = nodeRootMap.get(e.source); + const tr = nodeRootMap.get(e.target); + if (sr && tr && sr !== tr && rootPreds.has(tr)) { + rootPreds.get(tr).add(sr); + } + } + const computedTier = new Map(); + const visitingRoot = new Set(); + function getTier(rc) { + if (typeof rc.tier === "number") return rc.tier; + if (computedTier.has(rc.id)) return computedTier.get(rc.id); + if (visitingRoot.has(rc.id)) return 0; + visitingRoot.add(rc.id); + let t = 0; + for (const pId of rootPreds.get(rc.id) || []) { + const pObj = containerById.get(pId); + if (pObj) t = Math.max(t, getTier(pObj) + 1); + } + visitingRoot.delete(rc.id); + computedTier.set(rc.id, t); + return t; + } + + const tiersMap = new Map(); + for (const rc of rawRoots) { + const t = getTier(rc); + if (!tiersMap.has(t)) tiersMap.set(t, []); + tiersMap.get(t).push(rc); + } + const sortedTierKeys = [...tiersMap.keys()].sort((a, b) => a - b); + + const PAD_TOP = 40; + const PAD_SIDE = 16; + const PAD_BOTTOM = 18; + const LEAF_COL_GAP = 48; + const LEAF_ROW_GAP = 48; + const SUB_GAP = 68; + const SUB_STACK_GAP = 48; + const PEER_ROOT_GAP = 50; + const TIER_GAP = 76; + + // Orthogonal leaf placement: in LR mode leaves stack vertically (up to container.maxCols columns) + function layoutLeafGroup(containerObj, leaves, originX, originY) { + if (!leaves.length) return { w: NODE_W, h: NODE_H }; + const ordered = [...leaves].sort((a, b) => (ranks.get(a.id) || 0) - (ranks.get(b.id) || 0)); + const maxSpan = Math.max(1, containerObj?.maxCols || 1); + + let cols, rows; + if (isLR) { + cols = Math.min(ordered.length, maxSpan); + rows = Math.ceil(ordered.length / cols); + } else { + rows = Math.min(ordered.length, maxSpan); + cols = Math.ceil(ordered.length / rows); + } + + ordered.forEach((node, idx) => { + const c = isLR ? idx % cols : Math.floor(idx / rows); + const r = isLR ? Math.floor(idx / cols) : idx % rows; + const x = originX + c * (NODE_W + LEAF_COL_GAP); + const y = originY + r * (NODE_H + LEAF_ROW_GAP); + nodePositions.set(node.id, { x, y, w: NODE_W, h: NODE_H, node }); + }); + + return { + w: cols * NODE_W + (cols - 1) * LEAF_COL_GAP, + h: rows * NODE_H + (rows - 1) * LEAF_ROW_GAP, + }; + } + + // Build a root container at local (0, 0) and then translate into its tier slot + function buildRootBoxAtOrigin(rc) { + const subs = [...(childContainers.get(rc.id) || [])].sort((a, b) => { + const la = descendantLeaves(a.id); + const lb = descendantLeaves(b.id); + const avgA = la.length ? la.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / la.length : 0; + const avgB = lb.length ? lb.reduce((s, n) => s + (ranks.get(n.id) || 0), 0) / lb.length : 0; + return avgA - avgB; + }); + + if (subs.length === 0) { + const leaves = leavesByContainer.get(rc.id) || []; + const inner = layoutLeafGroup(rc, leaves, PAD_SIDE, PAD_TOP); + return { + id: rc.id, + label: rc.label, + tone: rc.tone, + isNested: false, + x: 0, + y: 0, + w: inner.w + PAD_SIDE * 2, + h: inner.h + PAD_TOP + PAD_BOTTOM, + leafIds: leaves.map((l) => l.id), + subIds: [], + }; + } + + const stackSubs = rc.subFlow === "stack"; + // Flow subs along primary axis unless subFlow === "stack" (which stacks along cross axis) + const subsHorizontal = isLR ? !stackSubs : stackSubs; + const gap = stackSubs ? SUB_STACK_GAP : SUB_GAP; + + let cursor = subsHorizontal ? PAD_SIDE : PAD_TOP; + const localSubs = []; + let maxOrtho = 0; + + for (const sub of subs) { + const leaves = leavesByContainer.get(sub.id) || []; + const sx = subsHorizontal ? cursor : PAD_SIDE; + const sy = subsHorizontal ? PAD_TOP : cursor; + const inner = layoutLeafGroup(sub, leaves, sx + PAD_SIDE, sy + PAD_TOP); + const sw = inner.w + PAD_SIDE * 2; + const sh = inner.h + PAD_TOP + PAD_BOTTOM; + localSubs.push({ + id: sub.id, + label: sub.label, + tone: sub.tone, + isNested: true, + x: sx, + y: sy, + w: sw, + h: sh, + leafIds: leaves.map((l) => l.id), + }); + maxOrtho = Math.max(maxOrtho, subsHorizontal ? sh : sw); + cursor += (subsHorizontal ? sw : sh) + gap; + } + + for (const sBox of localSubs) { + const delta = subsHorizontal ? (maxOrtho - sBox.h) / 2 : (maxOrtho - sBox.w) / 2; + if (delta > 0) { + if (subsHorizontal) sBox.y += delta; + else sBox.x += delta; + for (const lid of sBox.leafIds) { + const np = nodePositions.get(lid); + if (np) { + if (subsHorizontal) np.y += delta; + else np.x += delta; + } + } + } + containerBoxes.push(sBox); + } + + const totalFlowSpan = + localSubs.reduce((acc, b) => acc + (subsHorizontal ? b.w : b.h), 0) + + Math.max(0, localSubs.length - 1) * gap; + + return { + id: rc.id, + label: rc.label, + tone: rc.tone, + isNested: false, + x: 0, + y: 0, + w: subsHorizontal ? totalFlowSpan + PAD_SIDE * 2 : maxOrtho + PAD_SIDE * 2, + h: subsHorizontal ? maxOrtho + PAD_TOP + PAD_BOTTOM : totalFlowSpan + PAD_TOP + PAD_BOTTOM, + leafIds: descendantLeaves(rc.id).map((l) => l.id), + subIds: localSubs.map((b) => b.id), + }; + } + + const subById = new Map(); + const placedTiers = []; + const placedRoots = []; + let primaryCursor = 36; + + for (const tk of sortedTierKeys) { + const peerRoots = tiersMap.get(tk) || []; + const builtPeers = peerRoots.map((rc) => { + const box = buildRootBoxAtOrigin(rc); + for (const sb of containerBoxes) subById.set(sb.id, sb); + return box; + }); + + const tierPrimarySpan = Math.max(...builtPeers.map((b) => (isLR ? b.w : b.h))); + let crossCursor = 36; + + for (const pBox of builtPeers) { + const primaryAlign = (tierPrimarySpan - (isLR ? pBox.w : pBox.h)) / 2; + const dx = isLR ? primaryCursor + primaryAlign : crossCursor; + const dy = isLR ? crossCursor : primaryCursor + primaryAlign; + shiftSubtree(pBox, dx, dy, subById, nodePositions); + crossCursor += (isLR ? pBox.h : pBox.w) + PEER_ROOT_GAP; + placedRoots.push(pBox); + } + + const tierCrossSpan = crossCursor - 36 - PEER_ROOT_GAP; + placedTiers.push({ peers: builtPeers, crossSpan: tierCrossSpan }); + primaryCursor += tierPrimarySpan + TIER_GAP; + } + + // Balance all tier columns along the cross axis + const maxTierCross = Math.max(...placedTiers.map((t) => t.crossSpan), 220); + for (const tier of placedTiers) { + const delta = (maxTierCross - tier.crossSpan) / 2; + if (delta > 0) { + for (const pBox of tier.peers) { + shiftSubtree(pBox, isLR ? 0 : delta, isLR ? delta : 0, subById, nodePositions); + } + } + } + + const allContainers = [...placedRoots, ...containerBoxes]; + + // Classify every edge's exit/entry card faces (top/bottom/left/right) and sort ports per face + const facePorts = new Map(); + const edgeMeta = []; + + function addPort(nodeId, face, edgeId, sortCoord) { + const key = `${nodeId}:${face}`; + if (!facePorts.has(key)) facePorts.set(key, []); + facePorts.get(key).push({ edgeId, sortCoord }); + } + + for (const e of activeEdgeList) { + const s = nodePositions.get(e.source); + const t = nodePositions.get(e.target); + if (!s || !t) continue; + + const scx = s.x + s.w / 2; + const scy = s.y + s.h / 2; + const tcx = t.x + t.w / 2; + const tcy = t.y + t.h / 2; + const dx = tcx - scx; + const dy = tcy - scy; + + let sFace, tFace, mode; + const useVertical = isLR + ? Math.abs(dx) < NODE_W * 0.68 + : Math.abs(dy) >= NODE_H * 0.68; + + if (useVertical) { + mode = "V"; + sFace = dy >= 0 ? "bottom" : "top"; + tFace = dy >= 0 ? "top" : "bottom"; + addPort(e.source, sFace, e.id, tcx); + addPort(e.target, tFace, e.id, scx); + } else if (dx >= 0) { + mode = "H_FWD"; + sFace = "right"; + tFace = "left"; + addPort(e.source, sFace, e.id, tcy); + addPort(e.target, tFace, e.id, scy); + } else { + mode = "H_REV"; + sFace = "left"; + tFace = "right"; + addPort(e.source, sFace, e.id, tcy); + addPort(e.target, tFace, e.id, scy); + } + edgeMeta.push({ e, s, t, sFace, tFace, mode }); + } + + for (const list of facePorts.values()) { + list.sort((a, b) => a.sortCoord - b.sortCoord); + } + + function portFraction(nodeId, face, edgeId) { + const list = facePorts.get(`${nodeId}:${face}`) || [{ edgeId }]; + const idx = list.findIndex((x) => x.edgeId === edgeId); + return (Math.max(0, idx) + 1) / (list.length + 1); + } + + const pairKeys = new Set(activeEdgeList.map((e) => `${e.source}->${e.target}`)); + + const routedEdges = edgeMeta.map(({ e, s, t, sFace, tFace, mode }, idx) => { + const hasReverse = pairKeys.has(`${e.target}->${e.source}`); + const sf = portFraction(e.source, sFace, e.id); + const tf = portFraction(e.target, tFace, e.id); + + let x1, y1, x2, y2, path, labelX, labelY; + if (mode === "V") { + const downward = sFace === "bottom"; + const laneShift = hasReverse ? (downward ? -22 : 22) : 0; + x1 = s.x + 28 + sf * (s.w - 56) + laneShift; + y1 = downward ? s.y + s.h : s.y; + x2 = t.x + 28 + tf * (t.w - 56) + laneShift; + y2 = downward ? t.y : t.y + t.h; + const dy = Math.max(24, Math.abs(y2 - y1) * 0.45); + path = downward + ? `M ${x1} ${y1} C ${x1} ${y1 + dy}, ${x2} ${y2 - dy}, ${x2} ${y2}` + : `M ${x1} ${y1} C ${x1} ${y1 - dy}, ${x2} ${y2 + dy}, ${x2} ${y2}`; + labelX = (x1 + x2) / 2; + labelY = (y1 + y2) / 2; + } else if (mode === "H_FWD") { + x1 = s.x + s.w; + x2 = t.x; + const laneShift = hasReverse ? -16 : 0; + y1 = s.y + 16 + sf * (s.h - 32) + laneShift; + y2 = t.y + 16 + tf * (t.h - 32) + laneShift; + const dx = Math.max(42, Math.abs(x2 - x1) * 0.44); + const bow = hasReverse ? -20 : 0; + path = `M ${x1} ${y1} C ${x1 + dx} ${y1 + bow}, ${x2 - dx} ${y2 + bow}, ${x2} ${y2}`; + const shortGap = Math.abs(x2 - x1) < 90; + const tParam = shortGap ? 0.5 : 0.48 + ((idx % 3) - 1) * 0.08; + labelX = x1 + (x2 - x1) * tParam; + labelY = shortGap ? Math.min(y1, y2) - 22 : y1 + (y2 - y1) * tParam + bow * 0.7; + } else { + x1 = s.x; + x2 = t.x + t.w; + const laneShift = hasReverse ? 16 : 0; + y1 = s.y + 16 + sf * (s.h - 32) + laneShift; + y2 = t.y + 16 + tf * (t.h - 32) + laneShift; + const dx = Math.max(42, Math.abs(x1 - x2) * 0.44); + const bow = hasReverse ? 22 : 0; + path = `M ${x1} ${y1} C ${x1 - dx} ${y1 + bow}, ${x2 + dx} ${y2 + bow}, ${x2} ${y2}`; + const tParam = 0.5 + ((idx % 3) - 1) * 0.08; + labelX = x1 + (x2 - x1) * tParam; + labelY = y1 + (y2 - y1) * tParam + bow * 0.7; + } + + return { ...e, x1, y1, x2, y2, path, labelX, labelY }; + }); + + const totalW = Math.max(...allContainers.map((c) => c.x + c.w), 640) + 44; + const totalH = Math.max(...allContainers.map((c) => c.y + c.h), 380) + 44; + + return { + width: totalW, + height: totalH, + containers: allContainers, + nodes: [...nodePositions.values()], + edges: routedEdges, + }; + } + + window.ArchLayoutEngine = { layoutCompoundGraph }; +})(); diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..efa9e8d4e --- /dev/null +++ b/docs/index.md @@ -0,0 +1,30 @@ +# Azure Functions Python Runtime — Engineering & Architecture + +Welcome to the engineering documentation for the **Azure Functions Python Worker & Runtime**. + +While preserving the exact same `FunctionRpc.proto` Host contract and customer-facing `azure-functions` programming models (V1 `function.json` and V2 `@app` decorators), the Python worker architecture has evolved across three distinct generations: + +| Generation | Python Versions | Worker Shell | Execution Engine | Architecture Guide | +| --- | --- | --- | --- | --- | +| **Generation 1 — Monolith** | `3.7 – 3.12` | `workers/azure_functions_worker` (`prodV4/worker.py`) | Bundled inside monolith | [Python ≤ 3.12 Architecture](architecture/python-312.md) | +| **Generation 2 — Proxy Split** | `3.13 – 3.14` | `workers/proxy_worker` (`proxyV4/worker.py`) | `runtimes/v2`, `runtimes/v1`, or Custom Agent Runtime | [Python 3.13 – 3.14 Architecture](architecture/python-313-314.md) | +| **Generation 3 — R2P2 (Rust)** | `3.15+` (GIL) | `workers/r2p2` (`tonic` + `prost` + `PyO3`) | Embedded CPython + `runtimes/v2` / `v1` (`native_invocation.py`) | [Python 3.15+ (R2P2) Architecture](architecture/python-315-r2p2.md) | + +--- + +## Interactive R2P2 Architecture Preview (Python 3.15+) + +Use the interactive explorer below to step through all **7 lifecycle phases** (`Worker Init`, `Specialization`, `Metadata`, `Functions Load`, `Invocation`, `Worker Status`, and `Worker Logs`), inspect extracted AST/Rust symbols, or click **⛶ Expand** for fullscreen mode. Every page in this site also preserves full static Markdown narratives and native `mermaid` diagrams in Git. + +
+ +--- + +## Documentation Map + +- **[Evolution Overview](architecture/overview.md)** — Side-by-side architectural matrix and design trade-offs across `≤ 3.12`, `3.13–3.14`, and `3.15+`. +- **[5-Repository Ecosystem](architecture/ecosystem.md)** — How `azure-functions-host`, `azure-functions-language-worker-protobuf`, `azure-functions-python-worker`, `azure-functions-python-library`, and `azure-functions-python-extensions` fit together. +- **[Python ≤ 3.12 (Monolith)](architecture/python-312.md)** — Single-package `grpcio` + `google.protobuf` worker and its 7 lifecycle flows. +- **[Python 3.13 – 3.14 (Proxy Split)](architecture/python-313-314.md)** — Decoupled `proxy_worker` transport shell, `WorkerRequest` seam, `protos` injection, and pluggable PyPI runtimes. +- **[Python 3.15+ (R2P2 Rust Worker)](architecture/python-315-r2p2.md)** — Native Rust `tonic`/`prost` transport, `PyO3` embedded CPython (GIL), hot `Datum` tuple FFI, and zero-protobuf `protos_adapter.py`. +- **[Dependency Isolation](architecture/dependency-isolation.md)** — Placeholder specialization, `sys.path` ordering, and PEP 420 namespace-safe module eviction. diff --git a/eng/ci/emulator-tests.yml b/eng/ci/emulator-tests.yml index c7792ea94..e343bf9f2 100644 --- a/eng/ci/emulator-tests.yml +++ b/eng/ci/emulator-tests.yml @@ -4,12 +4,24 @@ trigger: # run for sdk and extension release branches include: - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml pr: branches: include: - dev - release* + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml schedules: - cron: "0 8 * * 1,2,3,4,5" diff --git a/eng/ci/integration-tests.yml b/eng/ci/integration-tests.yml index 656317ca2..5002c07d4 100644 --- a/eng/ci/integration-tests.yml +++ b/eng/ci/integration-tests.yml @@ -4,12 +4,24 @@ trigger: # run for sdk and extension release branches include: - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml pr: branches: include: - dev - release* + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml schedules: - cron: "0 8 * * 1,2,3,4,5" diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 9aa464710..03f0961f3 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -6,6 +6,12 @@ trigger: - release/* # proxy worker releases - release-v1/* # V1 worker releases - release-v2/* # V2 worker releases + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml # CI only, does not trigger on PRs. pr: none diff --git a/eng/ci/public-build.yml b/eng/ci/public-build.yml index e32028304..1f491b96f 100644 --- a/eng/ci/public-build.yml +++ b/eng/ci/public-build.yml @@ -5,11 +5,23 @@ trigger: - dev - sdk/* - extensions/* + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml pr: branches: include: - dev + paths: + exclude: + - docs/* + - eng/docs/* + - mkdocs.yml + - .github/workflows/architecture-pages.yml schedules: - cron: '0 0 * * MON' diff --git a/eng/docs/extract_arch.py b/eng/docs/extract_arch.py new file mode 100644 index 000000000..5b8fcdd7b --- /dev/null +++ b/eng/docs/extract_arch.py @@ -0,0 +1,1641 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +""" +Multi-Repository Code-to-Architecture Graph Extractor for Azure Functions Python. + +Synthesizes live AST/Rust/Proto facts from `Azure/azure-functions-python-worker` +and cross-links the 5-repository Python Functions runtime ecosystem: + 1. Azure/azure-functions-host (.NET Script Host runtime & WorkerChannel) + 2. Azure/azure-functions-language-worker-protobuf (FunctionRpc.proto + settlement.proto wire contract) + 3. Azure/azure-functions-python-worker (workers + runtimes v1/v2) + 4. Azure/azure-functions-python-library (azure-functions PyPI SDK) + 5. Azure/azure-functions-python-extensions (HTTP v2, SdkType deferred bindings, Agent runtimes) + +Outputs: + - docs/assets/architecture-graph.json (machine-readable graph + drift fingerprint) + - docs/assets/architecture-data.js (zero-fetch bundle for MkDocs Material & GitHub Pages) + - Bounded `` sections in `docs/architecture/*.md` +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from phases_spec import ( + PHASE_CATALOG, + get_py312_phases, + get_py313_phases, + get_py315_phases, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOCS_DIR = REPO_ROOT / "docs" +JSON_OUT = DOCS_DIR / "assets" / "architecture-graph.json" +JS_OUT = DOCS_DIR / "assets" / "architecture-data.js" + +REPOS: dict[str, dict[str, Any]] = { + "functions-host": { + "id": "functions-host", + "fullName": "Azure/azure-functions-host", + "shortName": "functions-host", + "url": "https://github.com/Azure/azure-functions-host", + "defaultBranch": "dev", + "color": "#f87171", + "pypi": [], + "role": ".NET Functions Script Host runtime: event/trigger listeners, WorkerChannel gRPC server, Consumption specialization, scale probes, HTTP v2 proxy, and AppInsights/Kusto telemetry.", + }, + "language-worker-protobuf": { + "id": "language-worker-protobuf", + "fullName": "Azure/azure-functions-language-worker-protobuf", + "shortName": "worker-protobuf", + "url": "https://github.com/Azure/azure-functions-language-worker-protobuf", + "defaultBranch": "dev", + "color": "#fbbf24", + "pypi": [], + "role": "Canonical gRPC wire contract (`FunctionRpc.proto`, `settlement.proto`, `ClaimsIdentityRpc.proto`) shared between `azure-functions-host` and all language workers.", + }, + "python-worker": { + "id": "python-worker", + "fullName": "Azure/azure-functions-python-worker", + "shortName": "python-worker", + "url": "https://github.com/Azure/azure-functions-python-worker", + "defaultBranch": "dev", + "color": "#38bdf8", + "pypi": ["azure-functions-runtime", "azure-functions-runtime-v1"], + "role": "Worker transports (monolith, proxy_worker, R2P2 Rust) and v1/v2 execution runtimes.", + }, + "python-library": { + "id": "python-library", + "fullName": "Azure/azure-functions-python-library", + "shortName": "python-library", + "url": "https://github.com/Azure/azure-functions-python-library", + "defaultBranch": "dev", + "color": "#34d399", + "pypi": ["azure-functions"], + "role": "Customer-facing Python SDK: V2 @app decorators, Blueprints, Datum converters, Trigger/Binding types, Out[T], Context.", + }, + "python-extensions": { + "id": "python-extensions", + "fullName": "Azure/azure-functions-python-extensions", + "shortName": "python-extensions", + "url": "https://github.com/Azure/azure-functions-python-extensions", + "defaultBranch": "dev", + "color": "#c084fc", + "pypi": [ + "azurefunctions-extensions-base", + "azurefunctions-extensions-http-fastapi", + "azurefunctions-extensions-bindings-blob", + "azurefunctions-extensions-bindings-servicebus", + "azurefunctions-agents-extensions-*", + ], + "role": "Extension contracts: WebApp HTTP v2 streaming, SdkType deferred bindings, Settlement gRPC client, and Custom/Agent Runtime hooks.", + }, +} + + +def git_info(repo_dir: Path = REPO_ROOT) -> dict[str, str]: + def run(args: list[str], default: str = "") -> str: + try: + return subprocess.check_output( + ["git", *args], cwd=repo_dir, stderr=subprocess.DEVNULL, text=True + ).strip() + except Exception: + return default + + return { + "sha": run(["rev-parse", "HEAD"], "dev"), + "shortSha": run(["rev-parse", "--short", "HEAD"], "dev"), + "branch": run(["rev-parse", "--abbrev-ref", "HEAD"], "dev"), + "subject": run(["log", "-1", "--pretty=%s"], ""), + "committedAt": run(["log", "-1", "--pretty=%cI"], ""), + } + + +def extract_python_file(rel_path: str, base_dir: Path = REPO_ROOT) -> dict[str, Any]: + full = base_dir / rel_path + if not full.exists(): + return { + "path": rel_path, + "exists": False, + "lines": 0, + "symbols": [], + "symbolLines": {}, + "routerCases": [], + } + + src = full.read_text(encoding="utf-8", errors="replace") + lines = src.count("\n") + 1 + symbols: list[dict[str, Any]] = [] + symbol_lines: dict[str, int] = {} + match_cases: list[str] = [] + + try: + tree = ast.parse(src, filename=rel_path) + doc = ast.get_docstring(tree) or "" + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbol_lines.setdefault(node.name, node.lineno) + args = [a.arg for a in node.args.args if a.arg != "self"] + symbols.append({ + "name": node.name, + "kind": "async def" if isinstance(node, ast.AsyncFunctionDef) else "def", + "line": node.lineno, + "signature": f"{node.name}({', '.join(args[:4])}{', ...' if len(args) > 4 else ''})", + }) + elif isinstance(node, ast.ClassDef): + symbol_lines.setdefault(node.name, node.lineno) + methods: list[str] = [] + for n in node.body: + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbol_lines[f"{node.name}.{n.name}"] = n.lineno + symbol_lines.setdefault(n.name, n.lineno) + if not n.name.startswith("__"): + methods.append(n.name) + symbols.append({ + "name": node.name, + "kind": "class", + "line": node.lineno, + "signature": f"class {node.name}", + "methods": methods[:6], + }) + for node in ast.walk(tree): + if isinstance(node, ast.Match): + for case in node.cases: + if isinstance(case.pattern, ast.MatchValue) and isinstance(case.pattern.value, ast.Constant): + val = str(case.pattern.value.value) + if val.endswith("_request") or val.endswith("_response"): + match_cases.append(val) + except SyntaxError: + doc = "" + + return { + "path": rel_path, + "exists": True, + "lines": lines, + "docSummary": doc.strip().splitlines()[0] if doc.strip() else "", + "symbols": symbols, + "symbolLines": symbol_lines, + "routerCases": sorted(set(match_cases)), + } + + +def _extract_rust_param_names(src: str, pos: int) -> list[str]: + """Skip optional `<...>` generics/lifetimes at *pos*, then parse balanced `(...)` params.""" + n = len(src) + while pos < n and src[pos].isspace(): + pos += 1 + if pos < n and src[pos] == "<": + depth = 0 + while pos < n: + if src[pos] == "<": + depth += 1 + elif src[pos] == ">": + depth -= 1 + if depth == 0: + pos += 1 + break + pos += 1 + while pos < n and src[pos].isspace(): + pos += 1 + if pos >= n or src[pos] != "(": + return [] + + pos += 1 + paren_depth = 1 + angle_depth = 0 + cur: list[str] = [] + raw_params: list[str] = [] + while pos < n and paren_depth > 0: + ch = src[pos] + if ch == "(": + paren_depth += 1 + cur.append(ch) + elif ch == ")": + paren_depth -= 1 + if paren_depth == 0: + raw_params.append("".join(cur)) + break + cur.append(ch) + elif ch == "<": + angle_depth += 1 + cur.append(ch) + elif ch == ">" and angle_depth > 0: + angle_depth -= 1 + cur.append(ch) + elif ch == "," and paren_depth == 1 and angle_depth == 0: + raw_params.append("".join(cur)) + cur = [] + else: + cur.append(ch) + pos += 1 + + names: list[str] = [] + for raw in raw_params: + cleaned = raw.strip() + if not cleaned: + continue + pname = cleaned.split(":", 1)[0].strip().lstrip("mut ").strip() + if pname: + names.append(pname) + return names + + +def extract_rust_file(rel_path: str) -> dict[str, Any]: + full = REPO_ROOT / rel_path + if not full.exists(): + return { + "path": rel_path, + "exists": False, + "lines": 0, + "symbols": [], + "symbolLines": {}, + "pyo3Calls": [], + } + + src = full.read_text(encoding="utf-8", errors="replace") + src_lines = src.splitlines() + symbols: list[dict[str, Any]] = [] + symbol_lines: dict[str, int] = {} + pyo3_calls: list[str] = [] + + item_re = re.compile( + r"^[ \t]*(?:(pub(?:\([^)]*\))?\s+)?(async\s+)?fn\s+([A-Za-z0-9_]+)|(pub(?:\([^)]*\))?\s+)?(struct|enum)\s+([A-Za-z0-9_]+))", + flags=re.M, + ) + call_method_re = re.compile(r'call_method1\(\s*"([A-Za-z0-9_]+)"') + + for m in item_re.finditer(src): + line_no = src.count("\n", 0, m.start()) + 1 + fn_pub, fn_async, fn_name, st_pub, st_kw, st_name = m.groups() + if fn_name: + if fn_name.startswith("test_"): + continue + symbol_lines.setdefault(fn_name, line_no) + kind = f"{'pub ' if fn_pub else ''}{'async ' if fn_async else ''}fn".strip() + pnames = _extract_rust_param_names(src, m.end()) + short_params = ", ".join(pnames[:4]) + (", ..." if len(pnames) > 4 else "") + symbols.append({ + "name": fn_name, + "kind": kind, + "line": line_no, + "signature": f"{fn_name}({short_params})", + }) + elif st_name: + symbol_lines.setdefault(st_name, line_no) + symbols.append({"name": st_name, "kind": st_kw, "line": line_no, "signature": st_name}) + + for m in call_method_re.finditer(src): + pyo3_calls.append(m.group(1)) + + return { + "path": rel_path, + "exists": True, + "lines": len(src_lines), + "symbols": symbols, + "symbolLines": symbol_lines, + "pyo3Calls": sorted(set(pyo3_calls)), + } + + +def extract_proto_messages(rel_path: str) -> list[dict[str, Any]]: + full = REPO_ROOT / rel_path + if not full.exists(): + return [] + src = full.read_text(encoding="utf-8", errors="replace") + m = re.search(r"message\s+StreamingMessage\s*\{(.*?)\n\}", src, flags=re.S) + if not m: + return [] + oneof = re.search(r"oneof\s+content\s*\{(.*?)\}", m.group(1), flags=re.S) + if not oneof: + return [] + fields = [] + for fm in re.finditer(r"^\s*([A-Za-z0-9_]+)\s+([a-z0-9_]+)\s*=\s*(\d+)\s*;", oneof.group(1), flags=re.M): + msg_type, field_name, tag = fm.groups() + fields.append({"type": msg_type, "field": field_name, "tag": int(tag)}) + return fields + + +def scan_package_files(rel_dir: str, suffixes: tuple[str, ...]) -> list[dict[str, Any]]: + base = REPO_ROOT / rel_dir + if not base.exists(): + return [] + out = [] + for p in sorted(base.rglob("*")): + if not p.is_file() or p.suffix not in suffixes: + continue + rel = p.relative_to(REPO_ROOT).as_posix() + if any(seg in rel for seg in ("/_thirdparty/", "/_vendored/", "/protos/", "__pycache__", "/target/")): + continue + lines = p.read_text(encoding="utf-8", errors="replace").count("\n") + 1 + out.append({"path": rel, "lines": lines}) + return out + + +def sym_line(meta: dict[str, Any], name: str, fallback: int = 1) -> int: + sym_map = meta.get("symbolLines", {}) + if name in sym_map: + return sym_map[name] + for s in meta.get("symbols", []): + if s["name"] == name: + return s["line"] + return fallback + + +def compile_topology_mermaid(era: dict[str, Any], edges_override: list[dict[str, Any]] | None = None) -> str: + lines = ["flowchart TB"] + by_parent: dict[str | None, list[dict[str, Any]]] = {} + for c in era["containers"]: + by_parent.setdefault(c.get("parentId"), []).append(c) + nodes_by_c: dict[str, list[dict[str, Any]]] = {} + for n in era["nodes"]: + nodes_by_c.setdefault(n["containerId"], []).append(n) + + def emit(c: dict[str, Any], ind: str) -> None: + cid = c["id"].replace("-", "_") + lines.append(f'{ind}subgraph {cid}["{c["label"]}"]') + for sub in by_parent.get(c["id"], []): + emit(sub, ind + " ") + for n in nodes_by_c.get(c["id"], []): + nid = n["id"].replace("-", "_") + repo_short = REPOS.get(n["repo"], {}).get("shortName", n["repo"]) + lines.append(f'{ind} {nid}["{n["label"]}
({repo_short})"]') + lines.append(f"{ind}end") + + for rc in by_parent.get(None, []): + emit(rc, " ") + for e in (edges_override if edges_override is not None else era["edges"]): + s = e["source"].replace("-", "_") + t = e["target"].replace("-", "_") + arr = "-.->" if e.get("dashed") else "-->" + lines.append(f' {s} {arr}|"{e["label"]}"| {t}') + return "\n".join(lines) + + +def compile_sequence_mermaid(flow: dict[str, Any]) -> str: + lines = ["sequenceDiagram"] + for a in flow["actors"]: + lines.append(f' participant {a["id"].replace("-", "_")} as {a["label"]}') + for st in flow["steps"]: + fr = st["from"].replace("-", "_") + to = st["to"].replace("-", "_") + arr = "-->>" if st.get("isReturn") else "->>" + lines.append(f' {fr}{arr}{to}: {st["message"]}') + return "\n".join(lines) + + +def resolve_github_url(node: dict[str, Any], repo_refs: dict[str, str]) -> str: + repo_id = node["repo"] + repo = REPOS[repo_id] + ref = repo_refs.get(repo_id) or repo["defaultBranch"] + line_sfx = f"#L{node['line']}" if node.get("line", 0) > 1 else "" + return f"{repo['url']}/blob/{ref}/{node['file']}{line_sfx}" + + +def build_graph(sibling_root: Path | None = None) -> dict[str, Any]: + git = git_info(REPO_ROOT) + repo_dirs: dict[str, Path] = {"python-worker": REPO_ROOT} + repo_refs: dict[str, str] = { + "python-worker": git["sha"] if len(git["sha"]) >= 7 else "dev" + } + if sibling_root and sibling_root.exists(): + for rid in ( + "functions-host", + "language-worker-protobuf", + "python-library", + "python-extensions", + ): + cand = sibling_root / rid + if cand.exists(): + repo_dirs[rid] = cand + s_git = git_info(cand) + if len(s_git["sha"]) >= 7: + repo_refs[rid] = s_git["sha"] + + py_paths = [ + "workers/python/prodV4/worker.py", + "workers/python/proxyV4/worker.py", + "workers/azure_functions_worker/main.py", + "workers/azure_functions_worker/dispatcher.py", + "workers/azure_functions_worker/loader.py", + "workers/azure_functions_worker/http_v2.py", + "workers/azure_functions_worker/extension.py", + "workers/azure_functions_worker/utils/dependency.py", + "workers/proxy_worker/start_worker.py", + "workers/proxy_worker/dispatcher.py", + "workers/proxy_worker/logging.py", + "workers/proxy_worker/utils/dependency.py", + "workers/r2p2/bridge/bridge.py", + "workers/r2p2/bridge/protos_adapter.py", + "runtimes/v2/azure_functions_runtime/handle_event.py", + "runtimes/v2/azure_functions_runtime/native_invocation.py", + "runtimes/v2/azure_functions_runtime/loader.py", + "runtimes/v2/azure_functions_runtime/http_v2.py", + "runtimes/v2/azure_functions_runtime/otel.py", + "runtimes/v1/azure_functions_runtime_v1/handle_event.py", + "runtimes/v1/azure_functions_runtime_v1/native_invocation.py", + ] + rs_paths = [ + "workers/r2p2/src/main.rs", + "workers/r2p2/src/bridge.rs", + "workers/r2p2/src/control.rs", + "workers/r2p2/src/convert.rs", + "workers/r2p2/src/codec.rs", + "workers/r2p2/src/log.rs", + "workers/r2p2/src/pb.rs", + ] + + extracted: dict[str, dict[str, Any]] = {p: extract_python_file(p) for p in py_paths} + for p in rs_paths: + extracted[p] = extract_rust_file(p) + + proto_messages = extract_proto_messages("workers/r2p2/proto/FunctionRpc.proto") + inventories = { + "azure_functions_worker": scan_package_files("workers/azure_functions_worker", (".py",)), + "proxy_worker": scan_package_files("workers/proxy_worker", (".py",)), + "r2p2_rust": scan_package_files("workers/r2p2/src", (".rs",)), + "r2p2_bridge": scan_package_files("workers/r2p2/bridge", (".py",)), + "runtime_v2": scan_package_files("runtimes/v2/azure_functions_runtime", (".py",)), + "runtime_v1": scan_package_files("runtimes/v1/azure_functions_runtime_v1", (".py",)), + } + + mono_disp = extracted["workers/azure_functions_worker/dispatcher.py"] + proxy_disp = extracted["workers/proxy_worker/dispatcher.py"] + r2p2_main = extracted["workers/r2p2/src/main.rs"] + r2p2_bridge_rs = extracted["workers/r2p2/src/bridge.rs"] + r2p2_bridge_py = extracted["workers/r2p2/bridge/bridge.py"] + rt_v2_event = extracted["runtimes/v2/azure_functions_runtime/handle_event.py"] + rt_v2_native = extracted["runtimes/v2/azure_functions_runtime/native_invocation.py"] + + eras = [ + # ===================================================================== + # VIEW 0: 5-Repository Ecosystem Topology + # ===================================================================== + { + "id": "ecosystem", + "shortTitle": "5-Repo Ecosystem", + "title": "Cross-Repository Architecture (Host, Protobuf Contract, Worker, SDK & Extensions)", + "pythonVersions": "All Supported Versions (3.7 → 3.15+)", + "packagePath": "5 Linked GitHub Repositories", + "launcherPath": "functions-host <-> worker-protobuf <-> python-worker <-> library & extensions", + "status": "Canonical 5-Repo Map", + "tagline": "How azure-functions-host, language-worker-protobuf, python-worker, python-library, and python-extensions compose the runtime.", + "summary": ( + "The Azure Functions Python stack spans five repositories with clean boundaries: " + "`azure-functions-host` houses the .NET Script Host runtime (`WorkerChannel`, trigger listeners, " + "placeholder specialization, scale probes, and telemetry); `language-worker-protobuf` defines the " + "gRPC wire contract (`FunctionRpc.proto`, `settlement.proto`) between Host and workers; " + "`python-worker` implements the Python/Rust worker shells and v1/v2 execution runtimes; " + "`python-library` ships the `azure-functions` SDK (`@app` decorators & `Datum`); and " + "`python-extensions` provides HTTP v2 streaming, `SdkType` deferred bindings, and Agent Runtimes." + ), + "tradeoffs": [ + {"kind": "strength", "text": "`azure-functions-host` and `python-worker` evolve independently across the stable `language-worker-protobuf` contract."}, + {"kind": "strength", "text": "`python-library` and `python-extensions` ship on PyPI independently of Azure Host VM images."}, + {"kind": "strength", "text": "Extension hooks (`WebApp`, `SdkType`, `RuntimeTrackerMeta`) enable FastAPI, SDK bindings, and Agent Frameworks cleanly."}, + ], + "erasDiff": { + "transport": "Host ServerDuplexChannel <-> FunctionRpc.proto <-> python-worker", + "boundary": "Protobuf wire contract (Host<->Worker) + Datum/Metaclass contracts (Worker<->SDK/Ext)", + "runtimeLoading": "python-worker runtimes/v1,v2 or python-extensions RuntimeTrackerMeta", + "hotPathOverhead": "Datum bindings take R2P2 Hot Path; Extensions (HTTP-v2 / SdkType) take Control Path", + }, + "containers": [ + {"id": "ce-host", "label": "Azure/azure-functions-host (.NET Script Host)", "parentId": None, "tone": "host", "tier": 0}, + {"id": "ce-proto", "label": "Azure/azure-functions-language-worker-protobuf (Contract)", "parentId": None, "tone": "proto", "tier": 1}, + {"id": "ce-worker", "label": "Azure/azure-functions-python-worker", "parentId": None, "tone": "rust-proc", "tier": 2, "subFlow": "stack"}, + {"id": "ce-w-shell", "label": "Worker Transport Shells (workers/)", "parentId": "ce-worker", "tone": "transport"}, + {"id": "ce-w-rt", "label": "Execution Runtimes (runtimes/v1 & v2)", "parentId": "ce-worker", "tone": "runtime"}, + {"id": "ce-lib", "label": "Azure/azure-functions-python-library", "parentId": None, "tone": "lib", "tier": 3}, + {"id": "ce-ext", "label": "Azure/azure-functions-python-extensions", "parentId": None, "tone": "ext", "tier": 3}, + ], + "nodes": [ + { + "id": "ne-host-chan", + "repo": "functions-host", + "containerId": "ce-host", + "label": "Script Host & WorkerChannel", + "badge": "WorkerChannel.cs", + "role": ".NET Script Host orchestrating triggers, worker process lifecycle, placeholder specialization, status probes, and RpcLog telemetry.", + "file": "src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs", + "line": 1, + "details": [ + "Spawns worker processes and drives the 7 lifecycle phases (`WorkerInit`, `EnvironmentReload`, `Metadata`, `Load`, `Invoke`, `Status`, `RpcLog`).", + "Proxies HTTP v2 streaming traffic (`HttpWorkerOptions`) and routes `RpcLog` to Application Insights / Kusto.", + ], + }, + { + "id": "ne-host-duplex", + "repo": "functions-host", + "containerId": "ce-host", + "label": "ServerDuplexChannel & ShMem", + "badge": "ServerDuplexChannel.cs", + "role": "Host-side bidirectional gRPC `EventStream` endpoint and POSIX/Windows shared-memory data transfer.", + "file": "src/WebJobs.Script.Grpc/Channel/ServerDuplexChannel.cs", + "line": 1, + "details": [ + "Registered in `FunctionRpcChannelEndpoints.cs`; pairs with `RpcSharedMemoryDataExtensions.cs`.", + ], + }, + { + "id": "ne-funcrpc", + "repo": "language-worker-protobuf", + "containerId": "ce-proto", + "label": "FunctionRpc.proto", + "badge": "Host ↔ Worker Wire Schema", + "role": "Canonical bidirectional gRPC protocol contract shared between `azure-functions-host` and all out-of-proc language workers.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Compiled into C# (`functions-host`), Python `grpcio-tools` (`≤3.14`), and Rust `prost`/`tonic` (`3.15+`).", + "Defines `StreamingMessage`, `TypedData`, `WorkerInit`, `InvocationRequest`, `WorkerStatus`, `RpcLog`.", + ], + }, + { + "id": "ne-settle", + "repo": "language-worker-protobuf", + "containerId": "ce-proto", + "label": "settlement.proto & ClaimsIdentity", + "badge": "Settlement Contract", + "role": "Protobuf service schema enabling Python extensions to settle ServiceBus messages directly against the Host.", + "file": "src/proto/servicebus/settlement.proto", + "line": 1, + "details": [ + "Implemented by `azure-functions-host` and called by `grpcClientType.py` in `python-extensions`.", + ], + }, + { + "id": "ne-shells", + "repo": "python-worker", + "containerId": "ce-w-shell", + "label": "Transports: Monolith / Proxy / R2P2", + "badge": "≤3.12 | 3.13-14 | 3.15+", + "role": "Version-selected worker shell managing Host gRPC stream, logging, and sys.path dependency isolation.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "main", 230), + "details": [ + "≤3.12: `workers/azure_functions_worker` (monolith).", + "3.13–3.14: `workers/proxy_worker` (Python transport shell).", + "3.15+: `workers/r2p2` (Rust `tonic`+`prost` + `PyO3` embedded CPython).", + ], + }, + { + "id": "ne-runtimes", + "repo": "python-worker", + "containerId": "ce-w-rt", + "label": "azure-functions-runtime (v2 & v1)", + "badge": "handle_event + native_invocation", + "role": "PyPI-published execution engines that index apps, decode/encode bindings, and run sync/async handlers.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "`runtimes/v2`: V2 decorator model, HTTP v2 coordinator, OpenTelemetry.", + "`runtimes/v1`: V1 `function.json` folder model.", + "Exposes both `handle_event.py` (5-verb contract) and `native_invocation.py` (R2P2 zero-protobuf seam).", + ], + }, + { + "id": "ne-lib-dec", + "repo": "python-library", + "containerId": "ce-lib", + "label": "Decorators & Blueprints", + "badge": "FunctionApp / @app.*", + "role": "Defines V2 Python programming model (`@app.route`, `@app.queue_trigger`, `Blueprint`, MCP & Connector triggers).", + "file": "azure/functions/__init__.py", + "line": 1, + "details": [ + "Inspected by `runtimes/v2/azure_functions_runtime/loader.py` (`index_function_app`) during WorkerInit/Metadata.", + "Includes `azure/functions/mcp.py`, `connectors.py`, `warmup.py`, `decorators/`.", + ], + }, + { + "id": "ne-lib-datum", + "repo": "python-library", + "containerId": "ce-lib", + "label": "meta.py Datum & Converters", + "badge": "Datum / InConverter / Out[T]", + "role": "Protobuf-agnostic `Datum` value container and trigger/binding converters (`HttpRequest`, `QueueMessage`, `Blob`).", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "`Datum` is the exact currency constructed by R2P2's `native_invocation.py` — enabling zero-protobuf execution!", + "`azure/functions/extension/` also defines `AppExtensionBase` & `FuncExtensionBase`.", + ], + }, + { + "id": "ne-ext-base", + "repo": "python-extensions", + "containerId": "ce-ext", + "label": "extensions-base (WebApp, SdkType, Runtime)", + "badge": "web.py | sdkType.py | runtime.py", + "role": "Metaclass registries bridging HTTP v2 ASGI servers, SDK-type deferred bindings, and custom Agent Runtimes.", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py", + "line": 1, + "details": [ + "`runtime.py` (`RuntimeTrackerMeta`): discovered when `PYTHON_ENABLE_AGENT_RUNTIME=1`.", + "`web.py` (`WebApp`): drives `http_v2.py` loopback streaming for FastAPI/ASGI.", + "`sdkType.py` (`SdkType`): drives deferred bindings (`requires_control_path() == true` in R2P2).", + ], + }, + { + "id": "ne-ext-pkgs", + "repo": "python-extensions", + "containerId": "ce-ext", + "label": "FastAPI, SDK Bindings & Agent Runtimes", + "badge": "http-fastapi | bindings-* | agents-*", + "role": "Concrete extension packages for FastAPI HTTP streaming, Blob/Cosmos/EventHub/ServiceBus SDK types, and Agents.", + "file": "azurefunctions-extensions-http-fastapi/README.md", + "line": 1, + "details": [ + "`azurefunctions-extensions-http-fastapi`: Request/Response streaming (`StreamingResponse`, SSE).", + "`azurefunctions-extensions-bindings-{blob,cosmosdb,eventhub,servicebus}`: rich Azure SDK client bindings.", + "`azurefunctions-agents-extensions-{agent-framework,copilot-sdk}`: hosted agent runtimes.", + ], + }, + ], + "edges": [ + {"id": "ee-0a", "source": "ne-host-chan", "target": "ne-host-duplex", "kind": "thread-queue", "label": "dispatch / RpcLog sink"}, + {"id": "ee-0b", "source": "ne-host-duplex", "target": "ne-funcrpc", "kind": "grpc-wire", "label": "EventStream server"}, + {"id": "ee-0c", "source": "ne-host-chan", "target": "ne-settle", "kind": "grpc-wire", "label": "Settlement server", "dashed": True}, + {"id": "ee-1", "source": "ne-funcrpc", "target": "ne-shells", "kind": "grpc-wire", "label": "prost / grpcio client"}, + {"id": "ee-2", "source": "ne-shells", "target": "ne-runtimes", "kind": "pyo3-hot", "label": "WorkerRequest / PyO3 FFI"}, + {"id": "ee-3", "source": "ne-runtimes", "target": "ne-lib-dec", "kind": "inproc-contract", "label": "index_function_app()"}, + {"id": "ee-4", "source": "ne-runtimes", "target": "ne-lib-datum", "kind": "inproc-contract", "label": "Datum <-> Binding decode"}, + {"id": "ee-5", "source": "ne-runtimes", "target": "ne-ext-base", "kind": "pyo3-control", "label": "http_v2 / deferred SdkType"}, + {"id": "ee-6", "source": "ne-shells", "target": "ne-ext-base", "kind": "syspath-mutation", "label": "entry_points(azurefunctions.runtimes)", "dashed": True}, + {"id": "ee-7", "source": "ne-settle", "target": "ne-ext-base", "kind": "grpc-wire", "label": "grpcClientType settlement", "dashed": True}, + {"id": "ee-8", "source": "ne-ext-base", "target": "ne-ext-pkgs", "kind": "exec-dispatch", "label": "metaclass registration"}, + ], + "flows": [ + { + "id": "eco_flow", + "title": "End-to-End 5-Repo Request Lifecycle", + "summary": "How a trigger event in azure-functions-host crosses the worker-protobuf contract into python-worker, python-library, and python-extensions.", + "actors": [ + {"id": "host", "label": "functions-host"}, + {"id": "proto", "label": "worker-protobuf"}, + {"id": "shell", "label": "python-worker (Shell)"}, + {"id": "rt", "label": "python-worker (Runtime)"}, + {"id": "lib", "label": "library & extensions"}, + ], + "steps": [ + {"from": "host", "to": "proto", "message": "WorkerChannel serializes event onto ServerDuplexChannel", "activeNodes": ["ne-host-chan", "ne-host-duplex", "ne-funcrpc"], "activeEdges": ["ee-0a", "ee-0b"], "payload": "WebJobs.Script.Grpc"}, + {"from": "proto", "to": "shell", "message": "FunctionRpc.proto StreamingMessage frame over HTTP/2", "activeNodes": ["ne-funcrpc", "ne-shells"], "activeEdges": ["ee-1"], "payload": "Shared wire contract across 3.7 -> 3.15+"}, + {"from": "shell", "to": "rt", "message": "Dispatch WorkerInit / Invocation across seam", "activeNodes": ["ne-shells", "ne-runtimes"], "activeEdges": ["ee-2"], "payload": "WorkerRequest (3.13) or PyO3 Tuple/Dict (3.15)"}, + {"from": "rt", "to": "lib", "message": "Decode Datum (python-library) or SdkType/WebApp (python-extensions)", "activeNodes": ["ne-runtimes", "ne-lib-dec", "ne-lib-datum", "ne-ext-base", "ne-ext-pkgs"], "activeEdges": ["ee-3", "ee-4", "ee-5", "ee-8"], "payload": "azure.functions + azurefunctions.extensions"}, + {"from": "lib", "to": "host", "message": "InvocationResponse / RpcLog / Settlement back to Host", "isReturn": True, "activeNodes": ["ne-ext-base", "ne-settle", "ne-funcrpc", "ne-host-duplex", "ne-host-chan"], "activeEdges": ["ee-7", "ee-0c", "ee-0b", "ee-0a"], "payload": "StatusResult + OutputBindings + AppInsights"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 1: Python <= 3.12 (Monolithic Worker) + # ===================================================================== + { + "id": "py312", + "shortTitle": "Python ≤ 3.12", + "title": "Monolithic Python Worker (≤ 3.12)", + "pythonVersions": "3.7 – 3.12", + "packagePath": "workers/azure_functions_worker", + "launcherPath": "workers/python/prodV4/worker.py", + "status": "Maintenance (Legacy Monolith)", + "tagline": "Single-package CPython worker owning gRPC transport, protobuf, indexing, bindings, and invocation.", + "summary": ( + "For Python 3.12 and below, `azure_functions_worker` is a single monolithic package. " + "Transport (`grpcio` + `google.protobuf` compiled from `language-worker-protobuf`), " + "dependency isolation, V1/V2 function indexing (`python-library`), HTTP v2 streaming " + "(`python-extensions`), and execution all live in one coupled tree." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Direct single-package Python calls; simplest mental model."}, + {"kind": "pain", "text": "Coupled releases: any binding/HTTP/OTel fix requires shipping a new Host worker bundle for every Python version."}, + {"kind": "pain", "text": "Worker `grpcio` and `google.protobuf` C-extensions live in the same interpreter as customer code."}, + ], + "erasDiff": { + "transport": "Python grpcio + google.protobuf (compiled from FunctionRpc.proto)", + "boundary": "Monolithic internal imports inside azure_functions_worker", + "runtimeLoading": "Bundled directly inside worker (no PyPI runtime split)", + "hotPathOverhead": "grpcio thread -> asyncio -> Python protobuf -> Datum -> handler", + }, + "containers": [ + {"id": "c312-host", "label": "Functions Host (.NET — azure-functions-host)", "parentId": None, "tone": "host", "tier": 0}, + {"id": "c312-wire", "label": "Wire Contract (worker-protobuf)", "parentId": None, "tone": "proto", "tier": 0}, + {"id": "c312-worker", "label": "Monolithic Worker Process (workers/azure_functions_worker)", "parentId": None, "tone": "python", "tier": 1}, + {"id": "c312-trans", "label": "gRPC Transport & Isolation", "parentId": "c312-worker", "tone": "transport"}, + {"id": "c312-eng", "label": "In-Tree Indexing, Bindings & Executor", "parentId": "c312-worker", "tone": "runtime"}, + {"id": "c312-lib", "label": "Azure/azure-functions-python-library", "parentId": None, "tone": "lib", "tier": 2}, + {"id": "c312-ext", "label": "Azure/azure-functions-python-extensions", "parentId": None, "tone": "ext", "tier": 2}, + ], + "nodes": [ + { + "id": "n312-host", + "repo": "functions-host", + "containerId": "c312-host", + "label": "Script Host (WorkerChannel)", + "badge": "WebJobs.Script.Grpc", + "role": ".NET Functions Host runtime spawning `prodV4/worker.py`, driving triggers, scale probes, and telemetry.", + "file": "src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs", + "line": 1, + "details": [ + "Hosts `ServerDuplexChannel` and `RpcSharedMemoryDataExtensions`.", + ], + }, + { + "id": "n312-proto", + "repo": "language-worker-protobuf", + "containerId": "c312-wire", + "label": "FunctionRpc.proto Contract", + "badge": "StreamingMessage Schema", + "role": "Language-neutral gRPC `.proto` contract between `azure-functions-host` and `azure_functions_worker`.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Compiled into `workers/azure_functions_worker/protos/` via `grpcio-tools`.", + ], + }, + { + "id": "n312-boot", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "prodV4/worker.py & main.py", + "badge": "Boot + DependencyManager", + "role": "Sets worker-first sys.path so bundled `grpcio`/`protobuf` import cleanly, then starts asyncio loop.", + "file": "workers/azure_functions_worker/main.py", + "line": sym_line(extracted["workers/azure_functions_worker/main.py"], "start_async", 1), + "details": [ + "`DependencyManager.initialize()` isolates worker deps during placeholder cold boot.", + ], + }, + { + "id": "n312-grpc", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "gRPC Pump (__poll_grpc)", + "badge": "grpcio + _grpc_resp_queue", + "role": "Background OS thread reading `stub.EventStream` and posting tasks via `call_soon_threadsafe`.", + "file": "workers/azure_functions_worker/dispatcher.py", + "line": sym_line(mono_disp, "connect", 180), + "details": [ + "Outbound responses and `AsyncLoggingHandler` `RpcLog` records drain from `_grpc_resp_queue`.", + ], + }, + { + "id": "n312-disp", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "Monolithic Dispatcher", + "badge": "dispatcher.py", + "role": "Central class owning message routing, logging switch, OTel/AppInsights setup, and `_sync_call_tp`.", + "file": "workers/azure_functions_worker/dispatcher.py", + "line": sym_line(mono_disp, "Dispatcher", 76), + "details": [ + "Directly imports `loader`, `bindings`, `http_v2`, `SharedMemoryManager`, and `ExtensionManager`.", + ], + }, + { + "id": "n312-dep", + "repo": "python-worker", + "containerId": "c312-trans", + "label": "DependencyManager", + "badge": "utils/dependency.py", + "role": "Manages worker-first placeholder boot vs customer `.python_packages` promotion on Init/Specialization.", + "file": "workers/azure_functions_worker/utils/dependency.py", + "line": sym_line(extracted["workers/azure_functions_worker/utils/dependency.py"], "DependencyManager", 1), + "details": [ + "Controls `use_worker_dependencies()` and `prioritize_customer_dependencies()`.", + ], + }, + { + "id": "n312-load", + "repo": "python-worker", + "containerId": "c312-eng", + "label": "Loader & ExtensionManager", + "badge": "loader.py + extension.py", + "role": "Indexes V2 `@app` decorators or V1 `function.json` files and invokes `AppExtensionBase` hooks.", + "file": "workers/azure_functions_worker/loader.py", + "line": sym_line(extracted["workers/azure_functions_worker/loader.py"], "index_function_app", 1), + "details": [ + "Populates `functions.Registry` with parameter binding metadata.", + ], + }, + { + "id": "n312-bind", + "repo": "python-worker", + "containerId": "c312-eng", + "label": "Bindings, SharedMem & http_v2", + "badge": "bindings/ + mmap + ASGI", + "role": "Converts `protos.TypedData` <-> `Datum`, manages shared-memory rings, and coordinates HTTP v2 streaming.", + "file": "workers/azure_functions_worker/http_v2.py", + "line": sym_line(extracted["workers/azure_functions_worker/http_v2.py"], "HttpV2Registry", 1), + "details": [ + "Calls into `python-library` converters and `python-extensions` `SdkType`/`WebApp`.", + ], + }, + { + "id": "n312-sdk", + "repo": "python-library", + "containerId": "c312-lib", + "label": "azure-functions SDK & User App", + "badge": "FunctionApp + Datum + Out", + "role": "Customer decorators, trigger/binding models (`HttpRequest`, `QueueMessage`), and `Datum` converters.", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "Shared by worker `bindings/meta.py` and customer `function_app.py`.", + ], + }, + { + "id": "n312-ext", + "repo": "python-extensions", + "containerId": "c312-ext", + "label": "azurefunctions-extensions-*", + "badge": "http-fastapi + bindings-*", + "role": "Optional HTTP v2 ASGI streaming (`WebApp`) and deferred SDK bindings (`BlobClient`, etc.).", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/sdkType.py", + "line": 1, + "details": [ + "Hooks into `loader.py` and `http_v2.py` via base metaclasses.", + ], + }, + ], + "edges": [ + {"id": "e312-0", "source": "n312-host", "target": "n312-proto", "kind": "grpc-wire", "label": "ServerDuplexChannel"}, + {"id": "e312-1", "source": "n312-proto", "target": "n312-grpc", "kind": "grpc-wire", "label": "EventStream (grpcio)"}, + {"id": "e312-2", "source": "n312-boot", "target": "n312-disp", "kind": "syspath-mutation", "label": "boot + DependencyManager"}, + {"id": "e312-3", "source": "n312-grpc", "target": "n312-disp", "kind": "thread-queue", "label": "call_soon_threadsafe / Queue"}, + {"id": "e312-4", "source": "n312-disp", "target": "n312-load", "kind": "inproc-contract", "label": "index & load"}, + {"id": "e312-5", "source": "n312-disp", "target": "n312-bind", "kind": "inproc-contract", "label": "TypedData <-> args"}, + {"id": "e312-6", "source": "n312-load", "target": "n312-sdk", "kind": "exec-dispatch", "label": "inspect @app / function.json"}, + {"id": "e312-7", "source": "n312-bind", "target": "n312-sdk", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e312-8", "source": "n312-bind", "target": "n312-ext", "kind": "pyo3-control", "label": "WebApp / SdkType", "dashed": True}, + ], + "flows": [ + { + "id": "init", + "title": "Cold Start & Monolithic Init", + "summary": "prodV4 boots monolith, opens grpcio EventStream, switches sys.path to customer packages, and indexes decorators.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "grpc", "label": "gRPC Thread"}, + {"id": "disp", "label": "Dispatcher"}, + {"id": "load", "label": "Loader"}, + {"id": "sdk", "label": "python-library"}, + ], + "steps": [ + {"from": "host", "to": "grpc", "message": "Spawn prodV4/worker.py + StartStream", "activeNodes": ["n312-proto", "n312-boot", "n312-grpc"], "activeEdges": ["e312-1", "e312-2"], "payload": "Worker grpcio/protobuf loaded first"}, + {"from": "host", "to": "disp", "message": "WorkerInitRequest(app_dir, capabilities)", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["e312-1", "e312-3"], "payload": "StreamingMessage.worker_init_request"}, + {"from": "disp", "to": "load", "message": "prioritize_customer_deps + index_function_app()", "activeNodes": ["n312-disp", "n312-load"], "activeEdges": ["e312-4"], "payload": "Customer .python_packages promoted"}, + {"from": "load", "to": "sdk", "message": "Read FunctionApp / Blueprints", "activeNodes": ["n312-load", "n312-sdk"], "activeEdges": ["e312-6"], "payload": "azure.functions decorators"}, + {"from": "disp", "to": "host", "message": "WorkerInitResponse(capabilities)", "isReturn": True, "activeNodes": ["n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["e312-3", "e312-1"], "payload": "SHARED_MEMORY + RAW_HTTP_BODY_BYTES"}, + ], + }, + { + "id": "invoke", + "title": "Invocation Lifecycle", + "summary": "Dispatcher decodes protobuf TypedData via bindings, runs sync threadpool or async coroutine, and encodes response.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "disp", "label": "Dispatcher"}, + {"id": "bind", "label": "Bindings / http_v2"}, + {"id": "sdk", "label": "SDK & User App"}, + ], + "steps": [ + {"from": "host", "to": "disp", "message": "InvocationRequest(function_id, input_data)", "activeNodes": ["n312-proto", "n312-grpc", "n312-disp"], "activeEdges": ["e312-1", "e312-3"], "payload": "google.protobuf InvocationRequest"}, + {"from": "disp", "to": "bind", "message": "from_incoming_proto() / SharedMemory", "activeNodes": ["n312-disp", "n312-bind"], "activeEdges": ["e312-5"], "payload": "TypedData -> Datum"}, + {"from": "bind", "to": "sdk", "message": "Decode SDK types + run handler (_sync_call_tp / await)", "activeNodes": ["n312-bind", "n312-sdk", "n312-ext"], "activeEdges": ["e312-7", "e312-8"], "payload": "HttpRequest / SdkType -> User fn"}, + {"from": "sdk", "to": "host", "message": "to_outgoing_proto() -> InvocationResponse", "isReturn": True, "activeNodes": ["n312-sdk", "n312-bind", "n312-disp", "n312-grpc", "n312-proto"], "activeEdges": ["e312-7", "e312-5", "e312-3", "e312-1"], "payload": "Queued onto _grpc_resp_queue"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 2: Python 3.13 - 3.14 (Split Proxy Worker + Pluggable Runtimes) + # ===================================================================== + { + "id": "py313_314", + "shortTitle": "Python 3.13 – 3.14", + "title": "Split Proxy Worker + PyPI Runtimes (3.13 – 3.14)", + "pythonVersions": "3.13 – 3.14", + "packagePath": "workers/proxy_worker + runtimes/{v1,v2}", + "launcherPath": "workers/python/proxyV4/worker.py", + "status": "Production (Python 3.13 / 3.14)", + "tagline": "Thin transport shell (`proxy_worker`) calling decoupled PyPI engines (`azure-functions-runtime` v2/v1 or Agent Runtimes).", + "summary": ( + "Python 3.13 split the worker into two independently versioned layers. " + "`workers/proxy_worker` is a thin, stable transport shell owning `grpcio`, " + "`DependencyManager`, logging switch, and 4-tier `invocation_id` tracking. " + "It delegates execution across a 5-verb `WorkerRequest` seam (injecting `protos` at init) " + "to `runtimes/v2`, `runtimes/v1`, or a custom `python-extensions` Agent Runtime." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Decoupled cadence: `azure-functions-runtime` updates ship via PyPI without waiting for Host worker deployments."}, + {"kind": "strength", "text": "Injected `protos` decouples runtime code from specific protobuf wheel builds; unlocks pluggable Agent Runtimes."}, + {"kind": "pain", "text": "`proxy_worker` still links CPython `grpcio` + `protobuf` wheels (delaying day-1 support for new Python releases) and contends on the GIL."}, + ], + "erasDiff": { + "transport": "Python proxy_worker (grpcio + worker protobuf + uvloop)", + "boundary": "In-process async WorkerRequest(name, request, properties={protos, host})", + "runtimeLoading": "reload_library_worker(): Agent EntryPoint -> V2 (function_app.py) -> V1", + "hotPathOverhead": "grpcio thread -> asyncio -> WorkerRequest -> runtime cached protos", + }, + "containers": [ + {"id": "c313-host", "label": "Functions Host (.NET — azure-functions-host)", "parentId": None, "tone": "host", "tier": 0}, + {"id": "c313-wire", "label": "Wire Contract (worker-protobuf)", "parentId": None, "tone": "proto", "tier": 0}, + {"id": "c313-worker", "label": "Python Worker Process (proxyV4/worker.py)", "parentId": None, "tone": "python", "tier": 1}, + {"id": "c313-proxy", "label": "Layer 1: proxy_worker (Transport Shell)", "parentId": "c313-worker", "tone": "transport"}, + {"id": "c313-rt", "label": "Layer 2: Pluggable Runtime Engine (runtimes/)", "parentId": "c313-worker", "tone": "runtime"}, + {"id": "c313-lib", "label": "Azure/azure-functions-python-library", "parentId": None, "tone": "lib", "tier": 2}, + {"id": "c313-ext", "label": "Azure/azure-functions-python-extensions", "parentId": None, "tone": "ext", "tier": 2}, + ], + "nodes": [ + { + "id": "n313-host", + "repo": "functions-host", + "containerId": "c313-host", + "label": "Script Host (WorkerChannel)", + "badge": "WebJobs.Script.Grpc", + "role": ".NET Functions Host runtime spawning `proxyV4/worker.py` (`python -m proxy_worker`) for Python 3.13/3.14.", + "file": "src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs", + "line": 1, + "details": [ + "Unchanged .NET Host runtime across Python worker versions.", + ], + }, + { + "id": "n313-proto", + "repo": "language-worker-protobuf", + "containerId": "c313-wire", + "label": "FunctionRpc.proto Contract", + "badge": "StreamingMessage Schema", + "role": "Canonical gRPC wire schema compiled into `proxy_worker/protos` and injected into PyPI runtimes.", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Identical wire contract; `worker_status_request` answered directly by `proxy_worker`.", + ], + }, + { + "id": "n313-grpc", + "repo": "python-worker", + "containerId": "c313-proxy", + "label": "start_worker + gRPC Pump", + "badge": "uvloop + grpc_local-thread", + "role": "Locks worker deps first, enables `uvloop`, opens `grpcio` stream, and switches console -> `RpcLog`.", + "file": "workers/proxy_worker/dispatcher.py", + "line": sym_line(proxy_disp, "__poll_grpc", 279), + "details": [ + "4-tier `invocation_id` tracking: global -> `ContextEnabledTask` -> `invocation_id_cv` -> thread-local.", + ], + }, + { + "id": "n313-router", + "repo": "python-worker", + "containerId": "c313-proxy", + "label": "Proxy Router & WorkerRequest", + "badge": "reload_library_worker()", + "role": "Promotes customer deps, dynamically selects V2 / V1 / Agent runtime, and passes `WorkerRequest` + `protos`.", + "file": "workers/proxy_worker/dispatcher.py", + "line": sym_line(proxy_disp, "_dispatch_grpc_request", 329), + "details": [ + "Injects `properties={'protos': protos, 'host': host}` on Init & EnvironmentReload so runtimes stay protobuf-version agnostic.", + ], + }, + { + "id": "n313-rtv2", + "repo": "python-worker", + "containerId": "c313-rt", + "label": "Runtime V2 (azure_functions_runtime)", + "badge": "runtimes/v2/handle_event.py", + "role": "PyPI V2 engine: decorator indexing, `Datum` bindings, HTTP v2 streaming, OpenTelemetry, threadpool.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "Selected when `function_app.py` (or configured script file) exists in app directory.", + ], + }, + { + "id": "n313-rtv1", + "repo": "python-worker", + "containerId": "c313-rt", + "label": "Runtime V1 (azure_functions_runtime_v1)", + "badge": "runtimes/v1/handle_event.py", + "role": "PyPI V1 engine supporting classic `function.json` apps across the exact same 5-verb contract.", + "file": "runtimes/v1/azure_functions_runtime_v1/handle_event.py", + "line": sym_line(extracted["runtimes/v1/azure_functions_runtime_v1/handle_event.py"], "worker_init_request", 45), + "details": [ + "Selected automatically when `function_app.py` is not present.", + ], + }, + { + "id": "n313-lib", + "repo": "python-library", + "containerId": "c313-lib", + "label": "azure-functions SDK & User App", + "badge": "Decorators + Datum + Handler", + "role": "Customer `function_app.py` + `azure-functions` SDK types invoked on asyncio loop or threadpool.", + "file": "azure/functions/__init__.py", + "line": 1, + "details": [ + "Customer can pin newer `azure-functions` and `azure-functions-runtime` in `requirements.txt`.", + ], + }, + { + "id": "n313-ext", + "repo": "python-extensions", + "containerId": "c313-ext", + "label": "Agent Runtimes, WebApp & SdkType", + "badge": "runtime.py | web.py | sdkType.py", + "role": "Custom `azurefunctions.runtimes` entry points (`PYTHON_ENABLE_AGENT_RUNTIME`), FastAPI HTTP v2, and SDK bindings.", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/runtime.py", + "line": 1, + "details": [ + "`RuntimeTrackerMeta` + `entry_points(group='azurefunctions.runtimes')` can replace V1/V2 engine entirely!", + ], + }, + ], + "edges": [ + {"id": "e313-0", "source": "n313-host", "target": "n313-proto", "kind": "grpc-wire", "label": "ServerDuplexChannel"}, + {"id": "e313-1", "source": "n313-proto", "target": "n313-grpc", "kind": "grpc-wire", "label": "EventStream (grpcio)"}, + {"id": "e313-2", "source": "n313-grpc", "target": "n313-router", "kind": "thread-queue", "label": "ContextEnabledTask / Queue"}, + {"id": "e313-3", "source": "n313-router", "target": "n313-rtv2", "kind": "inproc-contract", "label": "WorkerRequest (V2)"}, + {"id": "e313-4", "source": "n313-router", "target": "n313-rtv1", "kind": "inproc-contract", "label": "WorkerRequest (V1)", "dashed": True}, + {"id": "e313-5", "source": "n313-router", "target": "n313-ext", "kind": "syspath-mutation", "label": "Agent Runtime EntryPoint", "dashed": True}, + {"id": "e313-6", "source": "n313-rtv2", "target": "n313-ext", "kind": "pyo3-control", "label": "WebApp HTTP-v2 / SdkType"}, + {"id": "e313-7", "source": "n313-rtv2", "target": "n313-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e313-8", "source": "n313-rtv1", "target": "n313-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke", "dashed": True}, + ], + "flows": [ + { + "id": "init", + "title": "Cold Start, Runtime Discovery & protos Injection", + "summary": "Proxy switches to customer deps, resolves Agent Runtime vs V2 vs V1, and injects its `protos` module once.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "router", "label": "proxy_worker"}, + {"id": "ext", "label": "python-extensions"}, + {"id": "rt", "label": "Runtime v2/v1"}, + ], + "steps": [ + {"from": "host", "to": "router", "message": "StartStream + WorkerInitRequest(dir)", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["e313-1", "e313-2"], "payload": "uvloop + grpc_local-thread active"}, + {"from": "router", "to": "ext", "message": "Check PYTHON_ENABLE_AGENT_RUNTIME entry_points", "activeNodes": ["n313-router", "n313-ext"], "activeEdges": ["e313-5"], "payload": "group='azurefunctions.runtimes'"}, + {"from": "router", "to": "rt", "message": "Import V2/V1 + await worker_init_request(WorkerRequest)", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["e313-3"], "payload": "properties={'protos': protos, 'host': host}"}, + {"from": "rt", "to": "host", "message": "Cache protos, index app, return WorkerInitResponse", "isReturn": True, "activeNodes": ["n313-rtv2", "n313-lib", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["e313-7", "e313-3", "e313-2", "e313-1"], "payload": "Capabilities negotiated"}, + ], + }, + { + "id": "invoke", + "title": "Standard & HTTP-v2 Invocation", + "summary": "Proxy stamps invocation_id and delegates `WorkerRequest` to runtime v2 (`python-library` + `python-extensions`).", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "router", "label": "proxy_worker"}, + {"id": "rt", "label": "Runtime v2"}, + {"id": "lib", "label": "SDK / Extensions / App"}, + ], + "steps": [ + {"from": "host", "to": "router", "message": "InvocationRequest(invocation_id, function_id)", "activeNodes": ["n313-proto", "n313-grpc", "n313-router"], "activeEdges": ["e313-1", "e313-2"], "payload": "ContextEnabledTask stamps invocation_id"}, + {"from": "router", "to": "rt", "message": "await _library_worker.invocation_request(req)", "activeNodes": ["n313-router", "n313-rtv2"], "activeEdges": ["e313-3"], "payload": "Uses cached protos"}, + {"from": "rt", "to": "lib", "message": "Decode Datum / SdkType / WebApp + invoke user fn", "activeNodes": ["n313-rtv2", "n313-ext", "n313-lib"], "activeEdges": ["e313-6", "e313-7"], "payload": "Sync threadpool or async loop"}, + {"from": "lib", "to": "host", "message": "InvocationResponse -> _grpc_resp_queue", "isReturn": True, "activeNodes": ["n313-lib", "n313-rtv2", "n313-router", "n313-grpc", "n313-proto"], "activeEdges": ["e313-7", "e313-3", "e313-2", "e313-1"], "payload": "StreamingMessage sent over grpcio"}, + ], + }, + ], + }, + # ===================================================================== + # ERA 3: Python 3.15+ (R2P2 — Rust to Python Proxy) + # ===================================================================== + { + "id": "py315_r2p2", + "shortTitle": "Python 3.15+ (R2P2)", + "title": "R2P2 — Rust Transport + Embedded CPython via PyO3 (3.15+)", + "pythonVersions": "3.15+ (GIL Build)", + "packagePath": "workers/r2p2 + runtimes/{v1,v2}", + "launcherPath": "workers/r2p2 (compiled native Rust binary)", + "status": "Next-Gen Architecture (Python 3.15+)", + "tagline": "Native Rust (`tonic` + `prost` + `tokio`) embedding CPython via `PyO3` — zero Python `grpcio`/`protobuf`.", + "summary": ( + "R2P2 replaces the Python proxy worker with a native Rust binary that compiles " + "`FunctionRpc.proto` via `prost`/`tonic` and embeds `libpython` (GIL) via `PyO3`. " + "Standard `python-library` `Datum` invocations take a **Zero-Protobuf Hot Path** " + "(`convert.rs` tuples -> `native_invocation.py`), while control-plane and " + "`python-extensions` (`SdkType` deferred / `WebApp` HTTP-v2) calls route through " + "`control.rs` + `protos_adapter.py`." + ), + "tradeoffs": [ + {"kind": "strength", "text": "Day-1 Python 3.15+ support: zero wait on `grpcio` or `google.protobuf` C-extension wheels."}, + {"kind": "strength", "text": "Native `tonic`/`prost` wire serialization + direct PyO3 datum-tuple invocation on the GIL build."}, + {"kind": "strength", "text": "Complete transport isolation: zero `google.protobuf` or `grpcio` loaded in `sys.modules`."}, + {"kind": "pain", "text": "Dual FFI paths (`invoke_native` hot path vs `handle_invocation_control` for HTTP-v2 / `SdkType` extensions)."}, + ], + "erasDiff": { + "transport": "Native Rust: tonic + prost + tokio multi-thread + BytesCodec", + "boundary": "PyO3 FFI (GIL): Hot `(name, Datum)` tuples + Control `PyDict` + `LogSink`", + "runtimeLoading": "Unchanged `runtimes/v2,v1` fed `protos_adapter.py` + `native_invocation.py`", + "hotPathOverhead": "Rust prost -> PyO3 tuple -> native_invocation.py -> python-library Datum (0 protobuf)", + }, + "containers": [ + {"id": "c315-host", "label": "Functions Host (.NET — azure-functions-host)", "parentId": None, "tone": "host", "tier": 0}, + {"id": "c315-wire", "label": "Wire Contract (worker-protobuf)", "parentId": None, "tone": "proto", "tier": 0}, + {"id": "c315-proc", "label": "R2P2 Worker Process (Rust Binary + Embedded CPython 3.15)", "parentId": None, "tone": "rust-proc", "tier": 1}, + {"id": "c315-rust", "label": "Layer 1: Native Rust Shell (workers/r2p2/src)", "parentId": "c315-proc", "tone": "rust"}, + {"id": "c315-ffi", "label": "PyO3 FFI Thin Waist (GIL)", "parentId": "c315-proc", "tone": "bridge"}, + {"id": "c315-py", "label": "Layer 2: Embedded Python Bridge & Runtime", "parentId": "c315-proc", "tone": "runtime"}, + {"id": "c315-lib", "label": "Azure/azure-functions-python-library", "parentId": None, "tone": "lib", "tier": 2}, + {"id": "c315-ext", "label": "Azure/azure-functions-python-extensions", "parentId": None, "tone": "ext", "tier": 2}, + ], + "nodes": [ + { + "id": "n315-host", + "repo": "functions-host", + "containerId": "c315-host", + "label": "Script Host (WorkerChannel)", + "badge": "WebJobs.Script.Grpc", + "role": ".NET Functions Host runtime spawning the compiled `r2p2` binary with zero Host-side protocol changes.", + "file": "src/WebJobs.Script.Grpc/Channel/WorkerChannel.cs", + "line": 1, + "details": [ + "Speaks standard `ServerDuplexChannel` HTTP/2 gRPC to R2P2.", + ], + }, + { + "id": "n315-proto", + "repo": "language-worker-protobuf", + "containerId": "c315-wire", + "label": "FunctionRpc.proto Contract", + "badge": "prost-build (build.rs)", + "role": "Shared Host ↔ Worker gRPC contract compiled at Rust build time into native `prost` structs (`pb.rs`).", + "file": "src/proto/FunctionRpc.proto", + "line": 1, + "details": [ + "Vendored in `workers/r2p2/proto/` and compiled by `build.rs` with zero system `protoc` dependency.", + ], + }, + { + "id": "n315-tonic", + "repo": "python-worker", + "containerId": "c315-rust", + "label": "Tokio + Tonic EventStream", + "badge": "main.rs + BytesCodec", + "role": "Multi-thread tokio pump dialing Host (`TCP_NODELAY`), spawning concurrent tasks, and draining `mpsc`.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "main", 230), + "details": [ + "Shared lock-free `mpsc::unbounded_channel` carries both responses and Python `LogSink` frames.", + ], + }, + { + "id": "n315-router", + "repo": "python-worker", + "containerId": "c315-rust", + "label": "Prost Router & Status Fast-Path", + "badge": "process_message()", + "role": "Answers `WorkerStatus` in pure Rust (0 GIL) and splits Hot vs Control invocations via `control_path_cache`.", + "file": "workers/r2p2/src/main.rs", + "line": sym_line(r2p2_main, "process_message", 193), + "details": [ + "Memoizes `needs_control_path(function_id)` once per function after `FunctionLoad`.", + ], + }, + { + "id": "n315-hot-ffi", + "repo": "python-worker", + "containerId": "c315-ffi", + "label": "Hot FFI: bridge::invoke + convert.rs", + "badge": "PyO3 Datum Tuples", + "role": "Translates prost `TypedData` directly into `(tag, value)` Python tuples across `Python::attach`.", + "file": "workers/r2p2/src/bridge.rs", + "line": sym_line(r2p2_bridge_rs, "invoke", 261), + "details": [ + "Eliminates all Python protobuf object allocation on standard HTTP/Queue/Timer/Blob invocations.", + ], + }, + { + "id": "n315-ctrl-ffi", + "repo": "python-worker", + "containerId": "c315-ffi", + "label": "Control FFI: bridge::handle + LogSink", + "badge": "control.rs + #[pyclass] LogSink", + "role": "Converts Init/Load/Reload & HTTP-v2/deferred requests to `PyDict`s; exposes non-blocking Rust `LogSink`.", + "file": "workers/r2p2/src/bridge.rs", + "line": sym_line(r2p2_bridge_rs, "handle", 107), + "details": [ + "`LogSink.emit_log(dict)` encodes `RpcLog` in Rust so Python logging never touches protobuf or blocks.", + ], + }, + { + "id": "n315-bridge-py", + "repo": "python-worker", + "containerId": "c315-py", + "label": "bridge.py & protos_adapter.py", + "badge": "Protobuf-Free + Namespace Dep Iso", + "role": "Pure-Python `protos` stand-in + `_clear_modules_from_path` namespace-safe customer dependency promotion.", + "file": "workers/r2p2/bridge/bridge.py", + "line": sym_line(r2p2_bridge_py, "_clear_modules_from_path", 328), + "details": [ + "Preserves shared namespace parents while evicting worker-backed packages and stale parent attributes.", + ], + }, + { + "id": "n315-native-py", + "repo": "python-worker", + "containerId": "c315-py", + "label": "native_invocation.py (v2 & v1)", + "badge": "Hot Datum Execution Seam", + "role": "Decodes Rust datum tuples directly into `python-library` bindings using `_InvocShim` and runs user code.", + "file": "runtimes/v2/azure_functions_runtime/native_invocation.py", + "line": sym_line(rt_v2_native, "_decode_inputs", 82), + "details": [ + "Supports sync (`ThreadPoolExecutor`) & async (`asyncio`), `Context`, `Out[T]`, and W3C OpenTelemetry.", + ], + }, + { + "id": "n315-rt", + "repo": "python-worker", + "containerId": "c315-py", + "label": "Runtime handle_event + http_v2", + "badge": "Control & Streaming Seam", + "role": "Handles `WorkerInit`, `FunctionLoad`, `EnvironmentReload`, and fallback invocations for Extensions.", + "file": "runtimes/v2/azure_functions_runtime/handle_event.py", + "line": sym_line(rt_v2_event, "worker_init_request", 58), + "details": [ + "Operates seamlessly against `protos_adapter.py` without code duplication.", + ], + }, + { + "id": "n315-lib", + "repo": "python-library", + "containerId": "c315-lib", + "label": "azure-functions (Datum + Decorators)", + "badge": "Hot-Path Compatible", + "role": "Standard V2/V1 triggers & bindings (`HttpRequest`, `QueueMessage`, `TimerRequest`, `Out[T]`) + User App.", + "file": "azure/functions/meta.py", + "line": 1, + "details": [ + "Runs at full native speed via `native_invocation.py` with zero protobuf dependency!", + ], + }, + { + "id": "n315-ext", + "repo": "python-extensions", + "containerId": "c315-ext", + "label": "extensions (HTTP-v2 FastAPI & SdkType)", + "badge": "requires_control_path = true", + "role": "`WebApp` ASGI streaming (`http-fastapi`) and `SdkType` deferred bindings (`BlobClient`, `ServiceBus`).", + "file": "azurefunctions-extensions-base/azurefunctions/extensions/base/web.py", + "line": 1, + "details": [ + "Detected by `bridge.requires_control_path(fn_id)` and routed cleanly via `handle_invocation_control`.", + ], + }, + ], + "edges": [ + {"id": "e315-0", "source": "n315-host", "target": "n315-proto", "kind": "grpc-wire", "label": "ServerDuplexChannel"}, + {"id": "e315-1", "source": "n315-proto", "target": "n315-tonic", "kind": "grpc-wire", "label": "EventStream (tonic + BytesCodec)"}, + {"id": "e315-2", "source": "n315-tonic", "target": "n315-router", "kind": "thread-queue", "label": "tokio::spawn + mpsc"}, + {"id": "e315-3", "source": "n315-router", "target": "n315-hot-ffi", "kind": "pyo3-hot", "label": "Hot Path: bridge::invoke()"}, + {"id": "e315-4", "source": "n315-router", "target": "n315-ctrl-ffi", "kind": "pyo3-control", "label": "Control / Extensions Path"}, + {"id": "e315-5", "source": "n315-hot-ffi", "target": "n315-native-py", "kind": "pyo3-hot", "label": "invoke_native(datum tuples)"}, + {"id": "e315-6", "source": "n315-ctrl-ffi", "target": "n315-bridge-py", "kind": "pyo3-control", "label": "handle_control(PyDict)"}, + {"id": "e315-7", "source": "n315-bridge-py", "target": "n315-rt", "kind": "inproc-contract", "label": "WorkerRequest + protos_adapter"}, + {"id": "e315-8", "source": "n315-native-py", "target": "n315-lib", "kind": "exec-dispatch", "label": "Datum decode + invoke"}, + {"id": "e315-9", "source": "n315-rt", "target": "n315-ext", "kind": "pyo3-control", "label": "WebApp stream / SdkType", "dashed": True}, + {"id": "e315-10", "source": "n315-rt", "target": "n315-lib", "kind": "inproc-contract", "label": "index @app decorators", "dashed": True}, + ], + "flows": [ + { + "id": "hot_invoke", + "title": "Hot Native Invocation (prost -> PyO3 Tuple -> python-library Datum)", + "summary": "Standard triggers bypass Python protobuf completely: prost decodes TypedData into PyO3 datum tuples for native_invocation.py.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "rust", "label": "R2P2 (tonic+prost)"}, + {"id": "ffi", "label": "bridge::invoke"}, + {"id": "native", "label": "native_invocation.py"}, + {"id": "lib", "label": "python-library + App"}, + ], + "steps": [ + {"from": "host", "to": "rust", "message": "StreamingMessage(InvocationRequest) bytes", "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], "activeEdges": ["e315-1", "e315-2"], "payload": "needs_control_path(fn_id) == false (cached)"}, + {"from": "rust", "to": "ffi", "message": "prost decode -> convert::typed_data_to_tuple", "activeNodes": ["n315-router", "n315-hot-ffi"], "activeEdges": ["e315-3"], "payload": "Rust TypedData -> compact PyTuple"}, + {"from": "ffi", "to": "native", "message": "Python::attach -> invoke_native(fn_id, inputs, meta)", "activeNodes": ["n315-hot-ffi", "n315-native-py"], "activeEdges": ["e315-5"], "payload": "Zero Python protobuf objects created"}, + {"from": "native", "to": "lib", "message": "Decode azure.functions Datum + execute handler", "activeNodes": ["n315-native-py", "n315-lib"], "activeEdges": ["e315-8"], "payload": "HttpRequest / QueueMessage -> User fn"}, + {"from": "lib", "to": "host", "message": "Return tuple -> prost encode -> mpsc tx", "isReturn": True, "activeNodes": ["n315-lib", "n315-native-py", "n315-hot-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-8", "e315-5", "e315-3", "e315-2", "e315-1"], "payload": "StreamingMessage(InvocationResponse) bytes"}, + ], + }, + { + "id": "ext_invoke", + "title": "Extensions Path (HTTP-v2 FastAPI Streaming & SdkType Bindings)", + "summary": "Functions using `python-extensions` (`WebApp` HTTP streaming or `SdkType` BlobClient) route through `handle_invocation_control`.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "rust", "label": "R2P2 Router"}, + {"id": "ctrl", "label": "bridge::handle + bridge.py"}, + {"id": "rt", "label": "Runtime + http_v2"}, + {"id": "ext", "label": "python-extensions"}, + ], + "steps": [ + {"from": "host", "to": "rust", "message": "InvocationRequest(function_id)", "activeNodes": ["n315-proto", "n315-tonic", "n315-router"], "activeEdges": ["e315-1", "e315-2"], "payload": "needs_control_path(fn_id) == true"}, + {"from": "rust", "to": "ctrl", "message": "control::invocation_request_to_py(PyDict)", "activeNodes": ["n315-router", "n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["e315-4", "e315-6"], "payload": "protos_adapter wraps dict (still 0 google.protobuf)"}, + {"from": "ctrl", "to": "rt", "message": "await handle_event.invocation_request()", "activeNodes": ["n315-bridge-py", "n315-rt"], "activeEdges": ["e315-7"], "payload": "Standard runtime invocation path"}, + {"from": "rt", "to": "ext", "message": "Stream ASGI FastAPI chunks / hydrate SdkType", "activeNodes": ["n315-rt", "n315-ext"], "activeEdges": ["e315-9"], "payload": "azurefunctions.extensions.base"}, + {"from": "ext", "to": "host", "message": "Dict -> py_to_invocation_response -> prost bytes", "isReturn": True, "activeNodes": ["n315-ext", "n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-9", "e315-7", "e315-6", "e315-4", "e315-1"], "payload": "StreamingMessage(InvocationResponse)"}, + ], + }, + { + "id": "env_reload", + "title": "Placeholder Specialization & Namespace-Safe Dep Promotion", + "summary": "Evicts worker-backed modules while keeping shared `azure.*` / `azurefunctions.*` namespace parents attached.", + "actors": [ + {"id": "host", "label": "Host (FunctionRpc)"}, + {"id": "ctrl", "label": "bridge::handle"}, + {"id": "bpy", "label": "bridge.py Dep Engine"}, + {"id": "rt", "label": "Runtime + SDK"}, + ], + "steps": [ + {"from": "host", "to": "ctrl", "message": "FunctionEnvironmentReloadRequest(app_dir)", "activeNodes": ["n315-proto", "n315-tonic", "n315-router", "n315-ctrl-ffi"], "activeEdges": ["e315-1", "e315-2", "e315-4"], "payload": "Warm placeholder specializes to customer app"}, + {"from": "ctrl", "to": "bpy", "message": "_clear_modules_from_path + _prioritize_customer_deps", "activeNodes": ["n315-ctrl-ffi", "n315-bridge-py"], "activeEdges": ["e315-6"], "payload": "Retains namespace parents (`azure`, `azurefunctions`); evicts stale worker children"}, + {"from": "bpy", "to": "rt", "message": "Import customer-pinned SDK/runtime & index app", "activeNodes": ["n315-bridge-py", "n315-rt", "n315-lib"], "activeEdges": ["e315-7", "e315-10"], "payload": "sys.path: [customer_pkgs, worker_deps, app_dir]"}, + {"from": "rt", "to": "host", "message": "FunctionEnvironmentReloadResponse", "isReturn": True, "activeNodes": ["n315-rt", "n315-ctrl-ffi", "n315-tonic", "n315-proto"], "activeEdges": ["e315-7", "e315-6", "e315-4", "e315-1"], "payload": "Specialized & ready"}, + ], + }, + ], + }, + ] + + phases_by_era = { + "py312": get_py312_phases(), + "py313_314": get_py313_phases(), + "py315_r2p2": get_py315_phases(), + } + + for era in eras: + for node in era["nodes"]: + fmeta: dict[str, Any] = {} + if node["repo"] == "python-worker": + fmeta = extracted.get(node["file"], {}) + elif node["repo"] in repo_dirs and node["file"].endswith(".py"): + fmeta = extract_python_file(node["file"], repo_dirs[node["repo"]]) + node["fileLines"] = fmeta.get("lines", 0) + node["extractedSymbols"] = fmeta.get("symbols", [])[:10] + node["routerCases"] = fmeta.get("routerCases", []) + node["pyo3Calls"] = fmeta.get("pyo3Calls", []) + node["githubUrl"] = resolve_github_url(node, repo_refs) + era["topologyMermaid"] = compile_topology_mermaid(era) + for flow in era["flows"]: + flow["sequenceMermaid"] = compile_sequence_mermaid(flow) + + era_phases = phases_by_era.get(era["id"], []) + prefix_map = {"py312": "312", "py313_314": "313", "py315_r2p2": "315"} + pfx = prefix_map.get(era["id"]) + for ph in era_phases: + if pfx: + host_nid = f"n{pfx}-host" + proto_nid = f"n{pfx}-proto" + host_eid = f"p{pfx}-{ph['id']}-host" + is_reverse_log = ph["id"] == "worker_logs" + host_edge = { + "id": host_eid, + "source": proto_nid if is_reverse_log else host_nid, + "target": host_nid if is_reverse_log else proto_nid, + "kind": "grpc-wire", + "label": "AppInsights / Kusto" if is_reverse_log else "WorkerChannel", + } + ph["topologyEdges"] = [host_edge, *ph["topologyEdges"]] + if proto_nid in ph["activeNodes"] and host_nid not in ph["activeNodes"]: + ph["activeNodes"] = [host_nid, *ph["activeNodes"]] + for st in ph["steps"]: + if proto_nid in st["activeNodes"] and host_nid not in st["activeNodes"]: + st["activeNodes"] = [host_nid, *st["activeNodes"]] + st["activeEdges"] = [host_eid, *st["activeEdges"]] + ph["topologyMermaid"] = compile_topology_mermaid(era, ph["topologyEdges"]) + ph["sequenceMermaid"] = compile_sequence_mermaid(ph) + era["phases"] = era_phases + + structural_payload = json.dumps( + { + "repos": list(REPOS.keys()), + "phases": [p["id"] for p in PHASE_CATALOG], + "inventories": {k: [x["path"] for x in v] for k, v in inventories.items()}, + "protoMessages": proto_messages, + "symbols": {k: [s["name"] for s in v.get("symbols", [])] for k, v in sorted(extracted.items())}, + }, + sort_keys=True, + ) + fingerprint = hashlib.sha256(structural_payload.encode("utf-8")).hexdigest()[:16] + + graph = { + "schemaVersion": "1.2.0", + "generatedAt": datetime.now(timezone.utc).isoformat(), + "structuralFingerprint": fingerprint, + "git": git, + "repoRefs": repo_refs, + "repos": REPOS, + "phaseCatalog": PHASE_CATALOG, + "protoStreamingMessages": proto_messages, + "packageInventories": inventories, + "eras": eras, + } + validate_graph(graph) + return graph + + +def validate_graph(graph: dict[str, Any]) -> None: + expected_phase_ids = [p["id"] for p in PHASE_CATALOG] + for era in graph["eras"]: + node_ids = {n["id"] for n in era["nodes"]} + container_ids = {c["id"] for c in era["containers"]} + for n in era["nodes"]: + if n["containerId"] not in container_ids: + raise ValueError(f"Node {n['id']} references missing container {n['containerId']}") + if n["repo"] not in REPOS: + raise ValueError(f"Node {n['id']} references unknown repo {n['repo']}") + if n["repo"] == "python-worker" and not (REPO_ROOT / n["file"]).exists(): + raise ValueError(f"Node {n['id']} points to missing python-worker file {n['file']}") + for e in era["edges"]: + if e["source"] not in node_ids or e["target"] not in node_ids: + raise ValueError(f"Edge {e['id']} in {era['id']} has invalid endpoint") + + if era["id"] in ("py312", "py313_314", "py315_r2p2"): + got_phases = [p["id"] for p in era.get("phases", [])] + if got_phases != expected_phase_ids: + raise ValueError(f"Era {era['id']} phases {got_phases} != {expected_phase_ids}") + for ph in era["phases"]: + ph_edge_ids = {e["id"] for e in ph["topologyEdges"]} + for e in ph["topologyEdges"]: + if e["source"] not in node_ids or e["target"] not in node_ids: + raise ValueError(f"Phase edge {e['id']} in {era['id']}.{ph['id']} invalid") + for nid in ph["activeNodes"]: + if nid not in node_ids: + raise ValueError(f"Phase {era['id']}.{ph['id']} activeNode {nid} missing") + actor_ids = {a["id"] for a in ph["actors"]} + for st in ph["steps"]: + if st["from"] not in actor_ids or st["to"] not in actor_ids: + raise ValueError(f"Step actor missing in {era['id']}.{ph['id']}") + for nid in st["activeNodes"]: + if nid not in node_ids: + raise ValueError(f"Step activeNode {nid} missing in {era['id']}.{ph['id']}") + for eid in st["activeEdges"]: + if eid not in ph_edge_ids: + raise ValueError(f"Step activeEdge {eid} missing in {era['id']}.{ph['id']}") + + +def _stable_repo_url(node: dict[str, Any]) -> str: + repo = REPOS[node["repo"]] + line_sfx = f"#L{node['line']}" if node.get("line", 0) > 1 else "" + return f"{repo['url']}/blob/{repo['defaultBranch']}/{node['file']}{line_sfx}" + + +def _render_era_components_md(era: dict[str, Any]) -> str: + lines = [ + "| Component | Repository | Source Anchor | Responsibility |", + "| --- | --- | --- | --- |", + ] + for n in era["nodes"]: + repo = REPOS[n["repo"]] + anchor_lbl = f"`{n['file']}" + (f":L{n['line']}`" if n.get("line", 0) > 1 else "`") + url = _stable_repo_url(n) + role = n["role"].replace("|", "\\|") + lines.append(f"| **{n['label']}** | `{repo['shortName']}` | [{anchor_lbl}]({url}) | {role} |") + return "\n".join(lines) + + +def _render_era_phases_md(era: dict[str, Any]) -> str: + blocks: list[str] = [] + for ph in era.get("phases", []): + blocks.append(f"### {ph['title']}\n") + blocks.append(f"- **Wire Protocol:** `{ph['proto']}`") + blocks.append(f"- **Execution Scope:** {ph['scope']}\n") + blocks.append(f"{ph['summary']}\n") + blocks.append(f"> **Bypassed / Idle Note:** {ph['bypassedNote']}\n") + blocks.append("#### Sequence Diagram\n") + blocks.append(f"```mermaid\n{ph['sequenceMermaid']}\n```\n") + blocks.append("#### Phase Compound Topology\n") + blocks.append(f"```mermaid\n{ph['topologyMermaid']}\n```") + return "\n\n---\n\n".join(blocks) + + +def render_markdown_blocks(graph: dict[str, Any]) -> dict[str, str]: + eras_by_id = {e["id"]: e for e in graph["eras"]} + eco = eras_by_id["ecosystem"] + + repo_rows = [ + "| Repository | PyPI Packages | Architectural Role |", + "| --- | --- | --- |", + ] + for r in graph["repos"].values(): + pypi = ", ".join(f"`{p}`" for p in r["pypi"]) if r["pypi"] else "—" + repo_rows.append(f"| [`{r['fullName']}`]({r['url']}) | {pypi} | {r['role']} |") + + eco_flow = eco["flows"][0] + eco_md = "\n\n".join([ + "\n".join(repo_rows), + "### Full 5-Repository Topology\n\n```mermaid\n" + eco["topologyMermaid"] + "\n```", + f"### {eco_flow['title']}\n\n```mermaid\n{eco_flow['sequenceMermaid']}\n```", + "### Key Cross-Repository Components\n\n" + _render_era_components_md(eco), + ]) + + out: dict[str, str] = { + "ecosystem-reference": eco_md, + } + for eid in ("py312", "py313_314", "py315_r2p2"): + era = eras_by_id[eid] + out[f"{eid}-components"] = ( + "### Full Static Topology (Mermaid)\n\n" + f"```mermaid\n{era['topologyMermaid']}\n```\n\n" + "### Component & Source Reference\n\n" + + _render_era_components_md(era) + ) + out[f"{eid}-phases"] = _render_era_phases_md(era) + return out + + +def sync_markdown_docs( + graph: dict[str, Any], docs_dir: Path = DOCS_DIR, check_only: bool = False +) -> list[str]: + """Update or verify `` blocks in `docs/**/*.md`.""" + blocks = render_markdown_blocks(graph) + marker_re = re.compile( + r"(\n)(.*?)()", + flags=re.S, + ) + stale_or_updated: list[str] = [] + + for md_path in sorted(docs_dir.rglob("*.md")): + original = md_path.read_text(encoding="utf-8") + if "