feat: align native camera horizontal field of view

docs/component-reference.md

@@ -120,3 +120,12 @@ For exportable motion switching, prefer the compiled declaration:
 Motion. The single-controller rule resets to frame zero with root body channel
 zero and native joint ordering; it supports editor, standalone browser and native
 execution without Python callbacks. It cannot be combined with `action(modify=...)`.
+
+### Independent camera field of view
+
+A `Sensor` preset/config accepts optional `horizontalFov` in degrees (finite,
+strictly between 0 and 180). `fov` remains the vertical FoV of the full sensor.
+Omitted/null `horizontalFov` preserves the aspect-derived horizontal view.
+For example, `fov: 58`, `horizontalFov: 87`, `fullWidth: 48`, `fullHeight: 27`
+calibrates the two axes independently, before applying `cropX`/`cropY` and
+`width`/`height`. Use a matching browser runtime for editor and published sites.

scripts/verify_wheel.py

@@ -186,7 +186,7 @@ def run(label: str, command: list[str], *, extra_env: dict[str, str] | None = No
         if (out / "browser-cache").exists():
             raise RuntimeError("compilation or native execution acquired a browser runtime")
         if not args.source_wheel:
-            run("native-viewer", [str(cli), "run", str(project), "--no-browser",
+            run("native-viewer", [str(cli), "run", str(project), "--native", "--no-browser",
                                   "--host", "127.0.0.1", "--port", "0"], extra_env=steps_env)
             if (out / "browser-cache").exists():
                 raise RuntimeError("the native viewer acquired a browser runtime")

src/botlab/_runtime/_display.py

@@ -117,22 +117,18 @@ def _handler(self, websocket):
             return
         scene = request.path == scene_path
         clients = self._scene_clients if scene else self._clients
-        if scene:
+        payload = (
+            self._scene_payload if scene
+            else self._text_payload({"type": "meta", **self._metadata})
+        )
+        # Initial message and registration must be atomic with broadcasts.
+        # A receiver may start stepping as soon as it receives meta or scene.
+        with self._lock:
             try:
-                websocket.send(self._scene_payload)
+                websocket.send(payload)
             except ConnectionClosed:
                 return
-            with self._lock:
-                clients.add(websocket)
-        else:
-            # Atomic with broadcasts: a client that received meta must already
-            # be registered before the stepping thread sends its next frame.
-            with self._lock:
-                try:
-                    websocket.send(self._text_payload({"type": "meta", **self._metadata}))
-                except ConnectionClosed:
-                    return
-                clients.add(websocket)
+            clients.add(websocket)
         try:
             try:
                 for _message in websocket:

src/botlab/_runtime/_observations.py

@@ -634,6 +634,12 @@ def read_camera_config(node_data):
     raw_xyaxes = source.get("cameraXyaxes")
     rotation_mode = "xyaxes" if not (raw_xyaxes is None or raw_xyaxes == "") else "euler"
 
+    horizontal_fov = None
+    if source.get("horizontalFov") is not None:
+        horizontal_fov = _positive_number_field(source, "horizontalFov")
+        if horizontal_fov >= 180:
+            raise _error("CameraObservation", "horizontalFov must be less than 180 degrees")
+
     config = {
         "outputMode": _enum_field(source, "outputMode", ("rgb", "depth", "rgbd")),
         "targetKind": _enum_field(source, "targetKind", ("body", "site")),
@@ -648,6 +654,7 @@ def read_camera_config(node_data):
         "cropX": crop_x,
         "cropY": crop_y,
         "fov": _positive_number_field(source, "fov"),
+        "horizontalFov": horizontal_fov,
         "near": near,
         "far": far,
         "depthScale": _positive_number_field(source, "depthScale"),
@@ -2389,16 +2396,11 @@ def _camera_scene_option(geom_groups):
 
 
 def _camera_render(renderer, model, data, config):
-    """Update the renderer scene for one parsed camera config.
-
-    The frustum mirrors ``CameraObservationPoseResolver.update``:
-    ``fov`` is the vertical FoV of the FULL sensor and the horizontal extent
-    follows the full-sensor aspect (``camera.aspect = fullWidth/fullHeight``
-    in TS). MuJoCo's ``setView`` derives the frustum halfwidth from the
-    viewport aspect when ``frustum_width`` is 0 — the renderer viewport IS
-    fullWidth x fullHeight, so leaving it 0 reproduces the TS projection; the
-    crop window is taken from the rendered full frame afterwards (TS applies
-    it via ``camera.setViewOffset`` before rendering at width x height).
+    """Render the full sensor, with an optional independent horizontal FoV.
+
+    A zero frustum_width preserves MuJoCo's viewport-derived horizontal view.
+    Otherwise MuJoCo uses this field as the frustum *halfwidth*. Cropping is
+    applied to the full frame later, matching the browser's setViewOffset.
     """
     position, forward, up = _camera_pose(model, data, config)
 
@@ -2419,6 +2421,9 @@ def _camera_render(renderer, model, data, config):
         model.vis.global_.fovy = previous_fovy
 
     half_height = config["near"] * math.tan(math.radians(config["fov"]) / 2)
+    horizontal_fov = config["horizontalFov"]
+    half_width = (0.0 if horizontal_fov is None else
+                  config["near"] * math.tan(math.radians(horizontal_fov) / 2))
     for side in (0, 1):
         view = renderer.scene.camera[side]
         view.pos[:] = position
@@ -2431,7 +2436,7 @@ def _camera_render(renderer, model, data, config):
         view.frustum_center = 0.0
         # 0 lets mjr_render derive the halfwidth from the viewport aspect
         # (the full-sensor aspect); a nonzero value pins it instead.
-        view.frustum_width = 0.0
+        view.frustum_width = half_width
         view.orthographic = 0
 
 

src/botlab/sdk/_resources.py

@@ -195,6 +195,14 @@ def __init__(self, signal, age_slot):
 
 
 class Sensor:
+    """Camera resource calibrated by preset/config.
+
+    ``fov`` is the full sensor's vertical field of view in degrees. Optional
+    ``horizontalFov`` (0 < value < 180) independently sets its horizontal view;
+    omitted/None preserves the aspect-derived view. Cropping does not change
+    either calibration. Both browser and native runtimes use these fields.
+    """
+
     def __init__(self, source, *, preset=None, config=None, name=None):
         self.id = _declare_resource(
             "sensor",

tests/runtime/test_camera_projection.py

@@ -0,0 +1,108 @@
+"""Shared pinhole golden rays and real rendered depth, without policy weights."""
+import json
+import math
+import subprocess
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import mujoco
+import numpy as np
+import pytest
+
+from botlab._runtime import _observations as obs
+
+GOLDEN = json.loads((Path(__file__).parents[1] / "fixtures/camera-projection.json").read_text())
+
+
+@pytest.mark.parametrize("value", [0, -1, 180, 181, float("nan"), float("inf"), "bad"])
+def test_invalid_horizontal_fov(value):
+    with pytest.raises(ValueError, match="horizontalFov"):
+        obs.read_camera_config({"horizontalFov": value})
+
+
+def test_optional_horizontal_fov():
+    assert obs.read_camera_config({})["horizontalFov"] is None
+    assert obs.read_camera_config({"horizontalFov": None})["horizontalFov"] is None
+
+
+def camera_scene(fixture):
+    # A tilted plane has an analytic depth at every ray. This also verifies
+    # MuJoCo actually consumes frustum_width, rather than only storing it.
+    angle = 0.3
+    model = mujoco.MjModel.from_xml_string(
+        f'<mujoco><worldbody><geom type="plane" size="10 10 .1" '
+        f'quat="{math.cos(angle / 2)} 0 {math.sin(angle / 2)} 0"/></worldbody></mujoco>'
+    )
+    data = mujoco.MjData(model)
+    mujoco.mj_forward(model, data)
+    config = obs.read_camera_config({**fixture["config"], "target": "0",
+        "cameraOffset": [0, 0, 2], "cameraOffsetFrame": "body",
+        "cameraXyaxes": [1, 0, 0, 0, 1, 0], "near": .05, "far": 100})
+    return model, data, config
+
+
+@pytest.mark.parametrize("fixture", GOLDEN["cases"], ids=lambda item: item["name"])
+def test_native_pixel_rays_without_gl(fixture):
+    model, data, config = camera_scene(fixture)
+    # Scene-camera fields are MuJoCo float32 storage, but constructing the
+    # scene and writing its projection does not need an OpenGL context.
+    renderer = SimpleNamespace(
+        scene=mujoco.MjvScene(model, maxgeom=10),
+        update_scene=lambda *args, **kwargs: None,
+    )
+    obs._camera_render(renderer, model, data, config)
+    view = renderer.scene.camera[0]
+    half_width = view.frustum_width or (view.frustum_top - view.frustum_bottom) / 2 * config["fullWidth"] / config["fullHeight"]
+    rays = []
+    for y in range(config["height"]):
+        for x in range(config["width"]):
+            rays.append([
+                (2 * (x + config["cropX"] + .5) / config["fullWidth"] - 1) * half_width / view.frustum_near,
+                (2 * (y + config["cropY"] + .5) / config["fullHeight"] - 1) * view.frustum_top / view.frustum_near,
+                1,
+            ])
+    np.testing.assert_allclose(rays, fixture["rays"], atol=1e-7, rtol=0)
+
+
+@pytest.mark.parametrize("fixture", GOLDEN["cases"], ids=lambda item: item["name"])
+def test_native_rendered_depth(fixture):
+    # A missing/broken GL context can abort native code, bypassing try/except
+    # and poisoning subsequent Renderer attempts. Keep GL outside pytest.
+    result = subprocess.run(
+        [sys.executable, __file__, fixture["name"]],
+        capture_output=True, text=True, timeout=30,
+    )
+    ready = "BOTLAB_RENDERER_READY" in result.stdout
+    if not ready and (result.returncode == 77 or result.returncode < 0):
+        pytest.skip(
+            f"MuJoCo GL renderer unavailable (probe exit {result.returncode}); "
+            f"set MUJOCO_GL=egl. {result.stdout[-1000:]} {result.stderr[-1000:]}"
+        )
+    assert result.returncode == 0, result.stdout + result.stderr
+
+
+def render_depth_case(fixture):
+    model, data, config = camera_scene(fixture)
+    try:
+        renderer = mujoco.Renderer(model, width=config["fullWidth"], height=config["fullHeight"])
+    except Exception as error:
+        print(f"MuJoCo GL renderer unavailable: {error}", flush=True)
+        raise SystemExit(77) from error
+    print("BOTLAB_RENDERER_READY", flush=True)
+    with renderer:
+        obs._camera_render(renderer, model, data, config)
+        renderer.enable_depth_rendering()
+        depth = obs._render_depth_meters(renderer, model, config)
+        crop = depth[config["cropY"]:config["cropY"] + config["height"], config["cropX"]:config["cropX"] + config["width"]]
+        expected = 2 / (1 - math.tan(0.3) * np.asarray(fixture["rays"])[:, 0])
+        # macOS GL lacks ARB_clip_control; depth quantization at 2 m is ~60 um.
+        np.testing.assert_allclose(crop.reshape(-1), expected, atol=1e-4, rtol=0)
+
+
+if __name__ == "__main__":
+    # Do not leave large core dumps when a headless GL driver aborts.
+    if sys.platform != "win32":
+        import resource
+        resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
+    render_depth_case(next(case for case in GOLDEN["cases"] if case["name"] == sys.argv[1]))

tests/runtime/test_display.py

@@ -1,7 +1,9 @@
 """Local display server tests (tier B: needs websockets)."""
 
 import json
+import threading
 import unittest
+from types import SimpleNamespace
 
 import mujoco  # noqa: E402
 import numpy as np  # noqa: E402
@@ -136,6 +138,60 @@ def model_and_data():
         mujoco.mj_forward(model, data)
         return model, data
 
+    def test_first_frame_is_not_lost_during_initial_message_send(self):
+        for scene in (False, True):
+            with self.subTest(scene=scene):
+                self.assert_initial_message_keeps_first_frame(scene)
+
+    def assert_initial_message_keeps_first_frame(self, scene):
+        server = _display.DisplayServer(
+            {}, scene_provider=SimpleNamespace(
+                payload='{"type":"scene"}', frame_size=11, frame=lambda _: b"scene-frame",
+            ),
+        )
+        messages = []
+        errors = []
+        finished = threading.Event()
+
+        def broadcast():
+            try:
+                server.broadcast(0.5, np.zeros(1), np.zeros(1))
+            except Exception as error:
+                errors.append(error)
+            finally:
+                finished.set()
+
+        worker = threading.Thread(target=broadcast, daemon=True)
+        test = self
+
+        class Client:
+            request = SimpleNamespace(
+                path=f"/display/{server._token}" + ("?scene=1" if scene else ""),
+                headers={},
+            )
+
+            def send(self, payload):
+                messages.append(payload)
+                if isinstance(payload, str):
+                    # Model a receiver starting a step before send returns.
+                    # Without handshake locking, force that step to finish
+                    # before the handler can register this client.
+                    worker.start()
+                    if not server._lock.locked():
+                        test.assertTrue(finished.wait(5))
+
+            def __iter__(self):
+                test.assertTrue(finished.wait(5))
+                return iter(())
+
+        server._handler(Client())
+        worker.join(timeout=5)
+        self.assertFalse(worker.is_alive())
+        self.assertEqual(errors, [])
+        self.assertEqual(len(messages), 2)
+        self.assertIsInstance(messages[0], str)
+        self.assertIsInstance(messages[1], bytes)
+
     def test_scene_exports_parser_model_and_world_transforms(self):
         model, data = self.model_and_data()
         provider = MujocoSceneProvider(model, data, mujoco, "Test scene")