From d6e501d77c25e50352871e8adeec44de145c3c98 Mon Sep 17 00:00:00 2001 From: Paulo Date: Sun, 4 Oct 2026 18:40:26 +0200 Subject: [PATCH] DRU-720 -- Sync managed services into the vault --- backend/druks/api/server.py | 11 +- backend/druks/services/base.py | 82 ++++++-- backend/druks/services/exceptions.py | 9 + backend/druks/services/routes.py | 11 +- backend/druks/services/schemas.py | 7 + backend/druks/settings.py | 9 +- backend/tests/test_managed_services.py | 192 ++++++++++++++++++ docs/configuration.md | 34 +++- frontend/src/api/types.ts | 2 + .../src/components/ResourcePages.test.tsx | 2 + frontend/src/components/ServicesPane.test.tsx | 14 ++ frontend/src/components/SettingsPanes.tsx | 11 +- 12 files changed, 354 insertions(+), 30 deletions(-) create mode 100644 backend/tests/test_managed_services.py diff --git a/backend/druks/api/server.py b/backend/druks/api/server.py index f332bad7d..76f3bf90d 100644 --- a/backend/druks/api/server.py +++ b/backend/druks/api/server.py @@ -49,7 +49,8 @@ from druks.notifications.routes import router as notifications_router from druks.redis import close_client from druks.sandbox.routes import router as secrets_router -from druks.services.exceptions import OauthPageError, ServiceNotConnectedError +from druks.services.base import Service +from druks.services.exceptions import OauthPageError, ServiceManagedError, ServiceNotConnectedError from druks.services.routes import oauth_router from druks.services.routes import router as service_identities_router from druks.settings import Settings, ensure_data_dirs, load_settings, setup_logging @@ -97,6 +98,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # the check for drift that happens while running. async with session_scope(app.state.engine) as session: await resolve_single_operator(session) + # Before DBOS launches: a recovered workflow reads a card at once. + async with session_scope(app.state.engine) as session: + await Service.sync_configuration(session) # DBOS runs embedded here: this process both serves HTTP and executes # durable workflows. Tests pre-populate app.state.settings and never # reach here — they drive DBOS through their own fixtures. @@ -193,6 +197,11 @@ async def _service_not_connected_handler( return JSONResponse(status_code=409, content={"error": "HTTP_409", "detail": str(exc)}) +@app.exception_handler(ServiceManagedError) +async def _service_managed_handler(request: Request, exc: ServiceManagedError) -> JSONResponse: + return JSONResponse(status_code=409, content={"error": "HTTP_409", "detail": str(exc)}) + + # The connect and callback doors are reached by full-page browser navigation, # so a failure renders an operator page, not the JSON envelope every fetch gets. @app.exception_handler(OauthPageError) diff --git a/backend/druks/services/base.py b/backend/druks/services/base.py index 1c1ca9eea..e0148ef78 100644 --- a/backend/druks/services/base.py +++ b/backend/druks/services/base.py @@ -15,8 +15,14 @@ from druks.secrets.datastructures import Audience from druks.secrets.enums import SecretKind from druks.secrets.models import VaultSecret - -from .exceptions import OauthExchangeError, ServiceConnectError, ServiceNotConnectedError +from druks.settings import load_settings + +from .exceptions import ( + OauthExchangeError, + ServiceConnectError, + ServiceManagedError, + ServiceNotConnectedError, +) from .oauth import OauthClient, fetch_identity, is_grant_revoked # GoogleCalendar -> google_calendar, HTTPServer -> http_server. @@ -265,6 +271,57 @@ async def get(cls) -> VaultSecret: return row raise ServiceNotConnectedError(cls.slug) + @classmethod + def is_managed(cls) -> bool: + """Whether druks.toml configures this service. Its card then refuses a change.""" + return cls.slug in load_settings().services + + @classmethod + async def sync_configuration(cls, session: AsyncSession) -> None: + """Store each service that druks.toml configures as its card. An entry replaces + the credentials of a card that exists.""" + for slug, entry in load_settings().services.items(): + service = services.get(slug) + if not service: + raise ServiceConnectError( + f"druks.toml configures the service {slug!r}, and no installed app " + f"declares it. Remove [services.{slug}]." + ) + try: + settings = service.settings_model.model_validate(entry) + except ValidationError as error: + invalid_fields = ", ".join( + f"services.{slug}.{item['loc'][0]}" for item in error.errors() + ) + # The cause shows the values of the entry, and startup prints the traceback. + raise ServiceConnectError( + f"The entry [services.{slug}] is not valid. Correct {invalid_fields}." + ) from None + await service._store(session, settings, proven={}) + + @classmethod + async def _store(cls, session: AsyncSession, settings: BaseModel, proven: dict) -> VaultSecret: + """Store the card from its validated ``settings``: secrets encrypted, the rest as + identity facts beside the ``proven`` ones.""" + fields = cls.settings_model.model_fields + secrets = { + name: getattr(settings, name).get_secret_value() + for name, field in fields.items() + if field_kind(field) == "secret" + } + identity = { + name: getattr(settings, name) + for name, field in fields.items() + if field_kind(field) != "secret" + } + return await VaultSecret.store( + session, + cls.secret_kind, + Audience.service(cls.slug), + identity={**identity, **proven}, + secrets=secrets, + ) + @classmethod def with_scopes(cls, *scopes: str) -> ScopedService: """Declare this app's use of the service and the scopes its @@ -383,6 +440,8 @@ async def connect(cls, payload: dict[str, str]) -> VaultSecret: """Verify and store a paste of the service's fields: secrets encrypted, the rest as identity facts. On a connected card a blank secret keeps the stored one. Another client ID revokes the connections that belong to the old one.""" + if cls.is_managed(): + raise ServiceManagedError(cls.slug) fields = cls.settings_model.model_fields session = db_session() card = await VaultSecret.lookup(session, cls.secret_kind, Audience.service(cls.slug)) @@ -402,24 +461,7 @@ async def connect(cls, payload: dict[str, str]) -> VaultSecret: labels = {spec["name"]: spec["label"] for spec in cls.connect_fields()} missing = ", ".join(labels[err["loc"][0]] for err in error.errors()) raise ServiceConnectError(f"Enter {missing}.") from error - secrets = { - name: getattr(settings, name).get_secret_value() - for name, field in fields.items() - if field_kind(field) == "secret" - } - identity = { - name: getattr(settings, name) - for name, field in fields.items() - if field_kind(field) != "secret" - } - proven = await cls.verify(settings) - row = await VaultSecret.store( - session, - cls.secret_kind, - Audience.service(cls.slug), - identity={**identity, **proven}, - secrets=secrets, - ) + row = await cls._store(session, settings, proven=await cls.verify(settings)) if cls.token_endpoint and row.identity["client_id"] != previous_client_id: client = OauthClient(provider=cls.slug) for connection in await VaultSecret.list_connections( diff --git a/backend/druks/services/exceptions.py b/backend/druks/services/exceptions.py index 286dc6f1b..0eb89fa01 100644 --- a/backend/druks/services/exceptions.py +++ b/backend/druks/services/exceptions.py @@ -17,6 +17,15 @@ class ServiceConnectError(DruksError): and safe to show; it never quotes anything the operator pasted.""" +class ServiceManagedError(DruksError): + """Configuration owns the service's credentials.""" + + def __init__(self, service: str) -> None: + super().__init__( + f"{service} is managed by configuration. Change druks.toml and its secrets." + ) + + class OauthPageError(DruksError): """A failure on a browser-navigated OAuth door — the connect and callback routes, whose failures render an operator page instead of the JSON envelope.""" diff --git a/backend/druks/services/routes.py b/backend/druks/services/routes.py index 9143c5af6..8cb5d2874 100644 --- a/backend/druks/services/routes.py +++ b/backend/druks/services/routes.py @@ -13,10 +13,12 @@ OauthExchangeError, OauthPageError, ServiceConnectError, + ServiceManagedError, ServiceNotConnectedError, ) from druks.services.oauth import OauthClient, complete_connect from druks.services.schemas import ConnectionResponse, ServiceResponse +from druks.settings import load_settings from druks.signals import publish router = APIRouter(prefix="/api/services", tags=["services"]) @@ -26,6 +28,7 @@ @router.get("", response_model=list[ServiceResponse], response_model_by_alias=True) async def list_services(session: SessionDep) -> list[ServiceResponse]: entries = [] + managed_by = load_settings().managed_by account_connections = await VaultSecret.list_owned_by(session, current_account_id.get()) for service in services.all(): audience = Audience.service(service.slug) @@ -33,7 +36,11 @@ async def list_services(session: SessionDep) -> list[ServiceResponse]: connections = [ connection for connection in account_connections if connection.audience == audience ] - entries.append(ServiceResponse.from_row(service, row, connections)) + entries.append( + ServiceResponse.from_row( + service, row, connections, managed=service.is_managed(), managed_by=managed_by + ) + ) return entries @@ -60,6 +67,8 @@ async def disconnect_service(session: SessionDep, slug: str) -> None: service = services.get(slug) if not service: raise HTTPException(status_code=404, detail=f"No service {slug!r}.") + if service.is_managed(): + raise ServiceManagedError(slug) await service.disconnect(session) diff --git a/backend/druks/services/schemas.py b/backend/druks/services/schemas.py index 3243964fb..ea98bc545 100644 --- a/backend/druks/services/schemas.py +++ b/backend/druks/services/schemas.py @@ -42,6 +42,8 @@ class ServiceResponse(Schema): description: str required: bool connected: bool + managed: bool + managed_by: str facts: dict[str, Any] connected_at: datetime | None fields: list[ServiceFieldSpec] @@ -57,6 +59,9 @@ def from_row( service: "type[Service]", row: "VaultSecret | None", connections: "list[VaultSecret] | None" = None, + *, + managed: bool = False, + managed_by: str = "", ) -> "ServiceResponse": return cls( slug=service.slug, @@ -64,6 +69,8 @@ def from_row( description=service.description, required=service.required, connected=bool(row), + managed=managed, + managed_by=managed_by if managed else "", facts=row.identity if row else {}, connected_at=row.updated_at if row else None, fields=[ServiceFieldSpec(**spec) for spec in service.connect_fields()], diff --git a/backend/druks/settings.py b/backend/druks/settings.py index bfef615b8..6ad393119 100644 --- a/backend/druks/settings.py +++ b/backend/druks/settings.py @@ -147,7 +147,9 @@ def __call__(self) -> dict[str, Any]: class _SecretsSource(NestedSecretsSettingsSource): def __call__(self) -> dict[str, Any]: - return _pick(super().__call__(), Settings.secret_keys()) + # Each file in the table of a configured service is a secret of its card. + tables = tuple(f"services.{slug}" for slug in self.current_state.get("services", {})) + return _pick(super().__call__(), Settings.secret_keys() + tables) class Identity(BaseModel): @@ -245,9 +247,14 @@ class Settings(BaseSettings): ) timezone: Annotated[str, AfterValidator(validate_timezone)] = "UTC" + # Who manages this installation. The Services page names it on each managed card. + managed_by: str = "" identity: Identity = Identity() urls: Urls = Urls() sandbox: Sandbox = Sandbox() + # A [services.] table makes the card of that service managed. Each table holds + # its secrets as plain text, so no output shows the field. + services: dict[str, dict] = Field(default={}, repr=False, exclude=True) # Encrypts stored secrets at rest. A missing or malformed key refuses boot; # `druks setup` generates one. diff --git a/backend/tests/test_managed_services.py b/backend/tests/test_managed_services.py new file mode 100644 index 000000000..589b1580f --- /dev/null +++ b/backend/tests/test_managed_services.py @@ -0,0 +1,192 @@ +import json +import traceback + +import httpx +import pytest +from conftest import bind_ambient_session, connect_service +from druks.accounts.models import Account +from druks.chat.channels.whatsapp.services import Waha +from druks.secrets.datastructures import Audience +from druks.secrets.models import VaultSecret +from druks.services import Service +from druks.services.exceptions import ServiceConnectError, ServiceManagedError +from druks.settings import Settings +from druks.testing import asgi_client, configure_app_for_test +from pydantic import BaseModel, SecretStr, field_validator +from sqlalchemy import select + + +@pytest.fixture +def acme(declared_services): + class Acme(Service): + class Settings(BaseModel): + url: str + key: SecretStr + + @field_validator("key") + @classmethod + def valid_key(cls, value): + if value.get_secret_value() == "invalid-secret": + raise ValueError(f"Rejected {value.get_secret_value()}") + return value + + return Acme + + +@pytest.fixture +def configuration(tmp_path, monkeypatch): + path = tmp_path / "druks.toml" + path.write_text('managed_by = "Druks Cloud"\n') + monkeypatch.setenv("DRUKS_CONFIG", str(path)) + return path + + +@pytest.fixture +def secrets(tmp_path, monkeypatch): + path = tmp_path / "secrets" + path.mkdir() + monkeypatch.setenv("DRUKS_SECRETS_DIR", str(path)) + return path + + +def test_service_entry_takes_its_secrets_from_files(configuration, secrets): + configuration.write_text('[services.acme]\nurl = "https://acme.test"\n') + (secrets / "services.acme.key").write_text("acme-secret") + (secrets / "services.other.key").write_text("unused-secret") + + settings = Settings() + + assert settings.services == {"acme": {"url": "https://acme.test", "key": "acme-secret"}} + assert "acme-secret" not in repr(settings) + assert "acme-secret" not in settings.model_dump_json() + + +@pytest.mark.parametrize( + "body, secret, message", + [ + ('[services.acme]\nurl = "https://acme.test"', "invalid-secret", "services.acme.key"), + ('[services.acme]\nurl = ["sensitive-value"]', "safe-key", "services.acme.url"), + ("[services.acme]", "safe-key", "services.acme.url"), + ('[services.unknown]\nurl = "sensitive-value"', "safe-key", "no installed app"), + ], +) +async def test_invalid_service_entry_stops_the_sync_without_its_values( + acme, configuration, secrets, druks_db, caplog, body, secret, message +): + configuration.write_text(body) + (secrets / "services.acme.key").write_text(secret) + with pytest.raises(ServiceConnectError, match=message) as error: + await Service.sync_configuration(druks_db) + output = "".join(traceback.format_exception(error.value)) + caplog.text + assert "invalid-secret" not in output + assert "sensitive-value" not in output + + +async def test_sync_keeps_the_vault_id_and_replaces_credentials( + acme, configuration, secrets, druks_db +): + bind_ambient_session(druks_db) + original = await connect_service( + "acme", identity={"url": "https://old.test"}, secrets={"key": "old-key"} + ) + original_id = original.id + configuration.write_text('[services.acme]\nurl = "https://acme.test"') + (secrets / "services.acme.key").write_text("configured-key") + for _ in range(2): + await Service.sync_configuration(druks_db) + row = await acme.get() + assert row.id == original_id + assert row.identity == {"url": "https://acme.test"} + assert row.secrets == {"key": "configured-key"} + assert await row.issue_token("", name="acme_key") == ("configured-key", None) + + (secrets / "services.acme.key").write_text("rotated-key") + await Service.sync_configuration(druks_db) + assert (await acme.get()).id == original_id + assert (await acme.get()).secrets == {"key": "rotated-key"} + rows = list( + await druks_db.scalars( + select(VaultSecret).where(VaultSecret.audience == Audience.service("acme")) + ) + ) + assert len(rows) == 1 + + configuration.write_text("") + await Service.sync_configuration(druks_db) + assert (await acme.get()).secrets == {"key": "rotated-key"} + pasted = await acme.connect({"url": "https://self-hosted.test", "key": "pasted-key"}) + assert pasted.id == original_id + + +async def test_managed_card_is_read_only(acme, configuration, secrets, druks_db, tmp_path, caplog): + configuration.write_text( + 'managed_by = "Druks Cloud"\n[services.acme]\nurl = "https://acme.test"' + ) + (secrets / "services.acme.key").write_text("configured-key") + settings = Settings(data_dir=tmp_path) + bind_ambient_session(druks_db) + await Service.sync_configuration(druks_db) + api = configure_app_for_test(settings=settings) + async with asgi_client(api) as client: + response = await client.get("/api/services") + [card] = [card for card in response.json() if card["slug"] == "acme"] + assert card["connected"] and card["managed"] + assert card["managedBy"] == "Druks Cloud" + assert card["facts"] == {"url": "https://acme.test"} + assert "configured-key" not in response.text + for method, kwargs in ( + ("POST", {"json": {"url": "https://changed.test", "key": "other-key"}}), + ("DELETE", {}), + ): + response = await client.request(method, "/api/services/acme", **kwargs) + assert response.status_code == 409 + assert "configured-key" not in response.text + with pytest.raises(ServiceManagedError): + await acme.connect({"url": "https://changed.test", "key": "other-key"}) + assert (await acme.get()).secrets == {"key": "configured-key"} + assert "configured-key" not in caplog.text + + +async def test_waha_link_uses_the_synced_card_and_keeps_the_session_key( + configuration, secrets, monkeypatch, druks_db, caplog +): + configuration.write_text( + '[urls]\nendpoint = "https://instance.test"\n[services.waha]\nurl = "http://cloud.test/_instance/123/waha"' + ) + (secrets / "services.waha.key").write_text("instance-key") + bind_ambient_session(druks_db) + await Service.sync_configuration(druks_db) + owner = await Account.get_or_create(druks_db, "operator@example.com") + requests = [] + send = httpx.AsyncClient.send + + async def waha_send(client, request, **kwargs): + if request.url.host == "cloud.test": + requests.append(request) + body = {} + if request.url.path.endswith("/api/sessions"): + body = {"name": "123-session"} + elif request.url.path.endswith("/api/keys"): + body = {"id": "key-id", "key": "session-key"} + return httpx.Response(200, request=request, json=body) + return await send(client, request, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "send", waha_send) + connection = await Waha.link(druks_db, owner, identity={}) + + assert [request.method for request in requests] == ["POST", "POST", "PUT"] + assert [request.headers["X-Api-Key"] for request in requests] == [ + "instance-key", + "instance-key", + "session-key", + ] + assert connection.secrets["key"] == "session-key" + assert all(request.url.path.startswith("/_instance/123/waha/") for request in requests) + assert ( + json.loads(requests[2].content)["config"]["webhooks"][0]["hmac"]["key"] + == connection.secrets["webhook_secret"] + ) + assert not any( + secret in caplog.text + for secret in ("instance-key", "session-key", connection.secrets["webhook_secret"]) + ) diff --git a/docs/configuration.md b/docs/configuration.md index d6f230a19..a70464d46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,6 +41,7 @@ host-run development template for that environment plane. | --- | --- | | `[identity]` | Browser identity mode and header or JWT verification inputs | | `[urls]` | Dashboard callback base URL and public webhook hostname | +| `[services.]` | Service credentials managed by deployment configuration | | `[paths]` | Host data and harness configuration paths | | `[sandbox]` | Drukbox provider, service URL, image override, registry host and user name, and the proxy and issuer addresses | | `[sandbox.]` | Provider environment passed through to the remote stack | @@ -113,6 +114,32 @@ as files. Set `DRUKS_SECRETS_DIR` to the directory, for example a mounted `sandbox.service_token`. Druks does not start when a secret is in the environment and in a file. The installer does not use files. +## Managed services + +A `[services.]` table makes that service managed. `druks.toml` and the +secret files own its card, and the Services page cannot change or disconnect it. +The table is for a platform that writes `druks.toml` itself: the installer +refuses it. The table holds the plain fields of the card. Each secret field of +the card is a file in `DRUKS_SECRETS_DIR`: + +```toml +managed_by = "Druks Cloud" + +[services.waha] +url = "http://control.example/_instance//waha" +``` + +Here the key is the file `services.waha.key`. Druks does not start when a table +names an unknown service or holds an invalid field. + +`managed_by` names who manages the installation, and each managed card shows it: +**Managed by Druks Cloud**. Without a name, the card shows **Managed by +configuration**. + +Druks reads the tables at startup, so restart Druks after a change. When you +remove a table, the card keeps its credentials, and you can change it on the +Services page again. + ## Personal and installation settings One Druks installation serves one organization. Separate organizations use @@ -338,7 +365,7 @@ through the MCP endpoint. See Druks acts at GitHub as one **operator GitHub App**. This app is its service identity. The GitHub App receives webhooks and does domain writes such as branches, pull requests, comments, labels, and merges. Its credentials live encrypted -in Postgres. They do not come from TOML, the environment, or a PEM file. +in Postgres. Until an operator connects GitHub, agent runs stop with a direct message. `druks doctor` reports that no GitHub connection exists. @@ -517,6 +544,8 @@ session key. That is WAHA's admin key when you run WAHA yourself. Every other call uses the session key of the number. Druks checks neither value when you save the card, so a wrong key shows up when you link a number. +For deployment-managed WAHA, use [managed services](#managed-services). + Run WAHA with the NOWEB or GOWS engine: Druks takes each reply's message ID from WAHA before it sends the reply. Druks refuses a number that links on another engine. WAHA must reach this webhook URL: @@ -940,5 +969,4 @@ exchange. Rotate it as you rotate `secrets_key`, with the new key first. The envelope does **not** cover notification webhook URLs. Postgres stores them as ordinary fields, although the API masks their values. Treat access to Postgres and its backups as access to those values. GitHub App private keys, -the operator identity's and the review identity's, are vault rows, not files -mounted into the process. +the operator identity's and the review identity's, are encrypted vault rows. diff --git a/frontend/src/api/types.ts b/frontend/src/api/types.ts index 9a2302ea7..bbc745443 100644 --- a/frontend/src/api/types.ts +++ b/frontend/src/api/types.ts @@ -725,6 +725,8 @@ export interface Service { description: string required: boolean connected: boolean + managed: boolean + managedBy: string facts: Record connectedAt: string | null fields: ServiceField[] diff --git a/frontend/src/components/ResourcePages.test.tsx b/frontend/src/components/ResourcePages.test.tsx index 1d3f72254..2b2f3cd61 100644 --- a/frontend/src/components/ResourcePages.test.tsx +++ b/frontend/src/components/ResourcePages.test.tsx @@ -154,6 +154,8 @@ describe('Account grant groups', () => { vi.spyOn(api, 'services').mockResolvedValue([ { slug: 'gmail', + managed: false, + managedBy: '', title: 'Gmail', description: 'Connect mailboxes.', required: true, diff --git a/frontend/src/components/ServicesPane.test.tsx b/frontend/src/components/ServicesPane.test.tsx index d90cb238c..2590a1076 100644 --- a/frontend/src/components/ServicesPane.test.tsx +++ b/frontend/src/components/ServicesPane.test.tsx @@ -34,6 +34,8 @@ const disconnected: Service = { description: 'The GitHub App druks acts as.', required: true, connected: false, + managed: false, + managedBy: '', facts: {}, connectedAt: null, fields: githubFields, @@ -57,6 +59,8 @@ const pasteOnly: Service = { description: 'The OAuth client every mailbox authenticates against.', required: true, connected: false, + managed: false, + managedBy: '', facts: {}, connectedAt: null, fields: [ @@ -331,4 +335,14 @@ describe('ServicesPane', () => { 'https://github.com/apps/druks-operator/installations/new', ) }) + + it.each(['Druks Cloud', ''])('locks a managed service with manager %j', async (managedBy) => { + stubFetch([[{ ...pasteOnly, connected: true, managed: true, managedBy }]]) + renderPane() + + expect(await screen.findByText(`Managed by ${managedBy || 'configuration'}`)).toBeTruthy() + fireEvent.click(screen.getByRole('button', { name: `Configure ${pasteOnly.title}` })) + expect(screen.getByRole('button', { name: 'Replace connection' })).toHaveProperty('disabled', true) + expect(screen.getByRole('button', { name: 'Disconnect service' })).toHaveProperty('disabled', true) + }) }) diff --git a/frontend/src/components/SettingsPanes.tsx b/frontend/src/components/SettingsPanes.tsx index 1bbd3252b..f2ec2ad3d 100644 --- a/frontend/src/components/SettingsPanes.tsx +++ b/frontend/src/components/SettingsPanes.tsx @@ -822,7 +822,7 @@ export function ServicesPane() { @@ -855,6 +855,9 @@ export function ServicesPane() { ) } +const managedLabel = (service: Service) => + service.managed ? `Managed by ${service.managedBy || 'configuration'}` : undefined + function ServiceStatus({ connected, label }: { connected: boolean; label?: string }) { return ( @@ -941,7 +944,7 @@ function ServiceDetail({ service, onBack }: { service: Service; onBack: () => vo

{service.title}

- +
@@ -980,10 +983,10 @@ function ServiceDetail({ service, onBack }: { service: Service; onBack: () => vo )} {service.slug === 'slack' && createSlackApp} - -