GitHub through the proxy: git and gh

deploy/proxy/swap.py

@@ -42,10 +42,11 @@ class ExchangeUnavailable(Exception):
 
 
 def placeholder_in(value: str) -> str:
-    """Basic carries it as the password. Inside a longer value it never counts."""
+    """Bearer and gh's ``token`` carry it after the scheme, Basic as the
+    password. Inside a longer value it never counts."""
     scheme, _, rest = value.partition(" ")
     candidate = value
-    if scheme.lower() == "bearer":
+    if scheme.lower() in ("bearer", "token"):
         candidate = rest.strip()
     elif scheme.lower() == "basic":
         try:

docs/architecture.md

@@ -107,10 +107,13 @@ Caller `env` is stored for provisioning and never returned by the API;
 keys in `hosts.schemas.RESERVED_HOST_ENV_KEYS` are rejected.
 
 `POST /hosts` takes `secrets`, keyed by service handle. A built-in handle
-resolves through the catalog. A custom entry names its own `host` and
-`auth_variable`. It can also set `auth_header` and `auth_prefix`.
-The default is a bearer token in `Authorization`. Drukbox does not consult the
-catalog for a custom entry.
+resolves through the catalog, which names the variable a client reads and
+the hosts the service reaches. `github` reaches `api.github.com` and
+`uploads.github.com` with a bearer, and `github.com` with Basic and
+`x-access-token` as the user, since git's smart HTTP refuses a bearer. A
+custom entry names its own `host` and `auth_variable`. It can also set
+`auth_header` and `auth_prefix`. The default is a bearer token in
+`Authorization`. Drukbox does not consult the catalog for a custom entry.
 
 A static entry stores `value`. A refreshable entry stores `issuer`: the URL,
 the request headers, and the refresh interval. Drukbox never stores a fetched
@@ -127,7 +130,10 @@ fingerprint of the random part. The sandbox receives the auth variable with
 the placeholder in its boot environment, next to the caller's `env`. On every
 provider but docker-sbx it also receives `HTTPS_PROXY`, `https_proxy`, and
 `NO_PROXY`, so it sends its HTTPS through the proxy at `SECRETS_PROXY_URL`,
-and the proxy's CA in `SECRETS_PROXY_CA`, which it installs at boot.
+and the proxy's CA in `SECRETS_PROXY_CA`, which it installs at boot. A box
+with a `github` secret also points git at gh for its credential and rewrites
+SSH remotes to HTTPS, so git sends the placeholder as a Basic password and
+the proxy swaps it.
 The proxy is the official mitmproxy image with the addon in `deploy/proxy`.
 It terminates TLS only for the hosts the exchange lists at `/upstreams`, the
 hosts with a registered secret, and tunnels every other host blind. For a

docs/deploy.md

@@ -331,7 +331,10 @@ every sandbox with secrets:
 A sandbox with secrets gets the certificate in `SECRETS_PROXY_CA`, base64,
 and installs it at boot with `update-ca-certificates`. `SSL_CERT_FILE`,
 `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, and `NODE_EXTRA_CA_CERTS` point curl,
-Python, and Node at it. Keep `flow_detail` at `1` or below. A
+Python, and Node at it. A sandbox with a `github` secret gets the placeholder
+in `GH_TOKEN`, git pointed at gh, and SSH remotes rewritten to HTTPS. That
+sandbox needs `git` and `gh`. The docker sandbox image has both. Keep
+`flow_detail` at `1` or below. A
 higher level prints request headers, and after the swap those carry the
 real credential. Real credentials exist in three places only. They are
 encrypted in Postgres, they pass through the exchange process for one
@@ -515,7 +518,7 @@ Docker provider:
 
 | Variable | Default | Purpose |
 | --- | --- | --- |
-| `DOCKER_DEFAULT_IMAGE` | `ghcr.io/czpython/drukbox/sandbox:latest` | Sandbox image with sshd; auto-pulled. Build `images/local/Dockerfile` to customize. |
+| `DOCKER_DEFAULT_IMAGE` | `ghcr.io/czpython/drukbox/sandbox:latest` | Sandbox image with sshd, git, and gh; auto-pulled. Build `images/local/Dockerfile` to customize. |
 | `DOCKER_SSH_USERNAME` | `root` | In-container user callers SSH as. |
 | `DOCKER_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | ssh-keyscan retry budget for a fresh container. |
 

docs/security.md

@@ -115,7 +115,9 @@ sandbox cannot reach the exchange or the API through it. Bind the exchange
 process where only the proxy can reach it, because its answer is the real
 credential. A sandbox with secrets installs the proxy's CA at boot, so whoever
 holds the CA key can impersonate any host to that sandbox. The key lives in
-the proxy's volume. The API reads the public certificate only.
+the proxy's volume. The API reads the public certificate only. A sandbox with
+a `github` secret sends git through gh and rewrites SSH remotes to HTTPS, so
+the usual clone and push go through the proxy.
 
 Two pieces of material reach the VM through its provider's user-data /
 setup-script mechanism, and that channel is the relevant exposure:

images/local/Dockerfile

@@ -1,4 +1,4 @@
-# Minimal sandbox image for the local `docker` provider: Ubuntu + sshd.
+# Sandbox image for the local `docker` provider: Ubuntu, sshd, git, and gh.
 # Build once, then point the provider at it:
 #
 #   docker build -t drukbox/sandbox:latest images/local/
@@ -7,8 +7,17 @@
 # drukbox per host) and runs sshd in the foreground. Key-only auth; no password.
 FROM ubuntu:24.04
 
+# gh comes from GitHub's own apt repository, as its install guide says.
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends openssh-server ca-certificates \
+    && apt-get install -y --no-install-recommends openssh-server ca-certificates curl git \
+    && mkdir -p -m 755 /etc/apt/keyrings \
+    && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
+        -o /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+    && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
+        > /etc/apt/sources.list.d/github-cli.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends gh \
     && rm -rf /var/lib/apt/lists/* \
     && mkdir -p /run/sshd
 

images/local/entrypoint.sh

@@ -19,6 +19,14 @@ if [ -n "${SECRETS_PROXY_CA:-}" ]; then
   update-ca-certificates >/dev/null
 fi
 
+# git takes its credential from gh, and SSH remotes go over HTTPS.
+if [ -n "${GH_TOKEN:-}" ]; then
+  git config --system --replace-all credential.https://github.com.helper ''
+  git config --system --add credential.https://github.com.helper '!gh auth git-credential'
+  git config --system --replace-all url.https://github.com/.insteadOf git@github.com:
+  git config --system --add url.https://github.com/.insteadOf ssh://git@github.com/
+fi
+
 # Generate host keys if the image doesn't ship any.
 ssh-keygen -A
 

src/host_secrets/catalog.py

@@ -1,38 +1,48 @@
-from typing import Any, TypedDict
+import base64
+from typing import Any, NamedTuple
 
 BEARER_HEADER = "Authorization"
 BEARER_PREFIX = "Bearer "
 SERVICE_FIELDS = frozenset({"host", "auth_header", "auth_prefix", "auth_variable"})
 
 
-class Service(TypedDict):
+class Upstream(NamedTuple):
     host: str
-    auth_header: str
-    auth_prefix: str
-    auth_variable: str
+    auth_header: str = BEARER_HEADER
+    auth_prefix: str = BEARER_PREFIX
+    # Basic carries the credential as this user's password.
+    basic_user: str = ""
+
+    def credential(self, value: str) -> str:
+        if self.basic_user:
+            return "Basic " + base64.b64encode(f"{self.basic_user}:{value}".encode()).decode()
+        return f"{self.auth_prefix}{value}"
 
 
-def bearer(host: str, auth_variable: str) -> Service:
-    return {
-        "host": host,
-        "auth_header": BEARER_HEADER,
-        "auth_prefix": BEARER_PREFIX,
-        "auth_variable": auth_variable,
-    }
+class Service(NamedTuple):
+    auth_variable: str
+    upstreams: tuple[Upstream, ...]
 
 
 CATALOG: dict[str, Service] = {
-    "anthropic": bearer("api.anthropic.com", "ANTHROPIC_AUTH_TOKEN"),
-    "github": bearer("api.github.com", "GH_TOKEN"),
-    "openai": bearer("api.openai.com", "OPENAI_API_KEY"),
+    "anthropic": Service("ANTHROPIC_AUTH_TOKEN", (Upstream("api.anthropic.com"),)),
+    # git's smart HTTP refuses a bearer. It takes Basic with x-access-token as the user.
+    "github": Service(
+        "GH_TOKEN",
+        (
+            Upstream("api.github.com"),
+            Upstream("uploads.github.com"),
+            Upstream("github.com", basic_user="x-access-token"),
+        ),
+    ),
+    "openai": Service("OPENAI_API_KEY", (Upstream("api.openai.com"),)),
 }
 
 
 def service(name: str, entry: dict[str, Any]) -> Service:
-    fields = entry if "host" in entry else CATALOG[name]
-    return {
-        "host": fields["host"],
-        "auth_header": fields["auth_header"],
-        "auth_prefix": fields["auth_prefix"],
-        "auth_variable": fields["auth_variable"],
-    }
+    if "host" in entry:
+        return Service(
+            entry["auth_variable"],
+            (Upstream(entry["host"], entry["auth_header"], entry["auth_prefix"]),),
+        )
+    return CATALOG[name]

src/host_secrets/tests/test_catalog.py

@@ -1,12 +1,42 @@
-from host_secrets.catalog import CATALOG
+import base64
 
-SERVICE_KEYS = {"host", "auth_header", "auth_prefix", "auth_variable"}
+from host_secrets.catalog import CATALOG, Service, Upstream, service
 
 
 def test_catalog_starts_with_the_three_built_in_services() -> None:
     assert set(CATALOG) == {"anthropic", "github", "openai"}
 
 
-def test_every_entry_describes_a_service_in_full() -> None:
-    for service in CATALOG.values():
-        assert set(service) == SERVICE_KEYS
+def test_every_entry_names_the_variable_a_client_reads_and_at_least_one_host() -> None:
+    for entry in CATALOG.values():
+        assert entry.auth_variable
+        assert entry.upstreams
+
+
+def test_github_is_one_service_with_the_api_uploads_and_git_hosts() -> None:
+    api, uploads, git = CATALOG["github"].upstreams
+
+    assert (api.host, api.credential("ghs_x")) == ("api.github.com", "Bearer ghs_x")
+    assert (uploads.host, uploads.credential("ghs_x")) == ("uploads.github.com", "Bearer ghs_x")
+    assert git.host == "github.com"
+    assert git.credential("ghs_x") == "Basic " + base64.b64encode(b"x-access-token:ghs_x").decode()
+    assert api.auth_header == uploads.auth_header == git.auth_header == "Authorization"
+
+
+def test_a_custom_entry_names_one_host_with_its_own_header_shape() -> None:
+    entry = {
+        "host": "api.acme.test",
+        "auth_header": "x-api-key",
+        "auth_prefix": "",
+        "auth_variable": "ACME_TOKEN",
+        "value": "ak_live",
+    }
+
+    assert service("acme", entry) == Service(
+        "ACME_TOKEN", (Upstream("api.acme.test", "x-api-key", ""),)
+    )
+    assert service("acme", entry).upstreams[0].credential("ak_live") == "ak_live"
+
+
+def test_a_built_in_entry_resolves_through_the_catalog() -> None:
+    assert service("github", {"value": "ghs_x"}) is CATALOG["github"]

src/hosts/tests/test_secrets.py

@@ -42,7 +42,7 @@ async def put_secret(
         self, *, vm: str, service: Service, placeholder: Placeholder, value: str
     ) -> dict[str, str]:
         self.values[placeholder.service] = value
-        return {service["auth_variable"]: str(placeholder)}
+        return {service.auth_variable: str(placeholder)}
 
     async def delete_secret(self, *, vm: str, placeholder: Placeholder) -> None:
         return
@@ -164,7 +164,7 @@ class OversizedInjection(RecordingInjection):
     async def put_secret(
         self, *, vm: str, service: Service, placeholder: Placeholder, value: str
     ) -> dict[str, str]:
-        return {service["auth_variable"]: "a" * 9000}
+        return {service.auth_variable: "a" * 9000}
 
 
 async def test_a_boot_environment_pam_cannot_read_fails_provisioning_before_the_vm(

src/providers/capabilities.py

@@ -65,7 +65,7 @@ async def put_secret(
     ) -> dict[str, str]:
         settings = get_settings()
         return {
-            service["auth_variable"]: str(placeholder),
+            service.auth_variable: str(placeholder),
             "HTTPS_PROXY": settings.secrets_proxy_url,
             "https_proxy": settings.secrets_proxy_url,
             "NO_PROXY": NO_PROXY,

src/providers/docker_sbx/api.py

@@ -66,18 +66,18 @@ async def set_custom_secret(
         self,
         *,
         sandbox: str,
-        host: str,
+        hosts: list[str],
         env: str,
         placeholder: str,
         command: str,
     ) -> None:
+        # One secret covers every host of its service. --host repeats.
         await self._run(
             "secret",
             "set-custom",
             "--sandbox",
             sandbox,
-            "--host",
-            host,
+            *(part for host in hosts for part in ("--host", host)),
             "--env",
             env,
             "--placeholder",

src/providers/docker_sbx/secrets.py

@@ -46,14 +46,14 @@ async def put_secret(
             else:
                 await self.api.set_custom_secret(
                     sandbox=vm,
-                    host=service["host"],
-                    env=service["auth_variable"],
+                    hosts=[upstream.host for upstream in service.upstreams],
+                    env=service.auth_variable,
                     placeholder=str(placeholder),
                     command=command,
                 )
         except DockerSbxProviderError as exc:
             raise ProviderTransportError(str(exc)) from exc
-        return {service["auth_variable"]: str(placeholder)}
+        return {service.auth_variable: str(placeholder)}
 
     async def delete_secret(self, *, vm: str, placeholder: Placeholder) -> None:
         try:

src/providers/docker_sbx/tests/test_api.py

@@ -69,6 +69,43 @@ async def fake_exec(*args, **kwargs):
     process.communicate.assert_awaited_once_with(script.encode())
 
 
+@pytest.mark.asyncio
+async def test_set_custom_secret_names_every_host_of_the_service(monkeypatch):
+    captured: dict = {}
+
+    async def fake_exec(*args, **kwargs):
+        captured["args"] = args
+        return _process()
+
+    monkeypatch.setattr("providers.docker_sbx.api.asyncio.create_subprocess_exec", fake_exec)
+
+    await SbxCLI().set_custom_secret(
+        sandbox="sb-test",
+        hosts=["api.github.com", "github.com"],
+        env="GH_TOKEN",
+        placeholder="drk.a.github.b",
+        command="cat /secrets/sb-test/github",
+    )
+
+    assert captured["args"] == (
+        "sbx",
+        "secret",
+        "set-custom",
+        "--sandbox",
+        "sb-test",
+        "--host",
+        "api.github.com",
+        "--host",
+        "github.com",
+        "--env",
+        "GH_TOKEN",
+        "--placeholder",
+        "drk.a.github.b",
+        "--command",
+        "cat /secrets/sb-test/github",
+    )
+
+
 @pytest.mark.asyncio
 async def test_sandbox_count_reads_the_listing(monkeypatch):
     listing = b'{"sandboxes": [{"name": "sb-a"}, {"name": "sb-b"}]}'

src/providers/docker_sbx/tests/test_secrets.py

@@ -54,7 +54,7 @@ async def test_any_other_secret_is_a_custom_secret_on_its_host_from_a_file(tmp_p
     assert stat.S_IMODE(path.parent.stat().st_mode) == 0o700
     api.set_custom_secret.assert_awaited_once_with(
         sandbox="sb-one",
-        host="api.anthropic.com",
+        hosts=["api.anthropic.com"],
         env="ANTHROPIC_AUTH_TOKEN",
         placeholder=str(placeholder),
         command=f"cat {path}",