Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
# • Agent : a main tag (no extension-specific changes)
# ─────────────────────────────────────────────────────────────────────────────

STUDIO_TAG=dev-1.0.6-45708c38
AGENT_TAG=dev-0.1.2-4b7fcf1
UI_TAG=e67ae329e9466304450e68a59e4f59d8bc97cb05
XTERM_TAG=dev-0.1.2-6cdf181
STUDIO_TAG=0.2.24
AGENT_TAG=0.2.24
UI_TAG=0.2.24
XTERM_TAG=0.2.24

# Qdrant vector database (Knowledge Base). The API key is fixed and not a secret — Qdrant is bound to
# the compose network and localhost only. Change it if you like; compose and scripts/register-qdrant.sh
Expand Down Expand Up @@ -46,12 +46,14 @@ QDRANT_API_KEY=devkit-local-qdrant
#RUNTIME=docker

# Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an
# extension — the platform itself doesn't use it. An already-pulled tag is reused without checking the
# registry: BUILDER_PULL=1 forces a re-pull (this tag moves), BUILDER_IMAGE=<ref> replaces the image
# extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh,
# so every build of a version uses one toolchain. BUILDER_TAG=<tag> (e.g. latest) opts out of the pin; an
# already-pulled tag is reused without checking the registry, and BUILDER_PULL=1 forces a re-pull (tags
# move). BUILDER_IMAGE=<ref> replaces the image
# outright, BUILDER_PLATFORM=<os/arch> forces an architecture. Each build's package caches live in
# per-uid named volumes that nothing prunes — `<runtime> volume ls | grep duplo_devkit` to find and
# `<runtime> volume rm` to reclaim them (each holds a full NuGet + npm closure).
BUILDER_TAG=latest
# BUILDER_TAG=latest

# Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8
# and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/builder-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
username: ${{ secrets.QUAY_USERNAME }}
password: ${{ secrets.QUAY_PASSWORD }}
- uses: docker/build-push-action@v7
id: push
with:
# `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the
# Dockerfile COPYs nothing from the build context (only --from=node), so sending the whole
Expand All @@ -61,3 +62,16 @@ jobs:
tags: |
quay.io/duplocloud/duplo-extension-builder:latest
quay.io/duplocloud/duplo-extension-builder:${{ github.sha }}
# scripts/_builder.sh pins the default toolchain by digest, so a publish here changes nothing until that pin
# moves in a reviewed PR. Hand the reviewer the exact value.
- name: Report the digest to pin
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: |
{
echo "### Builder image published"
echo "To build with it by default, set in scripts/_builder.sh and docker-compose.yml:"
echo '```'
echo "quay.io/duplocloud/duplo-extension-builder@${DIGEST}"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
9 changes: 9 additions & 0 deletions .github/workflows/extension-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ jobs:
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
steps:
- uses: actions/checkout@v7
with:
# The PR's base history, so the release checks below can diff each extension against the merge base.
fetch-depth: 0

- name: Discover extensions
id: guard
Expand Down Expand Up @@ -65,6 +68,12 @@ jobs:
if: steps.guard.outputs.build == 'true'
run: ./scripts/build-all.sh

# Warnings only: what each changed extension's release will need (version bump, declared skills, Native
# Federation frontend). !cancelled() so a failed build still reports the checks that need no bundle.
- name: Release checks
if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.guard.outputs.build == 'true' }}
run: ./scripts/check-extension-pr.sh "origin/${{ github.base_ref }}"

- name: Upload bundles
if: steps.guard.outputs.build == 'true'
uses: actions/upload-artifact@v7
Expand Down
55 changes: 13 additions & 42 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
# Publishes each extension in this repo as a GitHub Release on push to main.
#
# For every extensions/<name>/ it builds the bundle and, ONLY when manifest.version is new (its
# <name>-v<version> tag doesn't exist yet), cuts a GitHub Release with extension.zip attached. Bumping
# manifest.version is therefore how you publish a new version to the extension library.
# For every extensions/<name>/ it builds the bundle and publishes one release per build, tagged
# <name>-v<version>-sdk-<sdkVersion> at the commit that was built. scripts/release-extensions.sh holds the rules:
# a build already released is skipped, and an extension whose content changed at a version that already has a
# release fails the job, so bumping manifest.version is how you publish a new version to the extension library.
#
# This file ships with the dev-kit. After scripts/init-project.sh makes the repo yours, set these repo secrets
# (same as Extension CI): DUPLO_HOST + DUPLO_TOKEN. The build fetches the host SDK feed from DUPLO_HOST, so it
# must be reachable from CI (a public DuploCloud, not localhost).
# must be reachable from CI (a public DuploCloud, not localhost). Turn on immutable releases in the repository's
# settings, too; the publish step warns on every release that lands mutable.
#
# NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not
# auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the
Expand All @@ -30,6 +32,10 @@ jobs:
DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }}
steps:
- uses: actions/checkout@v7
with:
# Every tag and the history it points at: the publish step compares each released version's commit with
# this one to tell an unchanged extension from one that changed without a version bump.
fetch-depth: 0

- name: Discover extensions
id: guard
Expand All @@ -39,15 +45,7 @@ jobs:
if [ ${#dirs[@]} -gt 0 ]; then echo "build=true" >> "$GITHUB_OUTPUT"
else echo "build=false" >> "$GITHUB_OUTPUT"; echo "::notice::No extensions/*/manifest.json — nothing to release."; fi

- uses: actions/setup-dotnet@v6
if: steps.guard.outputs.build == 'true'
with:
dotnet-version: '8.0.x'
- uses: actions/setup-node@v7
if: steps.guard.outputs.build == 'true'
with:
node-version: '22'

# Runs in the builder container (scripts/_builder.sh) on the runner's Docker, with the digest-pinned toolchain.
- name: Build all extensions
if: steps.guard.outputs.build == 'true'
run: ./scripts/build-all.sh
Expand All @@ -58,35 +56,8 @@ jobs:
# if: steps.guard.outputs.build == 'true'
# run: ./scripts/deploy-all.sh

- name: Publish GitHub releases (only for new manifest.version)
- name: Publish GitHub releases (one per new build)
if: steps.guard.outputs.build == 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
shopt -s nullglob
published=0; skipped=0
for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do
[ -f "$m" ] || continue
dir="$(dirname "$m")"
zip="$dir/dist/extension.zip"
if [ ! -f "$zip" ]; then
echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping."
continue
fi
name="$(jq -r '.name // (.id)' "$m")"
version="$(jq -r '.version' "$m")"
slug="$(basename "$dir")"
tag="${slug}-v${version}"
if gh release view "$tag" >/dev/null 2>&1; then
echo "==> $tag already released — skipping (bump manifest.version to publish a new one)."
skipped=$((skipped+1))
continue
fi
echo "==> Publishing release $tag ($name $version)"
gh release create "$tag" "$zip" \
--title "$name $version" \
--notes "Automated release of $name v$version from $slug (extension.zip attached)."
published=$((published+1))
done
echo "==> Released $published, skipped $skipped (already-published versions)."
run: ./scripts/release-extensions.sh
58 changes: 50 additions & 8 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@ services:
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
- AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
# Docker-in-Docker: the dind service shares this container's network namespace, so its daemon
# is reachable on loopback here and nowhere else.
- DOCKER_HOST=tcp://127.0.0.1:2375
- CCA_ENABLE_DOCKER=true
volumes:
# Shared /data so the agent reads the skills the studio seeded (CCA reads /data/skills).
- platform_data:/data
Expand Down Expand Up @@ -166,6 +170,46 @@ services:
- ./nginx/default.conf:/tpl/default.conf:ro
- nginx_conf:/out

# Docker-in-Docker daemon for the agent, mirroring the dind sidecar in the helpdesk-helm chart.
#
# network_mode joins claude-code-agent's network namespace — the compose equivalent of a pod's
# shared netns — so dockerd bound to 127.0.0.1 is reachable by the agent and by nothing else: not
# the compose network, not the host. The agent keeps owning its network identity, so
# http://claude-code-agent:8000 still resolves for the studio. The agent cannot depends_on this
# service (network_mode already makes this one depend on the agent); docker calls simply fail for
# the few seconds until the daemon is up.
#
# The image entrypoint is bypassed: with TLS off it force-binds tcp://0.0.0.0:2375 and ignores any
# --host passed. modprobe replicates the entrypoint's kernel-module loading; --tls=false suppresses
# the insecure-bind startup delay.
dind:
image: docker:27-dind
restart: unless-stopped
privileged: true
network_mode: "service:claude-code-agent"
entrypoint: ["/bin/sh", "-c"]
command:
- |
modprobe ip_tables || true
modprobe ip6_tables || true
modprobe nf_tables || true
modprobe overlay || true
exec dockerd --host=tcp://127.0.0.1:2375 --tls=false
environment:
- DOCKER_HOST=tcp://127.0.0.1:2375
healthcheck:
test: ["CMD", "docker", "info"]
interval: 10s
start_period: 5s
volumes:
# Same /data as the agent, so `docker run -v /data/...` bind-mount paths resolve daemon-side.
- platform_data:/data
# Image/container storage. Wiped with the rest by run.sh's `down -v`, like the chart's emptyDir.
- dind_storage:/var/lib/docker
depends_on:
claude-code-agent:
condition: service_started

duplo-ui:
image: quay.io/duplocloud/duplo-ai-helpdesk-ui:${UI_TAG}
restart: unless-stopped
Expand Down Expand Up @@ -204,14 +248,11 @@ services:
# below). Both routes are live; the compose network is simply the preferred one.
builder:
profiles: [tools]
# Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-latest}}:
# podman-compose 1.5 stops at the first '}' and appends the remainder literally, yielding
# "…busybox:1.36}" and an "invalid reference format" error even when BUILDER_IMAGE is set. Nothing
# is lost — BUILDER_TAG is resolved into BUILDER_IMAGE by scripts/_builder.sh:builder_resolve_image,
# which always exports it before compose is invoked, so the default below is only ever reached by a
# hand-run `compose run` with no BUILDER_IMAGE set (where it now means :latest regardless of
# BUILDER_TAG — use the build scripts, or set BUILDER_IMAGE outright, to pin another tag).
image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder:latest}
# Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-…}}: podman-compose 1.5
# stops at the first '}' and appends the remainder literally. scripts/_builder.sh resolves BUILDER_TAG and exports
# BUILDER_IMAGE before every build; this default is only for a hand-run `compose run builder`, and must match
# _BUILDER_PINNED_IMAGE there.
image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd}
# Unset by default, so the build runs on the host's native architecture. Set BUILDER_PLATFORM only
# to force one (e.g. linux/amd64 if a publish ever ends up amd64-only) — an unset value is dropped
# by compose, whereas copying duplo-ai-studio's linux/amd64 default would silently emulate a .NET
Expand Down Expand Up @@ -260,6 +301,7 @@ volumes:
extension_studio_data:
claude_sessions:
qdrant_data:
dind_storage:
# Keyed by uid on purpose: these are written as the calling user (see the service's `user:`), and one
# uid's runtime-created cache subdirectories are not writable by another, so sharing a single pair
# would break the second uid to build. build/Dockerfile.builder has the full reasoning, including why
Expand Down
1 change: 1 addition & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Nothing here builds from source. Every service pulls a published image pinned by
| `duplo-ai-studio` | `quay.io/duplocloud/backend` | 60031 | The platform API, serving `/v1/aiservicedesk`. Owns extensions, workspaces, resources, and tickets, and enforces the license in `Licensing__Token`. |
| `duplo-ui` | `quay.io/duplocloud/duplo-ai-helpdesk-ui` | 4210 | The Angular portal. The image serves the SPA only; the mounted `nginx/default.conf` proxies API prefixes to the studio, so the browser sees one origin and there is no CORS. |
| `claude-code-agent` | `quay.io/duplocloud/duplo-agent` | 8010 | The LLM agent that executes provisioning tickets. Calls back to the studio at `http://duplo-ai-studio:60021`. |
| `dind` | `docker:27-dind` | — | Docker-in-Docker daemon for the agent, mirroring the helpdesk-helm chart's sidecar. Shares `claude-code-agent`'s network namespace and listens on its loopback only (`DOCKER_HOST=tcp://127.0.0.1:2375`), so the agent can run containers and nothing else can reach the daemon. Mounts the same `/data` so bind-mount paths match. Runs privileged. |
| `mongo` | `mongo:7.0.14` | 27018 | All platform state, including the `loaded_extensions` records. Matches the version pinned in the helpdesk-helm chart. |
| `qdrant` | `qdrant/qdrant:v1.18.2` | 6333 | The vector database behind the Knowledge Base. The studio reaches it in-network at `http://qdrant:6333`; the host port is published so its dashboard is available for debugging. Runs with RBAC on, so the studio can mint read-only per-collection tokens instead of sharing the master key. |
| `xterm` | `quay.io/duplocloud/duplo-xterm` | 6061 | In-browser terminal. The browser reaches it on the published host port, so `AIStudio__XtermHost` must be host-reachable. |
Expand Down
2 changes: 1 addition & 1 deletion docs/cli-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ extensions, Mongo, the file store, and the Knowledge Base vectors are lost.
```

A thin wrapper around `docker compose logs -f`; every argument is passed straight through. Services are
`duplo-ai-studio`, `claude-code-agent`, `duplo-ui`, `mongo`, `qdrant`, and `xterm`.
`duplo-ai-studio`, `claude-code-agent`, `dind`, `duplo-ui`, `mongo`, `qdrant`, and `xterm`.

## What each command destroys

Expand Down
2 changes: 1 addition & 1 deletion logs.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Tail logs. Optional service name: duplo-ai-studio | claude-code-agent | duplo-ui | mongo | xterm
# Tail logs. Optional service name: duplo-ai-studio | claude-code-agent | dind | duplo-ui | mongo | xterm
set -euo pipefail
cd "$(dirname "$0")"
. ./scripts/_runtime.sh # → $RUNTIME (docker | podman)
Expand Down
34 changes: 23 additions & 11 deletions scripts/_builder.sh
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,13 @@ builder_keep_local_copy() { # builder_keep_local_copy <pull-error-text>
echo " $RUNTIME pull said: $(printf '%s' "${1-}" | tail -1)" >&2
}

# The toolchain image a build uses when neither BUILDER_IMAGE nor BUILDER_TAG is set, pinned by digest so two
# builds of one extension version compile with the same toolchain. `:latest` moves on every toolchain publish,
# which would let two builds of one version differ. The digest is the multi-arch index, so it resolves on amd64
# and arm64 alike. .github/workflows/builder-image.yml prints the new digest after each publish. Moving this pin
# (and docker-compose.yml's matching default) is a reviewed change, since it changes what every extension builds with.
_BUILDER_PINNED_IMAGE="quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd"

builder_resolve_image() {
# The raw value is kept for the error message: BUILDER_PULL may have come from .env rather than the
# environment, and naming a value the user cannot see in their shell is not a diagnostic.
Expand All @@ -269,17 +276,22 @@ builder_resolve_image() {
return 1
fi

local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"; tag="${tag:-latest}"
# OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is
# a pull that fails for an unobvious reason and a silent fall-through to building something else.
case "$tag" in
*[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* )
echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2
echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2
echo " Windows line endings." >&2
return 1 ;;
esac
BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"; export BUILDER_IMAGE
local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"
if [ -z "$tag" ]; then
BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE"
else
# OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is
# a pull that fails for an unobvious reason and a silent fall-through to building something else.
case "$tag" in
*[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* )
echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2
echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2
echo " Windows line endings." >&2
return 1 ;;
esac
BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"
fi
export BUILDER_IMAGE
if [ "$force_pull" != 1 ] && "$RUNTIME" image inspect "$BUILDER_IMAGE" >/dev/null 2>&1; then return 0; fi

echo "==> Fetching the build toolchain image ($BUILDER_IMAGE)" >&2
Expand Down
28 changes: 28 additions & 0 deletions scripts/_sdk_digest.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# shellcheck shell=bash
# Sourced by scripts/build-extension.sh — fingerprints the SDK feed a bundle was compiled against.
#
# sdkVersion alone does not identify an SDK: the host's SDK surface and dependency floors can move under an
# unchanged version, so a bundle that fails at run time could be a broken extension or one built against a drifted
# SDK. build-extension.sh stamps this digest beside sdkVersion so the two can be told apart by recomputing it over
# the host's own sdk-bundle.
#
# Definition (anything recomputing it must match this exactly): for every file in the extracted SDK bundle, the line
# "<sha256 hex> <path relative to the bundle root>\n"; the lines sorted bytewise by path; sdkDigest is the lowercase
# sha256 hex of their concatenation. That is `sha256sum` output over the sorted file list, hashed once more.

# _sdk_sha256 [file] -> lowercase sha256 hex of the file (or stdin). sha256sum on Linux, shasum on macOS.
_sdk_sha256() {
if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" | cut -d' ' -f1
else shasum -a 256 "$@" | cut -d' ' -f1; fi
}

# sdk_digest <extracted-sdk-bundle-dir> -> sdkDigest on stdout. Fails on a directory with no files.
sdk_digest() {
local root="${1:?usage: sdk_digest <dir>}" files
files="$(cd "$root" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)"
[ -n "$files" ] || { echo "ERROR: no files under $root to digest" >&2; return 1; }
local f
while IFS= read -r f; do
printf '%s %s\n' "$(_sdk_sha256 "$root/$f")" "$f"
done <<<"$files" | _sdk_sha256
}
Loading
Loading